fix(dedup): parse vector field in Rust IPC Search results
RustLanceDBClient.Search decoded the Rust IPC response into a struct
that had no vector field, so every MemoryRecord returned to Go had an
empty Vector. Consequence chain:
len(results[i].Vector) > 0 && len(results[j].Vector) > 0
always false → the cosine branch was skipped in:
- commit path near_dup (core.go:634)
- distill direct-write dedup gate (server.go:179) <-- the fix I just
deployed this round was silently a no-op
- admin/dedup endpoint (core.go:634) <-- merged≈0 false
negative
Without this, no near-duplicate detection in the whole system works —
it silently fell back to exact string comparison only.
Retained: the vector is forwarded from the raw IPC JSON; if Rust stops
returning it, the degenerate behaviour is exact-string comparison
(which still works), rather than a crash.
This commit is contained in:
@@ -34,24 +34,24 @@ func NewRustLanceDBClient(sockPath string, emb *Embedder) *RustLanceDBClient {
|
||||
// ─── IPC 协议 ────────────────────────────────────────────
|
||||
|
||||
type ipcReq struct {
|
||||
Type string `json:"type"`
|
||||
Table string `json:"table,omitempty"`
|
||||
Records string `json:"records,omitempty"`
|
||||
Vector []float32 `json:"vector,omitempty"`
|
||||
TopK int `json:"top_k,omitempty"`
|
||||
Namespace string `json:"namespace,omitempty"`
|
||||
ID string `json:"id,omitempty"`
|
||||
Fields string `json:"fields,omitempty"`
|
||||
MinRecall int `json:"min_recall,omitempty"`
|
||||
QueryLimit int `json:"limit,omitempty"`
|
||||
Items string `json:"items,omitempty"` // lancedb_update_batch: [{"id":..,"delta":N}]
|
||||
TS string `json:"ts,omitempty"` // lancedb_update_batch: last_recalled_at
|
||||
Type string `json:"type"`
|
||||
Table string `json:"table,omitempty"`
|
||||
Records string `json:"records,omitempty"`
|
||||
Vector []float32 `json:"vector,omitempty"`
|
||||
TopK int `json:"top_k,omitempty"`
|
||||
Namespace string `json:"namespace,omitempty"`
|
||||
ID string `json:"id,omitempty"`
|
||||
Fields string `json:"fields,omitempty"`
|
||||
MinRecall int `json:"min_recall,omitempty"`
|
||||
QueryLimit int `json:"limit,omitempty"`
|
||||
Items string `json:"items,omitempty"` // lancedb_update_batch: [{"id":..,"delta":N}]
|
||||
TS string `json:"ts,omitempty"` // lancedb_update_batch: last_recalled_at
|
||||
}
|
||||
|
||||
type ipcResp struct {
|
||||
Status string `json:"status"`
|
||||
ReportJSON string `json:"report_json"` // consolidator uses this
|
||||
Result string `json:"result"` // lancedb search uses this
|
||||
ReportJSON string `json:"report_json"` // consolidator uses this
|
||||
Result string `json:"result"` // lancedb search uses this
|
||||
FailureStep string `json:"failure_step"`
|
||||
ErrorDetail string `json:"error_detail"`
|
||||
}
|
||||
@@ -187,6 +187,11 @@ func (rc *RustLanceDBClient) Search(table string, vector []float32, topK int, na
|
||||
VolatileFlag bool `json:"volatile_flag"`
|
||||
Tier string `json:"tier"`
|
||||
LastRecalledAt string `json:"last_recalled_at"`
|
||||
// vector 必须解析:近重复去重(commit near_dup / 蒸馏写入 gate /
|
||||
// admin/dedup 端点)全部依赖 records 携带向量做 cosine 比较。
|
||||
// 此前缺失该字段 → Vector 恒为空 → 所有 len(Vector)>0 判定失败 →
|
||||
// 去重静默退化为「只比字符串相等」,端点恒报 merged≈0(假阴性)。
|
||||
Vector []float32 `json:"vector"`
|
||||
}
|
||||
json.Unmarshal([]byte(resp.ReportJSON), &raw)
|
||||
if len(raw) == 0 {
|
||||
@@ -258,7 +263,8 @@ func (rc *RustLanceDBClient) Search(table string, vector []float32, topK int, na
|
||||
CreatedAt: parseTime(r.CreatedAt), UpdatedAt: parseTime(r.UpdatedAt),
|
||||
QualityScore: finalQuality, Importance: r.Importance,
|
||||
RecallCount: int(r.RecallCount), VolatileFlag: r.VolatileFlag,
|
||||
Tier: r.Tier,
|
||||
Tier: r.Tier,
|
||||
Vector: r.Vector, // 去重判据依赖,勿删
|
||||
})
|
||||
if len(out) >= topK {
|
||||
break
|
||||
@@ -437,16 +443,16 @@ func (rc *RustLanceDBClient) Update(table, id string, fields map[string]any) err
|
||||
log.Printf("[ipc] Update: id=%s fields_count=%d", id, len(fieldList))
|
||||
log.Printf("[ipc] Update fields_json: %s", string(fj))
|
||||
_, err := rc.rpc(ipcReq{
|
||||
Type: "lancedb_update",
|
||||
Table: table,
|
||||
ID: id,
|
||||
Fields: string(fj),
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("[ipc] Update FAILED: id=%s fields=%s error=%v", id, string(fj), err)
|
||||
} else {
|
||||
log.Printf("[ipc] Update OK: id=%s fields=%s", id, string(fj))
|
||||
}
|
||||
Type: "lancedb_update",
|
||||
Table: table,
|
||||
ID: id,
|
||||
Fields: string(fj),
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("[ipc] Update FAILED: id=%s fields=%s error=%v", id, string(fj), err)
|
||||
} else {
|
||||
log.Printf("[ipc] Update OK: id=%s fields=%s", id, string(fj))
|
||||
}
|
||||
}
|
||||
_local.mu.Unlock()
|
||||
return nil
|
||||
@@ -489,9 +495,13 @@ func (rc *RustLanceDBClient) GetTopByQuality(agentID string, limit int) ([]model
|
||||
defer _local.mu.RUnlock()
|
||||
var out []models.MemoryRecord
|
||||
for _, m := range _local.memories {
|
||||
if m.IsDeleted { continue }
|
||||
if m.IsDeleted {
|
||||
continue
|
||||
}
|
||||
out = append(out, *m)
|
||||
if len(out) >= limit { break }
|
||||
if len(out) >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -516,7 +526,6 @@ func (rc *RustLanceDBClient) SoftDelete(id, reason string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
// TODO: 需要 Rust 侧支持 lancedb_update 才能持久化 SoftDelete
|
||||
// 当前仅标记内存,重启后失效。测试数据已直接通过 Python LanceDB API 清理。
|
||||
|
||||
@@ -536,7 +545,9 @@ func (rc *RustLanceDBClient) SoftDeleteBatch(table string, ids []string, updated
|
||||
return updated, nil
|
||||
}
|
||||
|
||||
func (rc *RustLanceDBClient) GetVersionHistory(id string) ([]map[string]interface{}, error) { return nil, nil }
|
||||
func (rc *RustLanceDBClient) GetVersionHistory(id string) ([]map[string]interface{}, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (rc *RustLanceDBClient) GetCandidatesForForgetting() ([]map[string]interface{}, error) {
|
||||
// P2 2026-09-06: 走 Rust 全表扫描(安全版: limit 2000 + 跳 vector + 真读 last_recalled_at)。
|
||||
// 替代纯缓存遍历 — 缓存只含近期 commit/recall/search 触碰的记忆, 到期旧记忆扫不到 → 9-25 遗忘闭环断。
|
||||
@@ -670,17 +681,17 @@ func (rc *RustLanceDBClient) GetSkillCandidates(minRecalls, limit int) ([]models
|
||||
records := make([]models.MemoryRecord, 0, len(rawRecords))
|
||||
for _, r := range rawRecords {
|
||||
m := models.MemoryRecord{
|
||||
ID: mapStr(r, "id"),
|
||||
AgentID: mapStr(r, "agent_id"),
|
||||
Namespace: mapStr(r, "namespace"),
|
||||
Content: mapStr(r, "content"),
|
||||
Category: mapStr(r, "category"),
|
||||
Tier: mapStr(r, "tier"),
|
||||
Source: mapStr(r, "source"),
|
||||
Freshness: mapStr(r, "freshness"),
|
||||
DerivedFrom: mapStr(r, "derived_from"),
|
||||
ID: mapStr(r, "id"),
|
||||
AgentID: mapStr(r, "agent_id"),
|
||||
Namespace: mapStr(r, "namespace"),
|
||||
Content: mapStr(r, "content"),
|
||||
Category: mapStr(r, "category"),
|
||||
Tier: mapStr(r, "tier"),
|
||||
Source: mapStr(r, "source"),
|
||||
Freshness: mapStr(r, "freshness"),
|
||||
DerivedFrom: mapStr(r, "derived_from"),
|
||||
VersionHistory: nil,
|
||||
DependsOn: nil,
|
||||
DependsOn: nil,
|
||||
}
|
||||
m.QualityScore = mapFloat(r, "quality_score")
|
||||
m.Importance = mapFloat(r, "importance")
|
||||
@@ -723,20 +734,28 @@ func (rc *RustLanceDBClient) Backup(path string) error { return nil }
|
||||
func (rc *RustLanceDBClient) GetAuditLog(limit int) ([]map[string]interface{}, error) {
|
||||
_local.mu.RLock()
|
||||
defer _local.mu.RUnlock()
|
||||
if limit > len(_local.auditLog) { limit = len(_local.auditLog) }
|
||||
if limit > len(_local.auditLog) {
|
||||
limit = len(_local.auditLog)
|
||||
}
|
||||
return _local.auditLog[:limit], nil
|
||||
}
|
||||
func (rc *RustLanceDBClient) IncrementUseful(id string) {
|
||||
_local.mu.Lock(); defer _local.mu.Unlock()
|
||||
if m, ok := _local.memories[id]; ok { m.RecallCount++ }
|
||||
_local.mu.Lock()
|
||||
defer _local.mu.Unlock()
|
||||
if m, ok := _local.memories[id]; ok {
|
||||
m.RecallCount++
|
||||
}
|
||||
}
|
||||
func (rc *RustLanceDBClient) IncrementNotUseful(id string) {}
|
||||
func (rc *RustLanceDBClient) UpdateMemoryContent(id, newContent, source string) error {
|
||||
_local.mu.Lock(); defer _local.mu.Unlock()
|
||||
_local.mu.Lock()
|
||||
defer _local.mu.Unlock()
|
||||
if m, ok := _local.memories[id]; ok {
|
||||
m.Content = newContent
|
||||
if rc.embedder != nil {
|
||||
if vec, err := rc.embedder.EncodeSingle(newContent); err == nil { m.Vector = vec }
|
||||
if vec, err := rc.embedder.EncodeSingle(newContent); err == nil {
|
||||
m.Vector = vec
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
||||
Reference in New Issue
Block a user