fix(dedup): parse vector field in Rust IPC Search results

RustLanceDBClient.Search decoded the Rust IPC response into a struct
that had no vector field, so every MemoryRecord returned to Go had an
empty Vector. Consequence chain:

  len(results[i].Vector) > 0 && len(results[j].Vector) > 0
always false → the cosine branch was skipped in:
  - commit path near_dup (core.go:634)
  - distill direct-write dedup gate (server.go:179)  <-- the fix I just
    deployed this round was silently a no-op
  - admin/dedup endpoint (core.go:634)                <-- merged≈0 false
    negative

Without this, no near-duplicate detection in the whole system works —
it silently fell back to exact string comparison only.

Retained: the vector is forwarded from the raw IPC JSON; if Rust stops
returning it, the degenerate behaviour is exact-string comparison
(which still works), rather than a crash.
This commit is contained in:
小唯
2026-09-20 01:07:27 +08:00
parent 4af85fa093
commit ec17c836a1
+63 -44
View File
@@ -34,24 +34,24 @@ func NewRustLanceDBClient(sockPath string, emb *Embedder) *RustLanceDBClient {
// ─── IPC 协议 ────────────────────────────────────────────
type ipcReq struct {
Type string `json:"type"`
Table string `json:"table,omitempty"`
Records string `json:"records,omitempty"`
Vector []float32 `json:"vector,omitempty"`
TopK int `json:"top_k,omitempty"`
Namespace string `json:"namespace,omitempty"`
ID string `json:"id,omitempty"`
Fields string `json:"fields,omitempty"`
MinRecall int `json:"min_recall,omitempty"`
QueryLimit int `json:"limit,omitempty"`
Items string `json:"items,omitempty"` // lancedb_update_batch: [{"id":..,"delta":N}]
TS string `json:"ts,omitempty"` // lancedb_update_batch: last_recalled_at
Type string `json:"type"`
Table string `json:"table,omitempty"`
Records string `json:"records,omitempty"`
Vector []float32 `json:"vector,omitempty"`
TopK int `json:"top_k,omitempty"`
Namespace string `json:"namespace,omitempty"`
ID string `json:"id,omitempty"`
Fields string `json:"fields,omitempty"`
MinRecall int `json:"min_recall,omitempty"`
QueryLimit int `json:"limit,omitempty"`
Items string `json:"items,omitempty"` // lancedb_update_batch: [{"id":..,"delta":N}]
TS string `json:"ts,omitempty"` // lancedb_update_batch: last_recalled_at
}
type ipcResp struct {
Status string `json:"status"`
ReportJSON string `json:"report_json"` // consolidator uses this
Result string `json:"result"` // lancedb search uses this
ReportJSON string `json:"report_json"` // consolidator uses this
Result string `json:"result"` // lancedb search uses this
FailureStep string `json:"failure_step"`
ErrorDetail string `json:"error_detail"`
}
@@ -187,6 +187,11 @@ func (rc *RustLanceDBClient) Search(table string, vector []float32, topK int, na
VolatileFlag bool `json:"volatile_flag"`
Tier string `json:"tier"`
LastRecalledAt string `json:"last_recalled_at"`
// vector 必须解析:近重复去重(commit near_dup / 蒸馏写入 gate /
// admin/dedup 端点)全部依赖 records 携带向量做 cosine 比较。
// 此前缺失该字段 → Vector 恒为空 → 所有 len(Vector)>0 判定失败 →
// 去重静默退化为「只比字符串相等」,端点恒报 merged≈0(假阴性)。
Vector []float32 `json:"vector"`
}
json.Unmarshal([]byte(resp.ReportJSON), &raw)
if len(raw) == 0 {
@@ -258,7 +263,8 @@ func (rc *RustLanceDBClient) Search(table string, vector []float32, topK int, na
CreatedAt: parseTime(r.CreatedAt), UpdatedAt: parseTime(r.UpdatedAt),
QualityScore: finalQuality, Importance: r.Importance,
RecallCount: int(r.RecallCount), VolatileFlag: r.VolatileFlag,
Tier: r.Tier,
Tier: r.Tier,
Vector: r.Vector, // 去重判据依赖,勿删
})
if len(out) >= topK {
break
@@ -437,16 +443,16 @@ func (rc *RustLanceDBClient) Update(table, id string, fields map[string]any) err
log.Printf("[ipc] Update: id=%s fields_count=%d", id, len(fieldList))
log.Printf("[ipc] Update fields_json: %s", string(fj))
_, err := rc.rpc(ipcReq{
Type: "lancedb_update",
Table: table,
ID: id,
Fields: string(fj),
})
if err != nil {
log.Printf("[ipc] Update FAILED: id=%s fields=%s error=%v", id, string(fj), err)
} else {
log.Printf("[ipc] Update OK: id=%s fields=%s", id, string(fj))
}
Type: "lancedb_update",
Table: table,
ID: id,
Fields: string(fj),
})
if err != nil {
log.Printf("[ipc] Update FAILED: id=%s fields=%s error=%v", id, string(fj), err)
} else {
log.Printf("[ipc] Update OK: id=%s fields=%s", id, string(fj))
}
}
_local.mu.Unlock()
return nil
@@ -489,9 +495,13 @@ func (rc *RustLanceDBClient) GetTopByQuality(agentID string, limit int) ([]model
defer _local.mu.RUnlock()
var out []models.MemoryRecord
for _, m := range _local.memories {
if m.IsDeleted { continue }
if m.IsDeleted {
continue
}
out = append(out, *m)
if len(out) >= limit { break }
if len(out) >= limit {
break
}
}
return out, nil
}
@@ -516,7 +526,6 @@ func (rc *RustLanceDBClient) SoftDelete(id, reason string) error {
return nil
}
// TODO: 需要 Rust 侧支持 lancedb_update 才能持久化 SoftDelete
// 当前仅标记内存,重启后失效。测试数据已直接通过 Python LanceDB API 清理。
@@ -536,7 +545,9 @@ func (rc *RustLanceDBClient) SoftDeleteBatch(table string, ids []string, updated
return updated, nil
}
func (rc *RustLanceDBClient) GetVersionHistory(id string) ([]map[string]interface{}, error) { return nil, nil }
func (rc *RustLanceDBClient) GetVersionHistory(id string) ([]map[string]interface{}, error) {
return nil, nil
}
func (rc *RustLanceDBClient) GetCandidatesForForgetting() ([]map[string]interface{}, error) {
// P2 2026-09-06: 走 Rust 全表扫描(安全版: limit 2000 + 跳 vector + 真读 last_recalled_at)。
// 替代纯缓存遍历 — 缓存只含近期 commit/recall/search 触碰的记忆, 到期旧记忆扫不到 → 9-25 遗忘闭环断。
@@ -670,17 +681,17 @@ func (rc *RustLanceDBClient) GetSkillCandidates(minRecalls, limit int) ([]models
records := make([]models.MemoryRecord, 0, len(rawRecords))
for _, r := range rawRecords {
m := models.MemoryRecord{
ID: mapStr(r, "id"),
AgentID: mapStr(r, "agent_id"),
Namespace: mapStr(r, "namespace"),
Content: mapStr(r, "content"),
Category: mapStr(r, "category"),
Tier: mapStr(r, "tier"),
Source: mapStr(r, "source"),
Freshness: mapStr(r, "freshness"),
DerivedFrom: mapStr(r, "derived_from"),
ID: mapStr(r, "id"),
AgentID: mapStr(r, "agent_id"),
Namespace: mapStr(r, "namespace"),
Content: mapStr(r, "content"),
Category: mapStr(r, "category"),
Tier: mapStr(r, "tier"),
Source: mapStr(r, "source"),
Freshness: mapStr(r, "freshness"),
DerivedFrom: mapStr(r, "derived_from"),
VersionHistory: nil,
DependsOn: nil,
DependsOn: nil,
}
m.QualityScore = mapFloat(r, "quality_score")
m.Importance = mapFloat(r, "importance")
@@ -723,20 +734,28 @@ func (rc *RustLanceDBClient) Backup(path string) error { return nil }
func (rc *RustLanceDBClient) GetAuditLog(limit int) ([]map[string]interface{}, error) {
_local.mu.RLock()
defer _local.mu.RUnlock()
if limit > len(_local.auditLog) { limit = len(_local.auditLog) }
if limit > len(_local.auditLog) {
limit = len(_local.auditLog)
}
return _local.auditLog[:limit], nil
}
func (rc *RustLanceDBClient) IncrementUseful(id string) {
_local.mu.Lock(); defer _local.mu.Unlock()
if m, ok := _local.memories[id]; ok { m.RecallCount++ }
_local.mu.Lock()
defer _local.mu.Unlock()
if m, ok := _local.memories[id]; ok {
m.RecallCount++
}
}
func (rc *RustLanceDBClient) IncrementNotUseful(id string) {}
func (rc *RustLanceDBClient) UpdateMemoryContent(id, newContent, source string) error {
_local.mu.Lock(); defer _local.mu.Unlock()
_local.mu.Lock()
defer _local.mu.Unlock()
if m, ok := _local.memories[id]; ok {
m.Content = newContent
if rc.embedder != nil {
if vec, err := rc.embedder.EncodeSingle(newContent); err == nil { m.Vector = vec }
if vec, err := rc.embedder.EncodeSingle(newContent); err == nil {
m.Vector = vec
}
}
}
return nil