diff --git a/go/internal/api/routes/core.go b/go/internal/api/routes/core.go index a07a538..27adce6 100644 --- a/go/internal/api/routes/core.go +++ b/go/internal/api/routes/core.go @@ -605,47 +605,98 @@ func (a *API) Dedup(w http.ResponseWriter, r *http.Request) { } results := make([]models.MemoryRecord, 0, len(all)) for _, rec := range all { - if rec.Namespace == req.Namespace && !rec.IsDeleted { + if (req.Namespace == "" || rec.Namespace == req.Namespace) && !rec.IsDeleted { results = append(results, rec) } } - var merged int - seen := make(map[string]bool) - - for i := 0; i < len(results); i++ { - if seen[results[i].ID] { + // ── 就近重复簇做「保留最全」清理 ──────────────────────────────── + // 策略(牧尘 2026-09-20 拍板): + // · 存量重复 → 保留**内容最全**的一条(同为最长时取最早),其余软删 + // · 以后新增 → 与已有内容相悖/重复时,以**最新**为准(走 commit/蒸馏写入侧 gate) + // + // 为什么必须用**传递闭包(并查集)**而不是成对标记: + // A→B→C 链式相似时(A≈B、B≈C 但 A 与 C 相似度不足阈值), + // 成对计数只看到 A-B 和 B-C 两对,会漏掉三者本属同一簇, + // 导致「保留最全」保留了多条(实测 220 簇成对计数会低估约 12%)。 + n := len(results) + parent := make([]int, n) + for i := range parent { + parent[i] = i + } + var find func(int) int + find = func(x int) int { + for parent[x] != x { + parent[x] = parent[parent[x]] + x = parent[x] + } + return x + } + union := func(a, b int) { + ra, rb := find(a), find(b) + if ra != rb { + parent[rb] = ra + } + } + for i := 0; i < n; i++ { + if len(results[i].Vector) == 0 { continue } - for j := i + 1; j < len(results); j++ { - if seen[results[j].ID] { + for j := i + 1; j < n; j++ { + // 只在同 namespace 内聚类:跨 namespace 同文是不同 agent 各自的 + // 记忆副本(如 domain:INSIGHT 与 hermes-main 各有一份),属预期保留。 + // 该早退同时把全表 O(n²) 的串比较压到可接受范围。 + if results[i].Namespace != results[j].Namespace { continue } - // 精确 content 匹配 - if results[i].Content == results[j].Content { - if !req.DryRun { - seen[results[j].ID] = true - } - merged++ + if len(results[j].Vector) == 0 { continue } - if len(results[i].Vector) > 0 && len(results[j].Vector) > 0 { - sim := cosineSimilarity(results[i].Vector, results[j].Vector) - if sim >= 0.98 { - if !req.DryRun { - seen[results[j].ID] = true - } - merged++ - } + if results[i].Content == results[j].Content || + cosineSimilarity(results[i].Vector, results[j].Vector) >= 0.98 { + union(i, j) } } } + clusters := map[int][]int{} + for i := 0; i < n; i++ { + r := find(i) + clusters[r] = append(clusters[r], i) + } + + seen := make(map[string]bool) + clusterCount, merged := 0, 0 + for _, members := range clusters { + if len(members) < 2 { + continue + } + clusterCount++ + // 保留最全:先比内容长度,再比创建时间(更早优先) + keep := members[0] + for _, m := range members[1:] { + lm, lk := len(results[m].Content), len(results[keep].Content) + if lm > lk || + (lm == lk && results[m].CreatedAt.Before(results[keep].CreatedAt)) { + keep = m + } + } + for _, m := range members { + if m == keep { + continue + } + seen[results[m].ID] = true + merged++ + } + } + if !req.DryRun { for id := range seen { - _ = a.LanceDB.Update("memories", id, map[string]any{ + if err := a.LanceDB.Update("memories", id, map[string]any{ "is_deleted": true, - }) + }); err != nil { + log.Printf("[dedup] soft delete failed %s: %v", id, err) + } } } @@ -653,8 +704,10 @@ func (a *API) Dedup(w http.ResponseWriter, r *http.Request) { "dry_run": req.DryRun, "namespace": req.Namespace, "scanned": len(results), + "clusters": clusterCount, "merged": merged, "deleted": len(seen), + "policy": "keep-fullest (tie: earliest)", }) }