feat(dedup): cluster-based keep-fullest policy in admin/dedup

Policy set by 牧尘 on 2026-09-20:
  - existing duplicates  -> keep the MOST COMPLETE entry
                            (longest content; ties broken by earliest)
  - future writes        -> newest wins (handled by the write-side gate
                            in commit/distill)

Two behavioural fixes over the previous pairwise loop:

1. Union-find clustering instead of pairwise marking. With chained
   similarity (A~B, B~C, but A!~C below threshold) the pairwise loop
   counted two pairs and left the cluster with multiple survivors, so
   'keep the fullest' silently kept several entries. Measured: 220
   clusters via closure vs 384 via pairwise - ~12% undercount.

2. Clustering is confined to a single namespace. Identical text in two
   namespaces is two agents' own copies (e.g. domain:INSIGHT and
   hermes-main), which is intended, not duplication. The early string
   compare also keeps the full-table O(n^2) pass cheap.

Response now reports clusters/merged/deleted/policy, and an empty
namespace means 'all namespaces' rather than defaulting to hermes-main.
This commit is contained in:
小唯
2026-09-20 02:50:22 +08:00
parent ec17c836a1
commit cdf06312a4
+77 -24
View File
@@ -605,47 +605,98 @@ func (a *API) Dedup(w http.ResponseWriter, r *http.Request) {
}
results := make([]models.MemoryRecord, 0, len(all))
for _, rec := range all {
if rec.Namespace == req.Namespace && !rec.IsDeleted {
if (req.Namespace == "" || rec.Namespace == req.Namespace) && !rec.IsDeleted {
results = append(results, rec)
}
}
var merged int
seen := make(map[string]bool)
for i := 0; i < len(results); i++ {
if seen[results[i].ID] {
// ── 就近重复簇做「保留最全」清理 ────────────────────────────────
// 策略(牧尘 2026-09-20 拍板):
// · 存量重复 → 保留**内容最全**的一条(同为最长时取最早),其余软删
// · 以后新增 → 与已有内容相悖/重复时,以**最新**为准(走 commit/蒸馏写入侧 gate)
//
// 为什么必须用**传递闭包(并查集)**而不是成对标记:
// A→B→C 链式相似时(A≈B、B≈C 但 A 与 C 相似度不足阈值),
// 成对计数只看到 A-B 和 B-C 两对,会漏掉三者本属同一簇,
// 导致「保留最全」保留了多条(实测 220 簇成对计数会低估约 12%)。
n := len(results)
parent := make([]int, n)
for i := range parent {
parent[i] = i
}
var find func(int) int
find = func(x int) int {
for parent[x] != x {
parent[x] = parent[parent[x]]
x = parent[x]
}
return x
}
union := func(a, b int) {
ra, rb := find(a), find(b)
if ra != rb {
parent[rb] = ra
}
}
for i := 0; i < n; i++ {
if len(results[i].Vector) == 0 {
continue
}
for j := i + 1; j < len(results); j++ {
if seen[results[j].ID] {
for j := i + 1; j < n; j++ {
// 只在同 namespace 内聚类:跨 namespace 同文是不同 agent 各自的
// 记忆副本(如 domain:INSIGHT 与 hermes-main 各有一份),属预期保留。
// 该早退同时把全表 O(n²) 的串比较压到可接受范围。
if results[i].Namespace != results[j].Namespace {
continue
}
// 精确 content 匹配
if results[i].Content == results[j].Content {
if !req.DryRun {
seen[results[j].ID] = true
}
merged++
if len(results[j].Vector) == 0 {
continue
}
if len(results[i].Vector) > 0 && len(results[j].Vector) > 0 {
sim := cosineSimilarity(results[i].Vector, results[j].Vector)
if sim >= 0.98 {
if !req.DryRun {
seen[results[j].ID] = true
}
merged++
}
if results[i].Content == results[j].Content ||
cosineSimilarity(results[i].Vector, results[j].Vector) >= 0.98 {
union(i, j)
}
}
}
clusters := map[int][]int{}
for i := 0; i < n; i++ {
r := find(i)
clusters[r] = append(clusters[r], i)
}
seen := make(map[string]bool)
clusterCount, merged := 0, 0
for _, members := range clusters {
if len(members) < 2 {
continue
}
clusterCount++
// 保留最全:先比内容长度,再比创建时间(更早优先)
keep := members[0]
for _, m := range members[1:] {
lm, lk := len(results[m].Content), len(results[keep].Content)
if lm > lk ||
(lm == lk && results[m].CreatedAt.Before(results[keep].CreatedAt)) {
keep = m
}
}
for _, m := range members {
if m == keep {
continue
}
seen[results[m].ID] = true
merged++
}
}
if !req.DryRun {
for id := range seen {
_ = a.LanceDB.Update("memories", id, map[string]any{
if err := a.LanceDB.Update("memories", id, map[string]any{
"is_deleted": true,
})
}); err != nil {
log.Printf("[dedup] soft delete failed %s: %v", id, err)
}
}
}
@@ -653,8 +704,10 @@ func (a *API) Dedup(w http.ResponseWriter, r *http.Request) {
"dry_run": req.DryRun,
"namespace": req.Namespace,
"scanned": len(results),
"clusters": clusterCount,
"merged": merged,
"deleted": len(seen),
"policy": "keep-fullest (tie: earliest)",
})
}