feat(dedup): cluster-based keep-fullest policy in admin/dedup
Policy set by 牧尘 on 2026-09-20:
- existing duplicates -> keep the MOST COMPLETE entry
(longest content; ties broken by earliest)
- future writes -> newest wins (handled by the write-side gate
in commit/distill)
Two behavioural fixes over the previous pairwise loop:
1. Union-find clustering instead of pairwise marking. With chained
similarity (A~B, B~C, but A!~C below threshold) the pairwise loop
counted two pairs and left the cluster with multiple survivors, so
'keep the fullest' silently kept several entries. Measured: 220
clusters via closure vs 384 via pairwise - ~12% undercount.
2. Clustering is confined to a single namespace. Identical text in two
namespaces is two agents' own copies (e.g. domain:INSIGHT and
hermes-main), which is intended, not duplication. The early string
compare also keeps the full-table O(n^2) pass cheap.
Response now reports clusters/merged/deleted/policy, and an empty
namespace means 'all namespaces' rather than defaulting to hermes-main.
This commit is contained in:
@@ -605,47 +605,98 @@ func (a *API) Dedup(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
results := make([]models.MemoryRecord, 0, len(all))
|
||||
for _, rec := range all {
|
||||
if rec.Namespace == req.Namespace && !rec.IsDeleted {
|
||||
if (req.Namespace == "" || rec.Namespace == req.Namespace) && !rec.IsDeleted {
|
||||
results = append(results, rec)
|
||||
}
|
||||
}
|
||||
|
||||
var merged int
|
||||
seen := make(map[string]bool)
|
||||
|
||||
for i := 0; i < len(results); i++ {
|
||||
if seen[results[i].ID] {
|
||||
// ── 就近重复簇做「保留最全」清理 ────────────────────────────────
|
||||
// 策略(牧尘 2026-09-20 拍板):
|
||||
// · 存量重复 → 保留**内容最全**的一条(同为最长时取最早),其余软删
|
||||
// · 以后新增 → 与已有内容相悖/重复时,以**最新**为准(走 commit/蒸馏写入侧 gate)
|
||||
//
|
||||
// 为什么必须用**传递闭包(并查集)**而不是成对标记:
|
||||
// A→B→C 链式相似时(A≈B、B≈C 但 A 与 C 相似度不足阈值),
|
||||
// 成对计数只看到 A-B 和 B-C 两对,会漏掉三者本属同一簇,
|
||||
// 导致「保留最全」保留了多条(实测 220 簇成对计数会低估约 12%)。
|
||||
n := len(results)
|
||||
parent := make([]int, n)
|
||||
for i := range parent {
|
||||
parent[i] = i
|
||||
}
|
||||
var find func(int) int
|
||||
find = func(x int) int {
|
||||
for parent[x] != x {
|
||||
parent[x] = parent[parent[x]]
|
||||
x = parent[x]
|
||||
}
|
||||
return x
|
||||
}
|
||||
union := func(a, b int) {
|
||||
ra, rb := find(a), find(b)
|
||||
if ra != rb {
|
||||
parent[rb] = ra
|
||||
}
|
||||
}
|
||||
for i := 0; i < n; i++ {
|
||||
if len(results[i].Vector) == 0 {
|
||||
continue
|
||||
}
|
||||
for j := i + 1; j < len(results); j++ {
|
||||
if seen[results[j].ID] {
|
||||
for j := i + 1; j < n; j++ {
|
||||
// 只在同 namespace 内聚类:跨 namespace 同文是不同 agent 各自的
|
||||
// 记忆副本(如 domain:INSIGHT 与 hermes-main 各有一份),属预期保留。
|
||||
// 该早退同时把全表 O(n²) 的串比较压到可接受范围。
|
||||
if results[i].Namespace != results[j].Namespace {
|
||||
continue
|
||||
}
|
||||
// 精确 content 匹配
|
||||
if results[i].Content == results[j].Content {
|
||||
if !req.DryRun {
|
||||
seen[results[j].ID] = true
|
||||
}
|
||||
merged++
|
||||
if len(results[j].Vector) == 0 {
|
||||
continue
|
||||
}
|
||||
if len(results[i].Vector) > 0 && len(results[j].Vector) > 0 {
|
||||
sim := cosineSimilarity(results[i].Vector, results[j].Vector)
|
||||
if sim >= 0.98 {
|
||||
if !req.DryRun {
|
||||
seen[results[j].ID] = true
|
||||
}
|
||||
merged++
|
||||
}
|
||||
if results[i].Content == results[j].Content ||
|
||||
cosineSimilarity(results[i].Vector, results[j].Vector) >= 0.98 {
|
||||
union(i, j)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
clusters := map[int][]int{}
|
||||
for i := 0; i < n; i++ {
|
||||
r := find(i)
|
||||
clusters[r] = append(clusters[r], i)
|
||||
}
|
||||
|
||||
seen := make(map[string]bool)
|
||||
clusterCount, merged := 0, 0
|
||||
for _, members := range clusters {
|
||||
if len(members) < 2 {
|
||||
continue
|
||||
}
|
||||
clusterCount++
|
||||
// 保留最全:先比内容长度,再比创建时间(更早优先)
|
||||
keep := members[0]
|
||||
for _, m := range members[1:] {
|
||||
lm, lk := len(results[m].Content), len(results[keep].Content)
|
||||
if lm > lk ||
|
||||
(lm == lk && results[m].CreatedAt.Before(results[keep].CreatedAt)) {
|
||||
keep = m
|
||||
}
|
||||
}
|
||||
for _, m := range members {
|
||||
if m == keep {
|
||||
continue
|
||||
}
|
||||
seen[results[m].ID] = true
|
||||
merged++
|
||||
}
|
||||
}
|
||||
|
||||
if !req.DryRun {
|
||||
for id := range seen {
|
||||
_ = a.LanceDB.Update("memories", id, map[string]any{
|
||||
if err := a.LanceDB.Update("memories", id, map[string]any{
|
||||
"is_deleted": true,
|
||||
})
|
||||
}); err != nil {
|
||||
log.Printf("[dedup] soft delete failed %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -653,8 +704,10 @@ func (a *API) Dedup(w http.ResponseWriter, r *http.Request) {
|
||||
"dry_run": req.DryRun,
|
||||
"namespace": req.Namespace,
|
||||
"scanned": len(results),
|
||||
"clusters": clusterCount,
|
||||
"merged": merged,
|
||||
"deleted": len(seen),
|
||||
"policy": "keep-fullest (tie: earliest)",
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user