From 4af85fa093331ddcafd1659742569bc98413593c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=94=AF?= Date: Sun, 20 Sep 2026 00:43:50 +0800 Subject: [PATCH] fix(dedup): scan full table before namespace filter in Dedup endpoint The endpoint passed the namespace straight into LanceDB.Search with top_k=200. Rust ranks by distance to the zero vector globally and only then applies the namespace filter, so a namespace's rows were mostly never returned: soulful-xiaowei has 238 rows but only 9 came back, hermes-main has 3387 but only 200 did. The endpoint therefore reported merged=0 on every call - a false negative that made memory deduplication look complete when a full cosine scan finds 404 redundant rows at the 0.98 threshold. Now it pulls the whole table (top_k=1000000, no namespace filter) and filters by namespace plus is_deleted in Go, so 'no duplicates' is a claim the endpoint can actually support. --- go/internal/api/routes/core.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/go/internal/api/routes/core.go b/go/internal/api/routes/core.go index a80fa6f..a07a538 100644 --- a/go/internal/api/routes/core.go +++ b/go/internal/api/routes/core.go @@ -591,13 +591,24 @@ func (a *API) Dedup(w http.ResponseWriter, r *http.Request) { req.Namespace = "hermes-main" } - // 用零向量搜出该 namespace 内最多 200 条记忆 + // 必须用 zeroVec + 超大 top_k 扫**全表**,再在 Go 侧按 namespace 过滤。 + // 若直接把 namespace 传给 Search,Rust 会在"全局零向量排名前 N 条"上做 + // 事后过滤,导致该 namespace 的多数行根本取不到(实测 soulful-xiaowei + // 实有 238 条却只返回 9 条,hermes-main 3387 条只返回 200 条), + // 从而使本端点恒定返回 merged=0 —— 假阴性。 + // 用真实向量检索才是正解,但去重需要全量比对,故此处全表拉取后过滤。 zeroVec := make([]float32, 1024) - results, err := a.LanceDB.Search("memories", zeroVec, 200, req.Namespace) + all, err := a.LanceDB.Search("memories", zeroVec, 1000000, "") if err != nil { respondError(w, 500, "get memories: "+err.Error()) return } + results := make([]models.MemoryRecord, 0, len(all)) + for _, rec := range all { + if rec.Namespace == req.Namespace && !rec.IsDeleted { + results = append(results, rec) + } + } var merged int seen := make(map[string]bool)