fix(dedup): scan full table before namespace filter in Dedup endpoint
The endpoint passed the namespace straight into LanceDB.Search with top_k=200. Rust ranks by distance to the zero vector globally and only then applies the namespace filter, so a namespace's rows were mostly never returned: soulful-xiaowei has 238 rows but only 9 came back, hermes-main has 3387 but only 200 did. The endpoint therefore reported merged=0 on every call - a false negative that made memory deduplication look complete when a full cosine scan finds 404 redundant rows at the 0.98 threshold. Now it pulls the whole table (top_k=1000000, no namespace filter) and filters by namespace plus is_deleted in Go, so 'no duplicates' is a claim the endpoint can actually support.
This commit is contained in:
@@ -591,13 +591,24 @@ func (a *API) Dedup(w http.ResponseWriter, r *http.Request) {
|
||||
req.Namespace = "hermes-main"
|
||||
}
|
||||
|
||||
// 用零向量搜出该 namespace 内最多 200 条记忆
|
||||
// 必须用 zeroVec + 超大 top_k 扫**全表**,再在 Go 侧按 namespace 过滤。
|
||||
// 若直接把 namespace 传给 Search,Rust 会在"全局零向量排名前 N 条"上做
|
||||
// 事后过滤,导致该 namespace 的多数行根本取不到(实测 soulful-xiaowei
|
||||
// 实有 238 条却只返回 9 条,hermes-main 3387 条只返回 200 条),
|
||||
// 从而使本端点恒定返回 merged=0 —— 假阴性。
|
||||
// 用真实向量检索才是正解,但去重需要全量比对,故此处全表拉取后过滤。
|
||||
zeroVec := make([]float32, 1024)
|
||||
results, err := a.LanceDB.Search("memories", zeroVec, 200, req.Namespace)
|
||||
all, err := a.LanceDB.Search("memories", zeroVec, 1000000, "")
|
||||
if err != nil {
|
||||
respondError(w, 500, "get memories: "+err.Error())
|
||||
return
|
||||
}
|
||||
results := make([]models.MemoryRecord, 0, len(all))
|
||||
for _, rec := range all {
|
||||
if rec.Namespace == req.Namespace && !rec.IsDeleted {
|
||||
results = append(results, rec)
|
||||
}
|
||||
}
|
||||
|
||||
var merged int
|
||||
seen := make(map[string]bool)
|
||||
|
||||
Reference in New Issue
Block a user