From 1060e620662158846f8d20b1eebd287cbd25661e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=94=AF?= Date: Mon, 21 Sep 2026 16:36:41 +0800 Subject: [PATCH] =?UTF-8?q?fix(admin):=20/memories=20=E4=B8=8E=20/memories?= =?UTF-8?q?/export=20=E6=94=B9=E7=94=A8=E5=85=A8=E8=A1=A8=E6=89=AB?= =?UTF-8?q?=E6=8F=8F=EF=BC=88Scan=EF=BC=89=EF=BC=8C=E5=BC=83=E9=9B=B6?= =?UTF-8?q?=E5=90=91=E9=87=8F=20ANN?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 病根:两个列表/导出端点一直用『零向量 ANN』冒充『列出全部』——零向量没有方向, ANN 退化成本存储序前 N 行,再按 ns 客户端过滤 ⇒ 计数完全失真 (实测 export hermes-main 报 1 条,真实 139+;shared 报 0,真实 29)。 修复: ① Rust lancedb_ops: scan_for_forgetting 加 namespace 下推(only_if),limit 作用于过滤后结果集 ② Go 接口 LanceDB 新增 Scan(limit, ns): - RustLanceDBClient → lancedb_scan RPC - SQLiteClient → SELECT ... WHERE is_deleted=0 [AND ns] ORDER BY created_at DESC LIMIT - MemLanceClient → 内存 map 过滤 + created_at 倒序 ③ 两个端点 ListMemories / ExportMemoriesMD 改用 aa.LanceDB.Scan(limit, ns) ④ 补 mock(zz_commit mockLance / write-buffer fakeLDB)+ 5 条新判据 (全表 3 / hermes-main 2 / shared 1 / limit 生效 / 不存在 ns 0) --- go/internal/api/routes/admin.go | 11 ++-- .../routes/zz_commit_conflict_gate_test.go | 18 ++++++ go/internal/storage/lancedb.go | 4 ++ go/internal/storage/lancedb_ipc.go | 64 +++++++++++++++++++ go/internal/storage/memvector.go | 41 ++++++++++++ .../storage/recall_write_buffer_test.go | 5 ++ go/internal/storage/scan_test.go | 53 +++++++++++++++ go/internal/storage/sqlite.go | 47 ++++++++++++++ rust/src/lancedb_ops.rs | 24 ++++++- rust/src/main.rs | 6 +- 10 files changed, 263 insertions(+), 10 deletions(-) create mode 100644 go/internal/storage/scan_test.go diff --git a/go/internal/api/routes/admin.go b/go/internal/api/routes/admin.go index 6ef1633..748eb74 100644 --- a/go/internal/api/routes/admin.go +++ b/go/internal/api/routes/admin.go @@ -449,7 +449,9 @@ func isAllDigits(s string) bool { return true } -// GET /api/v1/memories — 批量列出记忆(按 importance 排序,不含向量) +// GET /api/v1/memories — 批量列出记忆(按时间倒序,不含向量) +// 2026-09-21:改用全表扫描(aa.LanceDB.Scan)替代「零向量 ANN 冒充列表」。 +// 修前零向量无方向 ⇒ 只拿到存储序前 N 行再按 ns 过滤 ⇒ 列表/计数失真。 func (aa *AdminAPI) ListMemories(w http.ResponseWriter, r *http.Request) { limit := 100 ns := r.URL.Query().Get("namespace") @@ -461,8 +463,7 @@ func (aa *AdminAPI) ListMemories(w http.ResponseWriter, r *http.Request) { limit = 1000 } } - zeroVec := make([]float32, 1024) - results, err := aa.LanceDB.Search("memories", zeroVec, limit, ns) + results, err := aa.LanceDB.Scan(limit, ns) if err != nil { respondError(w, 500, "list memories: "+err.Error()) return @@ -503,8 +504,8 @@ func (aa *AdminAPI) ExportMemoriesMD(w http.ResponseWriter, r *http.Request) { limit = 5000 } } - zeroVec := make([]float32, 1024) - results, err := aa.LanceDB.Search("memories", zeroVec, limit, ns) + // 2026-09-21:改用全表扫描(aa.LanceDB.Scan)替代「零向量 ANN 冒充列表」。 + results, err := aa.LanceDB.Scan(limit, ns) if err != nil { respondError(w, 500, "export memories: "+err.Error()) return diff --git a/go/internal/api/routes/zz_commit_conflict_gate_test.go b/go/internal/api/routes/zz_commit_conflict_gate_test.go index 4cde498..a26a12d 100644 --- a/go/internal/api/routes/zz_commit_conflict_gate_test.go +++ b/go/internal/api/routes/zz_commit_conflict_gate_test.go @@ -30,6 +30,24 @@ type mockLance struct { memories []models.MemoryRecord } +// Scan mock:返回按 namespace 过滤后的记忆(测试列表/导出路径用)。 +func (m *mockLance) Scan(limit int, ns string) ([]models.MemoryRecord, error) { + if limit <= 0 { + limit = 100 + } + var out []models.MemoryRecord + for _, r := range m.memories { + if ns != "" && r.Namespace != ns { + continue + } + out = append(out, r) + if len(out) >= limit { + break + } + } + return out, nil +} + func (m *mockLance) Search(table string, vector []float32, topK int, ns string) ([]models.MemoryRecord, error) { if table != "memories" { return nil, nil diff --git a/go/internal/storage/lancedb.go b/go/internal/storage/lancedb.go index c1ffcbc..236a953 100644 --- a/go/internal/storage/lancedb.go +++ b/go/internal/storage/lancedb.go @@ -32,6 +32,10 @@ type LanceDB interface { UpdateMemoryContent(id, newContent, source string) error InsertMemory(m models.MemoryRecord) error Search(table string, vector []float32, topK int, namespaceFilter string) ([]models.MemoryRecord, error) + // Scan 全表扫描存活记忆(is_deleted=false),按 namespace 可选过滤,跳向量。 + // 2026-09-21:列表/导出端点的正确底座 —— 修前它们用「零向量 ANN」冒充列表, + // 零向量无方向 ⇒ 只拿到任意前 N 行再按 ns 过滤 ⇒ 计数完全失真。 + Scan(limit int, namespaceFilter string) ([]models.MemoryRecord, error) Insert(table string, record any) error Update(table, id string, fields map[string]any) error } diff --git a/go/internal/storage/lancedb_ipc.go b/go/internal/storage/lancedb_ipc.go index 3c6561a..aa5fccc 100644 --- a/go/internal/storage/lancedb_ipc.go +++ b/go/internal/storage/lancedb_ipc.go @@ -273,6 +273,70 @@ func (rc *RustLanceDBClient) Search(table string, vector []float32, topK int, na return out, nil } +// Scan 全表扫描存活记忆(走 Rust IPC `lancedb_scan`:is_deleted=false + namespace 下推)。 +// 2026-09-21:为 /api/v1/memories 与 /api/v1/memories/export 提供正确底座。 +// 修前这两个端点用 `Search(zeroVec, ...)` 冒充列表:零向量没有方向, +// ANN 返回的是存储序前 N 行再按 ns 过滤 ⇒ 实测 hermes-main 报 1 条(真实 139+), +// 计数完全失真、导出内容残缺。 +// 注意:Rust 侧硬上限 5000;namespace 由 only_if 下推,limit 作用于**过滤后**结果集。 +func (rc *RustLanceDBClient) Scan(limit int, namespaceFilter string) ([]models.MemoryRecord, error) { + if limit <= 0 { + limit = 100 + } + if limit > 5000 { + limit = 5000 + } + resp, err := rc.rpc(ipcReq{Type: "lancedb_scan", QueryLimit: limit, Namespace: namespaceFilter}) + if err != nil { + return nil, err + } + + var raw []struct { + ID string `json:"id"` + Content string `json:"content"` + Category string `json:"category"` + Namespace string `json:"namespace"` + IsDeleted bool `json:"is_deleted"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` + Importance float64 `json:"importance"` + QualityScore float64 `json:"quality_score"` + RecallCount int64 `json:"recall_count"` + Tier string `json:"tier"` + Freshness string `json:"freshness"` + Source string `json:"source"` + } + if err := json.Unmarshal([]byte(resp.ReportJSON), &raw); err != nil { + return nil, fmt.Errorf("scan unmarshal: %w", err) + } + parseTime := func(s string) time.Time { + if t, err := time.Parse(time.RFC3339, s); err == nil { + return t + } + return time.Time{} + } + out := make([]models.MemoryRecord, 0, limit) + for _, r := range raw { + if r.IsDeleted { + continue + } + // 兜底:Rust only_if 下推失效时客户端再过滤一次(同 Search 的做法) + if namespaceFilter != "" && r.Namespace != namespaceFilter { + continue + } + out = append(out, models.MemoryRecord{ + ID: r.ID, Content: r.Content, Category: r.Category, Namespace: r.Namespace, + CreatedAt: parseTime(r.CreatedAt), UpdatedAt: parseTime(r.UpdatedAt), + Importance: r.Importance, QualityScore: r.QualityScore, + RecallCount: int(r.RecallCount), Tier: r.Tier, Source: r.Source, + }) + if len(out) >= limit { + break + } + } + return out, nil +} + func (rc *RustLanceDBClient) localSearch(vec []float32, topK int, ns string) []models.MemoryRecord { _local.mu.RLock() defer _local.mu.RUnlock() diff --git a/go/internal/storage/memvector.go b/go/internal/storage/memvector.go index 761bf53..afe9ba3 100644 --- a/go/internal/storage/memvector.go +++ b/go/internal/storage/memvector.go @@ -202,6 +202,47 @@ func (mlc *MemLanceClient) GetCandidatesForForgetting() ([]map[string]interface{ return results, nil } +// Scan 内存实现的全表扫描(过滤 is_deleted + namespace)。 +// 按 CreatedAt 倒序返回(列表/导出更符合直觉:新的在前)。 +func (mlc *MemLanceClient) Scan(limit int, namespaceFilter string) ([]models.MemoryRecord, error) { + if limit <= 0 { + limit = 100 + } + mlc.mu.RLock() + defer mlc.mu.RUnlock() + type item struct { + rec models.MemoryRecord + ts time.Time + } + items := make([]item, 0, len(mlc.memories)) + for _, e := range mlc.memories { + if e.IsDeleted { + continue + } + if namespaceFilter != "" && e.Namespace != namespaceFilter { + continue + } + items = append(items, item{ + rec: models.MemoryRecord{ + ID: e.ID, Content: e.Content, Category: e.Category, + Namespace: e.Namespace, CreatedAt: e.CreatedAt, + QualityScore: e.QualityScore, RecallCount: e.RecallCount, + Tier: e.Tier, Source: e.Source, Vector: e.Vector, + }, + ts: e.CreatedAt, + }) + } + sort.Slice(items, func(i, j int) bool { return items[i].ts.After(items[j].ts) }) + out := make([]models.MemoryRecord, 0, limit) + for _, it := range items { + if len(out) >= limit { + break + } + out = append(out, it.rec) + } + return out, nil +} + func (mlc *MemLanceClient) GetSkillCandidates(minRecalls, limit int) ([]models.MemoryRecord, error) { return nil, nil // 内存实现,暂不支持 } diff --git a/go/internal/storage/recall_write_buffer_test.go b/go/internal/storage/recall_write_buffer_test.go index 39b17b9..38915ce 100644 --- a/go/internal/storage/recall_write_buffer_test.go +++ b/go/internal/storage/recall_write_buffer_test.go @@ -20,6 +20,11 @@ type fakeLDB struct { forceBatchErr bool } +// Scan mock:write-buffer 测试只需占位实现(被测路径不实际调用)。 +func (f *fakeLDB) Scan(limit int, ns string) ([]models.MemoryRecord, error) { + return nil, nil +} + func (f *fakeLDB) InsertEpisode(agentID, namespace, content, category string) (string, error) { return "", nil } diff --git a/go/internal/storage/scan_test.go b/go/internal/storage/scan_test.go new file mode 100644 index 0000000..886e07b --- /dev/null +++ b/go/internal/storage/scan_test.go @@ -0,0 +1,53 @@ +package storage + +import "testing" + +// TestMemLanceScanFiltersAndLimits 验证全表扫描的「is_deleted 过滤 + namespace 下推 + limit」。 +// 这是 /api/v1/memories 与 /api/v1/memories/export 的正确底座(修前它们用零向量 ANN 冒充列表, +// 零向量无方向 ⇒ 计数失真)。 +func TestMemLanceScanFiltersAndLimits(t *testing.T) { + mlc := NewMemLanceClient(&Embedder{endpoint: "http://127.0.0.1:1/never", dim: 1024}) + mlc.memories = map[string]*memEntry{ + "m1": {ID: "m1", Namespace: "hermes-main", Category: "distilled", Content: "x"}, + "m2": {ID: "m2", Namespace: "hermes-main", Category: "distilled", Content: "x"}, + "m3": {ID: "m3", Namespace: "shared", Category: "distilled", Content: "x"}, + } + + // ① 全表(无 ns)→ 3 条 + all, err := mlc.Scan(100, "") + if err != nil { + t.Fatalf("Scan 出错: %v", err) + } + if len(all) != 3 { + t.Errorf("✗ 全表应得 3 条,实际 %d", len(all)) + } + t.Logf("✓ 全表扫描 %d 条", len(all)) + + // ② namespace=hermes-main → 2 条 + hm, _ := mlc.Scan(100, "hermes-main") + if len(hm) != 2 { + t.Errorf("✗ hermes-main 应得 2 条,实际 %d", len(hm)) + } + t.Logf("✓ hermes-main 过滤 %d 条", len(hm)) + + // ③ namespace=shared → 1 条 + sh, _ := mlc.Scan(100, "shared") + if len(sh) != 1 { + t.Errorf("✗ shared 应得 1 条,实际 %d", len(sh)) + } + t.Logf("✓ shared 过滤 %d 条", len(sh)) + + // ④ limit 生效(limit=1 → 最多 1 条) + one, _ := mlc.Scan(1, "") + if len(one) > 1 { + t.Errorf("✗ limit=1 应 ≤1 条,实际 %d", len(one)) + } + t.Logf("✓ limit 生效 %d 条", len(one)) + + // ⑤ 空 ns(不存在的命名空间)→ 0 条 + none, _ := mlc.Scan(100, "does-not-exist") + if len(none) != 0 { + t.Errorf("✗ 不存在 ns 应得 0 条,实际 %d", len(none)) + } + t.Logf("✓ 空 ns 过滤 %d 条", len(none)) +} diff --git a/go/internal/storage/sqlite.go b/go/internal/storage/sqlite.go index 8497b5e..b254919 100644 --- a/go/internal/storage/sqlite.go +++ b/go/internal/storage/sqlite.go @@ -330,6 +330,53 @@ func (sc *SQLiteClient) Insert(table string, record any) error { } } +// Scan SQLite 后端的全表扫描(is_deleted=0 + 可选 namespace,按 created_at 倒序)。 +func (sc *SQLiteClient) Scan(limit int, namespaceFilter string) ([]models.MemoryRecord, error) { + if limit <= 0 { + limit = 100 + } + sc.mu.RLock() + defer sc.mu.RUnlock() + + q := "SELECT id, content, category, namespace, importance, quality_score, recall_count, tier, source, created_at, updated_at FROM memories WHERE is_deleted=0" + if namespaceFilter != "" { + q += " AND namespace='" + sanitizeIdent(namespaceFilter) + "'" + } + q += " ORDER BY created_at DESC LIMIT " + fmt.Sprintf("%d", limit) + + rows, err := sc.query(q) + if err != nil { + return nil, err + } + out := make([]models.MemoryRecord, 0, len(rows)) + for _, row := range rows { + rec := models.MemoryRecord{} + rec.ID, _ = row["id"].(string) + rec.Content, _ = row["content"].(string) + rec.Category, _ = row["category"].(string) + rec.Namespace, _ = row["namespace"].(string) + rec.Importance, _ = row["importance"].(float64) + rec.QualityScore, _ = row["quality_score"].(float64) + rec.Tier, _ = row["tier"].(string) + rec.Source, _ = row["source"].(string) + if n, ok := row["recall_count"].(int64); ok { + rec.RecallCount = int(n) + } + if ts, ok := row["created_at"].(string); ok { + if t, e := time.Parse(time.RFC3339, ts); e == nil { + rec.CreatedAt = t + } + } + if ts, ok := row["updated_at"].(string); ok { + if t, e := time.Parse(time.RFC3339, ts); e == nil { + rec.UpdatedAt = t + } + } + out = append(out, rec) + } + return out, nil +} + func (sc *SQLiteClient) Search(table string, vector []float32, topK int, namespaceFilter string) ([]models.MemoryRecord, error) { sc.mu.RLock() defer sc.mu.RUnlock() diff --git a/rust/src/lancedb_ops.rs b/rust/src/lancedb_ops.rs index c9175d2..494b013 100644 --- a/rust/src/lancedb_ops.rs +++ b/rust/src/lancedb_ops.rs @@ -517,13 +517,31 @@ impl LanceDBOps { // 1. 强制 limit(调用方传, main.rs 钳制硬上限 5000) // 2. 不读 vector 列(1024 维 × 3000 条 ≈ 12MB+,IPC JSON 会卡) // 3. 真读 last_recalled_at 列(scan_all 旧实现 String::new() 恒空 → 遗忘判定失真) - pub fn scan_for_forgetting(&self, limit: usize) -> Result, Box> { + /// 全表扫描存活记忆(过滤 is_deleted,跳过 vector)。 + /// ⚠️ 名字来自历史用途(遗忘候选),语义实为「列出存活记忆」。 + /// 2026-09-21:新增 namespace 下推过滤 —— 修前 Go 侧 /memories 与 /memories/export + /// 用「零向量 ANN」冒充列表,零向量无方向 ⇒ 返回任意前 N 行再按 ns 过滤, + /// 计数完全失真(实测 hermes-main 报 1 条,实际 139+)。 + /// 用 only_if 下推过滤,limit 作用于**过滤后**的结果集,避免 5000 硬上限截断。 + pub fn scan_for_forgetting( + &self, + limit: usize, + namespace: Option<&str>, + ) -> Result, Box> { let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?; let tbl = rt().block_on(db.open_table("memories").execute())?; + let mut filter = String::from("is_deleted = false"); + if let Some(ns) = namespace { + if !ns.is_empty() { + let escaped = ns.replace('\\', "\\\\").replace('\'', "''"); + filter.push_str(&format!(" AND namespace = '{}'", escaped)); + } + } + let mut results = Box::pin(rt().block_on( tbl.query() - .only_if("is_deleted = false") + .only_if(&filter) .limit(limit) .execute() )?); @@ -558,7 +576,7 @@ impl LanceDBOps { }); } } - eprintln!("[lancedb] scan_for_forgetting → {} records (limit={})", records.len(), limit); + eprintln!("[lancedb] scan_for_forgetting → {} records (limit={}, ns={:?})", records.len(), limit, namespace); Ok(records) } diff --git a/rust/src/main.rs b/rust/src/main.rs index 0b0d5f9..ad3b9bc 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -369,8 +369,10 @@ fn handle_client(mut stream: UnixStream, args: &Args, lancedb: LanceDBOps, state if limit > 5000 { limit = 5000; } - eprintln!("[lancedb] scan_for_forgetting: limit={}", limit); - match lancedb.scan_for_forgetting(limit) { + // 2026-09-21: 可选 namespace 下推过滤(列表/导出端点用) + let namespace = msg["namespace"].as_str().filter(|s| !s.is_empty()); + eprintln!("[lancedb] scan_for_forgetting: limit={} namespace={:?}", limit, namespace); + match lancedb.scan_for_forgetting(limit, namespace) { Ok(r) => send_ok(&mut stream, &serde_json::to_string(&r).unwrap_or_default()), Err(e) => send_error(&mut stream, "lancedb_scan", &e.to_string()), }