fix(admin): /memories 与 /memories/export 改用全表扫描(Scan),弃零向量 ANN

病根:两个列表/导出端点一直用『零向量 ANN』冒充『列出全部』——零向量没有方向,
ANN 退化成本存储序前 N 行,再按 ns 客户端过滤 ⇒ 计数完全失真
(实测 export hermes-main 报 1 条,真实 139+;shared 报 0,真实 29)。

修复:
① Rust lancedb_ops: scan_for_forgetting 加 namespace 下推(only_if),limit 作用于过滤后结果集
② Go 接口 LanceDB 新增 Scan(limit, ns):
   - RustLanceDBClient → lancedb_scan RPC
   - SQLiteClient → SELECT ... WHERE is_deleted=0 [AND ns] ORDER BY created_at DESC LIMIT
   - MemLanceClient → 内存 map 过滤 + created_at 倒序
③ 两个端点 ListMemories / ExportMemoriesMD 改用 aa.LanceDB.Scan(limit, ns)
④ 补 mock(zz_commit mockLance / write-buffer fakeLDB)+ 5 条新判据
   (全表 3 / hermes-main 2 / shared 1 / limit 生效 / 不存在 ns 0)
This commit is contained in:
小唯
2026-09-21 16:36:41 +08:00
parent 1b937cc990
commit 1060e62066
10 changed files with 263 additions and 10 deletions
+6 -5
View File
@@ -449,7 +449,9 @@ func isAllDigits(s string) bool {
return true
}
// GET /api/v1/memories — 批量列出记忆(按 importance 排序,不含向量)
// GET /api/v1/memories — 批量列出记忆(按时间倒序,不含向量)
// 2026-09-21:改用全表扫描(aa.LanceDB.Scan)替代「零向量 ANN 冒充列表」。
// 修前零向量无方向 ⇒ 只拿到存储序前 N 行再按 ns 过滤 ⇒ 列表/计数失真。
func (aa *AdminAPI) ListMemories(w http.ResponseWriter, r *http.Request) {
limit := 100
ns := r.URL.Query().Get("namespace")
@@ -461,8 +463,7 @@ func (aa *AdminAPI) ListMemories(w http.ResponseWriter, r *http.Request) {
limit = 1000
}
}
zeroVec := make([]float32, 1024)
results, err := aa.LanceDB.Search("memories", zeroVec, limit, ns)
results, err := aa.LanceDB.Scan(limit, ns)
if err != nil {
respondError(w, 500, "list memories: "+err.Error())
return
@@ -503,8 +504,8 @@ func (aa *AdminAPI) ExportMemoriesMD(w http.ResponseWriter, r *http.Request) {
limit = 5000
}
}
zeroVec := make([]float32, 1024)
results, err := aa.LanceDB.Search("memories", zeroVec, limit, ns)
// 2026-09-21:改用全表扫描(aa.LanceDB.Scan)替代「零向量 ANN 冒充列表」。
results, err := aa.LanceDB.Scan(limit, ns)
if err != nil {
respondError(w, 500, "export memories: "+err.Error())
return
@@ -30,6 +30,24 @@ type mockLance struct {
memories []models.MemoryRecord
}
// Scan mock:返回按 namespace 过滤后的记忆(测试列表/导出路径用)。
func (m *mockLance) Scan(limit int, ns string) ([]models.MemoryRecord, error) {
if limit <= 0 {
limit = 100
}
var out []models.MemoryRecord
for _, r := range m.memories {
if ns != "" && r.Namespace != ns {
continue
}
out = append(out, r)
if len(out) >= limit {
break
}
}
return out, nil
}
func (m *mockLance) Search(table string, vector []float32, topK int, ns string) ([]models.MemoryRecord, error) {
if table != "memories" {
return nil, nil
+4
View File
@@ -32,6 +32,10 @@ type LanceDB interface {
UpdateMemoryContent(id, newContent, source string) error
InsertMemory(m models.MemoryRecord) error
Search(table string, vector []float32, topK int, namespaceFilter string) ([]models.MemoryRecord, error)
// Scan 全表扫描存活记忆(is_deleted=false),按 namespace 可选过滤,跳向量。
// 2026-09-21:列表/导出端点的正确底座 —— 修前它们用「零向量 ANN」冒充列表,
// 零向量无方向 ⇒ 只拿到任意前 N 行再按 ns 过滤 ⇒ 计数完全失真。
Scan(limit int, namespaceFilter string) ([]models.MemoryRecord, error)
Insert(table string, record any) error
Update(table, id string, fields map[string]any) error
}
+64
View File
@@ -273,6 +273,70 @@ func (rc *RustLanceDBClient) Search(table string, vector []float32, topK int, na
return out, nil
}
// Scan 全表扫描存活记忆(走 Rust IPC `lancedb_scan`:is_deleted=false + namespace 下推)。
// 2026-09-21:为 /api/v1/memories 与 /api/v1/memories/export 提供正确底座。
// 修前这两个端点用 `Search(zeroVec, ...)` 冒充列表:零向量没有方向,
// ANN 返回的是存储序前 N 行再按 ns 过滤 ⇒ 实测 hermes-main 报 1 条(真实 139+),
// 计数完全失真、导出内容残缺。
// 注意:Rust 侧硬上限 5000;namespace 由 only_if 下推,limit 作用于**过滤后**结果集。
func (rc *RustLanceDBClient) Scan(limit int, namespaceFilter string) ([]models.MemoryRecord, error) {
if limit <= 0 {
limit = 100
}
if limit > 5000 {
limit = 5000
}
resp, err := rc.rpc(ipcReq{Type: "lancedb_scan", QueryLimit: limit, Namespace: namespaceFilter})
if err != nil {
return nil, err
}
var raw []struct {
ID string `json:"id"`
Content string `json:"content"`
Category string `json:"category"`
Namespace string `json:"namespace"`
IsDeleted bool `json:"is_deleted"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
Importance float64 `json:"importance"`
QualityScore float64 `json:"quality_score"`
RecallCount int64 `json:"recall_count"`
Tier string `json:"tier"`
Freshness string `json:"freshness"`
Source string `json:"source"`
}
if err := json.Unmarshal([]byte(resp.ReportJSON), &raw); err != nil {
return nil, fmt.Errorf("scan unmarshal: %w", err)
}
parseTime := func(s string) time.Time {
if t, err := time.Parse(time.RFC3339, s); err == nil {
return t
}
return time.Time{}
}
out := make([]models.MemoryRecord, 0, limit)
for _, r := range raw {
if r.IsDeleted {
continue
}
// 兜底:Rust only_if 下推失效时客户端再过滤一次(同 Search 的做法)
if namespaceFilter != "" && r.Namespace != namespaceFilter {
continue
}
out = append(out, models.MemoryRecord{
ID: r.ID, Content: r.Content, Category: r.Category, Namespace: r.Namespace,
CreatedAt: parseTime(r.CreatedAt), UpdatedAt: parseTime(r.UpdatedAt),
Importance: r.Importance, QualityScore: r.QualityScore,
RecallCount: int(r.RecallCount), Tier: r.Tier, Source: r.Source,
})
if len(out) >= limit {
break
}
}
return out, nil
}
func (rc *RustLanceDBClient) localSearch(vec []float32, topK int, ns string) []models.MemoryRecord {
_local.mu.RLock()
defer _local.mu.RUnlock()
+41
View File
@@ -202,6 +202,47 @@ func (mlc *MemLanceClient) GetCandidatesForForgetting() ([]map[string]interface{
return results, nil
}
// Scan 内存实现的全表扫描(过滤 is_deleted + namespace)。
// 按 CreatedAt 倒序返回(列表/导出更符合直觉:新的在前)。
func (mlc *MemLanceClient) Scan(limit int, namespaceFilter string) ([]models.MemoryRecord, error) {
if limit <= 0 {
limit = 100
}
mlc.mu.RLock()
defer mlc.mu.RUnlock()
type item struct {
rec models.MemoryRecord
ts time.Time
}
items := make([]item, 0, len(mlc.memories))
for _, e := range mlc.memories {
if e.IsDeleted {
continue
}
if namespaceFilter != "" && e.Namespace != namespaceFilter {
continue
}
items = append(items, item{
rec: models.MemoryRecord{
ID: e.ID, Content: e.Content, Category: e.Category,
Namespace: e.Namespace, CreatedAt: e.CreatedAt,
QualityScore: e.QualityScore, RecallCount: e.RecallCount,
Tier: e.Tier, Source: e.Source, Vector: e.Vector,
},
ts: e.CreatedAt,
})
}
sort.Slice(items, func(i, j int) bool { return items[i].ts.After(items[j].ts) })
out := make([]models.MemoryRecord, 0, limit)
for _, it := range items {
if len(out) >= limit {
break
}
out = append(out, it.rec)
}
return out, nil
}
func (mlc *MemLanceClient) GetSkillCandidates(minRecalls, limit int) ([]models.MemoryRecord, error) {
return nil, nil // 内存实现,暂不支持
}
@@ -20,6 +20,11 @@ type fakeLDB struct {
forceBatchErr bool
}
// Scan mock:write-buffer 测试只需占位实现(被测路径不实际调用)。
func (f *fakeLDB) Scan(limit int, ns string) ([]models.MemoryRecord, error) {
return nil, nil
}
func (f *fakeLDB) InsertEpisode(agentID, namespace, content, category string) (string, error) {
return "", nil
}
+53
View File
@@ -0,0 +1,53 @@
package storage
import "testing"
// TestMemLanceScanFiltersAndLimits 验证全表扫描的「is_deleted 过滤 + namespace 下推 + limit」。
// 这是 /api/v1/memories 与 /api/v1/memories/export 的正确底座(修前它们用零向量 ANN 冒充列表,
// 零向量无方向 ⇒ 计数失真)。
func TestMemLanceScanFiltersAndLimits(t *testing.T) {
mlc := NewMemLanceClient(&Embedder{endpoint: "http://127.0.0.1:1/never", dim: 1024})
mlc.memories = map[string]*memEntry{
"m1": {ID: "m1", Namespace: "hermes-main", Category: "distilled", Content: "x"},
"m2": {ID: "m2", Namespace: "hermes-main", Category: "distilled", Content: "x"},
"m3": {ID: "m3", Namespace: "shared", Category: "distilled", Content: "x"},
}
// ① 全表(无 ns)→ 3 条
all, err := mlc.Scan(100, "")
if err != nil {
t.Fatalf("Scan 出错: %v", err)
}
if len(all) != 3 {
t.Errorf("✗ 全表应得 3 条,实际 %d", len(all))
}
t.Logf("✓ 全表扫描 %d 条", len(all))
// ② namespace=hermes-main → 2 条
hm, _ := mlc.Scan(100, "hermes-main")
if len(hm) != 2 {
t.Errorf("✗ hermes-main 应得 2 条,实际 %d", len(hm))
}
t.Logf("✓ hermes-main 过滤 %d 条", len(hm))
// ③ namespace=shared → 1 条
sh, _ := mlc.Scan(100, "shared")
if len(sh) != 1 {
t.Errorf("✗ shared 应得 1 条,实际 %d", len(sh))
}
t.Logf("✓ shared 过滤 %d 条", len(sh))
// ④ limit 生效(limit=1 → 最多 1 条)
one, _ := mlc.Scan(1, "")
if len(one) > 1 {
t.Errorf("✗ limit=1 应 ≤1 条,实际 %d", len(one))
}
t.Logf("✓ limit 生效 %d 条", len(one))
// ⑤ 空 ns(不存在的命名空间)→ 0 条
none, _ := mlc.Scan(100, "does-not-exist")
if len(none) != 0 {
t.Errorf("✗ 不存在 ns 应得 0 条,实际 %d", len(none))
}
t.Logf("✓ 空 ns 过滤 %d 条", len(none))
}
+47
View File
@@ -330,6 +330,53 @@ func (sc *SQLiteClient) Insert(table string, record any) error {
}
}
// Scan SQLite 后端的全表扫描(is_deleted=0 + 可选 namespace,按 created_at 倒序)。
func (sc *SQLiteClient) Scan(limit int, namespaceFilter string) ([]models.MemoryRecord, error) {
if limit <= 0 {
limit = 100
}
sc.mu.RLock()
defer sc.mu.RUnlock()
q := "SELECT id, content, category, namespace, importance, quality_score, recall_count, tier, source, created_at, updated_at FROM memories WHERE is_deleted=0"
if namespaceFilter != "" {
q += " AND namespace='" + sanitizeIdent(namespaceFilter) + "'"
}
q += " ORDER BY created_at DESC LIMIT " + fmt.Sprintf("%d", limit)
rows, err := sc.query(q)
if err != nil {
return nil, err
}
out := make([]models.MemoryRecord, 0, len(rows))
for _, row := range rows {
rec := models.MemoryRecord{}
rec.ID, _ = row["id"].(string)
rec.Content, _ = row["content"].(string)
rec.Category, _ = row["category"].(string)
rec.Namespace, _ = row["namespace"].(string)
rec.Importance, _ = row["importance"].(float64)
rec.QualityScore, _ = row["quality_score"].(float64)
rec.Tier, _ = row["tier"].(string)
rec.Source, _ = row["source"].(string)
if n, ok := row["recall_count"].(int64); ok {
rec.RecallCount = int(n)
}
if ts, ok := row["created_at"].(string); ok {
if t, e := time.Parse(time.RFC3339, ts); e == nil {
rec.CreatedAt = t
}
}
if ts, ok := row["updated_at"].(string); ok {
if t, e := time.Parse(time.RFC3339, ts); e == nil {
rec.UpdatedAt = t
}
}
out = append(out, rec)
}
return out, nil
}
func (sc *SQLiteClient) Search(table string, vector []float32, topK int, namespaceFilter string) ([]models.MemoryRecord, error) {
sc.mu.RLock()
defer sc.mu.RUnlock()
+21 -3
View File
@@ -517,13 +517,31 @@ impl LanceDBOps {
// 1. 强制 limit(调用方传, main.rs 钳制硬上限 5000)
// 2. 不读 vector 列(1024 维 × 3000 条 ≈ 12MB+,IPC JSON 会卡)
// 3. 真读 last_recalled_at 列(scan_all 旧实现 String::new() 恒空 → 遗忘判定失真)
pub fn scan_for_forgetting(&self, limit: usize) -> Result<Vec<MemoryRecord>, Box<dyn std::error::Error>> {
/// 全表扫描存活记忆(过滤 is_deleted,跳过 vector)。
/// ⚠️ 名字来自历史用途(遗忘候选),语义实为「列出存活记忆」。
/// 2026-09-21:新增 namespace 下推过滤 —— 修前 Go 侧 /memories 与 /memories/export
/// 用「零向量 ANN」冒充列表,零向量无方向 ⇒ 返回任意前 N 行再按 ns 过滤,
/// 计数完全失真(实测 hermes-main 报 1 条,实际 139+)。
/// 用 only_if 下推过滤,limit 作用于**过滤后**的结果集,避免 5000 硬上限截断。
pub fn scan_for_forgetting(
&self,
limit: usize,
namespace: Option<&str>,
) -> Result<Vec<MemoryRecord>, Box<dyn std::error::Error>> {
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
let tbl = rt().block_on(db.open_table("memories").execute())?;
let mut filter = String::from("is_deleted = false");
if let Some(ns) = namespace {
if !ns.is_empty() {
let escaped = ns.replace('\\', "\\\\").replace('\'', "''");
filter.push_str(&format!(" AND namespace = '{}'", escaped));
}
}
let mut results = Box::pin(rt().block_on(
tbl.query()
.only_if("is_deleted = false")
.only_if(&filter)
.limit(limit)
.execute()
)?);
@@ -558,7 +576,7 @@ impl LanceDBOps {
});
}
}
eprintln!("[lancedb] scan_for_forgetting → {} records (limit={})", records.len(), limit);
eprintln!("[lancedb] scan_for_forgetting → {} records (limit={}, ns={:?})", records.len(), limit, namespace);
Ok(records)
}
+4 -2
View File
@@ -369,8 +369,10 @@ fn handle_client(mut stream: UnixStream, args: &Args, lancedb: LanceDBOps, state
if limit > 5000 {
limit = 5000;
}
eprintln!("[lancedb] scan_for_forgetting: limit={}", limit);
match lancedb.scan_for_forgetting(limit) {
// 2026-09-21: 可选 namespace 下推过滤(列表/导出端点用)
let namespace = msg["namespace"].as_str().filter(|s| !s.is_empty());
eprintln!("[lancedb] scan_for_forgetting: limit={} namespace={:?}", limit, namespace);
match lancedb.scan_for_forgetting(limit, namespace) {
Ok(r) => send_ok(&mut stream, &serde_json::to_string(&r).unwrap_or_default()),
Err(e) => send_error(&mut stream, "lancedb_scan", &e.to_string()),
}