fix(admin): /memories 与 /memories/export 改用全表扫描(Scan),弃零向量 ANN
病根:两个列表/导出端点一直用『零向量 ANN』冒充『列出全部』——零向量没有方向, ANN 退化成本存储序前 N 行,再按 ns 客户端过滤 ⇒ 计数完全失真 (实测 export hermes-main 报 1 条,真实 139+;shared 报 0,真实 29)。 修复: ① Rust lancedb_ops: scan_for_forgetting 加 namespace 下推(only_if),limit 作用于过滤后结果集 ② Go 接口 LanceDB 新增 Scan(limit, ns): - RustLanceDBClient → lancedb_scan RPC - SQLiteClient → SELECT ... WHERE is_deleted=0 [AND ns] ORDER BY created_at DESC LIMIT - MemLanceClient → 内存 map 过滤 + created_at 倒序 ③ 两个端点 ListMemories / ExportMemoriesMD 改用 aa.LanceDB.Scan(limit, ns) ④ 补 mock(zz_commit mockLance / write-buffer fakeLDB)+ 5 条新判据 (全表 3 / hermes-main 2 / shared 1 / limit 生效 / 不存在 ns 0)
This commit is contained in:
@@ -449,7 +449,9 @@ func isAllDigits(s string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// GET /api/v1/memories — 批量列出记忆(按 importance 排序,不含向量)
|
||||
// GET /api/v1/memories — 批量列出记忆(按时间倒序,不含向量)
|
||||
// 2026-09-21:改用全表扫描(aa.LanceDB.Scan)替代「零向量 ANN 冒充列表」。
|
||||
// 修前零向量无方向 ⇒ 只拿到存储序前 N 行再按 ns 过滤 ⇒ 列表/计数失真。
|
||||
func (aa *AdminAPI) ListMemories(w http.ResponseWriter, r *http.Request) {
|
||||
limit := 100
|
||||
ns := r.URL.Query().Get("namespace")
|
||||
@@ -461,8 +463,7 @@ func (aa *AdminAPI) ListMemories(w http.ResponseWriter, r *http.Request) {
|
||||
limit = 1000
|
||||
}
|
||||
}
|
||||
zeroVec := make([]float32, 1024)
|
||||
results, err := aa.LanceDB.Search("memories", zeroVec, limit, ns)
|
||||
results, err := aa.LanceDB.Scan(limit, ns)
|
||||
if err != nil {
|
||||
respondError(w, 500, "list memories: "+err.Error())
|
||||
return
|
||||
@@ -503,8 +504,8 @@ func (aa *AdminAPI) ExportMemoriesMD(w http.ResponseWriter, r *http.Request) {
|
||||
limit = 5000
|
||||
}
|
||||
}
|
||||
zeroVec := make([]float32, 1024)
|
||||
results, err := aa.LanceDB.Search("memories", zeroVec, limit, ns)
|
||||
// 2026-09-21:改用全表扫描(aa.LanceDB.Scan)替代「零向量 ANN 冒充列表」。
|
||||
results, err := aa.LanceDB.Scan(limit, ns)
|
||||
if err != nil {
|
||||
respondError(w, 500, "export memories: "+err.Error())
|
||||
return
|
||||
|
||||
@@ -30,6 +30,24 @@ type mockLance struct {
|
||||
memories []models.MemoryRecord
|
||||
}
|
||||
|
||||
// Scan mock:返回按 namespace 过滤后的记忆(测试列表/导出路径用)。
|
||||
func (m *mockLance) Scan(limit int, ns string) ([]models.MemoryRecord, error) {
|
||||
if limit <= 0 {
|
||||
limit = 100
|
||||
}
|
||||
var out []models.MemoryRecord
|
||||
for _, r := range m.memories {
|
||||
if ns != "" && r.Namespace != ns {
|
||||
continue
|
||||
}
|
||||
out = append(out, r)
|
||||
if len(out) >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (m *mockLance) Search(table string, vector []float32, topK int, ns string) ([]models.MemoryRecord, error) {
|
||||
if table != "memories" {
|
||||
return nil, nil
|
||||
|
||||
@@ -32,6 +32,10 @@ type LanceDB interface {
|
||||
UpdateMemoryContent(id, newContent, source string) error
|
||||
InsertMemory(m models.MemoryRecord) error
|
||||
Search(table string, vector []float32, topK int, namespaceFilter string) ([]models.MemoryRecord, error)
|
||||
// Scan 全表扫描存活记忆(is_deleted=false),按 namespace 可选过滤,跳向量。
|
||||
// 2026-09-21:列表/导出端点的正确底座 —— 修前它们用「零向量 ANN」冒充列表,
|
||||
// 零向量无方向 ⇒ 只拿到任意前 N 行再按 ns 过滤 ⇒ 计数完全失真。
|
||||
Scan(limit int, namespaceFilter string) ([]models.MemoryRecord, error)
|
||||
Insert(table string, record any) error
|
||||
Update(table, id string, fields map[string]any) error
|
||||
}
|
||||
|
||||
@@ -273,6 +273,70 @@ func (rc *RustLanceDBClient) Search(table string, vector []float32, topK int, na
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Scan 全表扫描存活记忆(走 Rust IPC `lancedb_scan`:is_deleted=false + namespace 下推)。
|
||||
// 2026-09-21:为 /api/v1/memories 与 /api/v1/memories/export 提供正确底座。
|
||||
// 修前这两个端点用 `Search(zeroVec, ...)` 冒充列表:零向量没有方向,
|
||||
// ANN 返回的是存储序前 N 行再按 ns 过滤 ⇒ 实测 hermes-main 报 1 条(真实 139+),
|
||||
// 计数完全失真、导出内容残缺。
|
||||
// 注意:Rust 侧硬上限 5000;namespace 由 only_if 下推,limit 作用于**过滤后**结果集。
|
||||
func (rc *RustLanceDBClient) Scan(limit int, namespaceFilter string) ([]models.MemoryRecord, error) {
|
||||
if limit <= 0 {
|
||||
limit = 100
|
||||
}
|
||||
if limit > 5000 {
|
||||
limit = 5000
|
||||
}
|
||||
resp, err := rc.rpc(ipcReq{Type: "lancedb_scan", QueryLimit: limit, Namespace: namespaceFilter})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var raw []struct {
|
||||
ID string `json:"id"`
|
||||
Content string `json:"content"`
|
||||
Category string `json:"category"`
|
||||
Namespace string `json:"namespace"`
|
||||
IsDeleted bool `json:"is_deleted"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
Importance float64 `json:"importance"`
|
||||
QualityScore float64 `json:"quality_score"`
|
||||
RecallCount int64 `json:"recall_count"`
|
||||
Tier string `json:"tier"`
|
||||
Freshness string `json:"freshness"`
|
||||
Source string `json:"source"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(resp.ReportJSON), &raw); err != nil {
|
||||
return nil, fmt.Errorf("scan unmarshal: %w", err)
|
||||
}
|
||||
parseTime := func(s string) time.Time {
|
||||
if t, err := time.Parse(time.RFC3339, s); err == nil {
|
||||
return t
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
out := make([]models.MemoryRecord, 0, limit)
|
||||
for _, r := range raw {
|
||||
if r.IsDeleted {
|
||||
continue
|
||||
}
|
||||
// 兜底:Rust only_if 下推失效时客户端再过滤一次(同 Search 的做法)
|
||||
if namespaceFilter != "" && r.Namespace != namespaceFilter {
|
||||
continue
|
||||
}
|
||||
out = append(out, models.MemoryRecord{
|
||||
ID: r.ID, Content: r.Content, Category: r.Category, Namespace: r.Namespace,
|
||||
CreatedAt: parseTime(r.CreatedAt), UpdatedAt: parseTime(r.UpdatedAt),
|
||||
Importance: r.Importance, QualityScore: r.QualityScore,
|
||||
RecallCount: int(r.RecallCount), Tier: r.Tier, Source: r.Source,
|
||||
})
|
||||
if len(out) >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (rc *RustLanceDBClient) localSearch(vec []float32, topK int, ns string) []models.MemoryRecord {
|
||||
_local.mu.RLock()
|
||||
defer _local.mu.RUnlock()
|
||||
|
||||
@@ -202,6 +202,47 @@ func (mlc *MemLanceClient) GetCandidatesForForgetting() ([]map[string]interface{
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// Scan 内存实现的全表扫描(过滤 is_deleted + namespace)。
|
||||
// 按 CreatedAt 倒序返回(列表/导出更符合直觉:新的在前)。
|
||||
func (mlc *MemLanceClient) Scan(limit int, namespaceFilter string) ([]models.MemoryRecord, error) {
|
||||
if limit <= 0 {
|
||||
limit = 100
|
||||
}
|
||||
mlc.mu.RLock()
|
||||
defer mlc.mu.RUnlock()
|
||||
type item struct {
|
||||
rec models.MemoryRecord
|
||||
ts time.Time
|
||||
}
|
||||
items := make([]item, 0, len(mlc.memories))
|
||||
for _, e := range mlc.memories {
|
||||
if e.IsDeleted {
|
||||
continue
|
||||
}
|
||||
if namespaceFilter != "" && e.Namespace != namespaceFilter {
|
||||
continue
|
||||
}
|
||||
items = append(items, item{
|
||||
rec: models.MemoryRecord{
|
||||
ID: e.ID, Content: e.Content, Category: e.Category,
|
||||
Namespace: e.Namespace, CreatedAt: e.CreatedAt,
|
||||
QualityScore: e.QualityScore, RecallCount: e.RecallCount,
|
||||
Tier: e.Tier, Source: e.Source, Vector: e.Vector,
|
||||
},
|
||||
ts: e.CreatedAt,
|
||||
})
|
||||
}
|
||||
sort.Slice(items, func(i, j int) bool { return items[i].ts.After(items[j].ts) })
|
||||
out := make([]models.MemoryRecord, 0, limit)
|
||||
for _, it := range items {
|
||||
if len(out) >= limit {
|
||||
break
|
||||
}
|
||||
out = append(out, it.rec)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (mlc *MemLanceClient) GetSkillCandidates(minRecalls, limit int) ([]models.MemoryRecord, error) {
|
||||
return nil, nil // 内存实现,暂不支持
|
||||
}
|
||||
|
||||
@@ -20,6 +20,11 @@ type fakeLDB struct {
|
||||
forceBatchErr bool
|
||||
}
|
||||
|
||||
// Scan mock:write-buffer 测试只需占位实现(被测路径不实际调用)。
|
||||
func (f *fakeLDB) Scan(limit int, ns string) ([]models.MemoryRecord, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeLDB) InsertEpisode(agentID, namespace, content, category string) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
package storage
|
||||
|
||||
import "testing"
|
||||
|
||||
// TestMemLanceScanFiltersAndLimits 验证全表扫描的「is_deleted 过滤 + namespace 下推 + limit」。
|
||||
// 这是 /api/v1/memories 与 /api/v1/memories/export 的正确底座(修前它们用零向量 ANN 冒充列表,
|
||||
// 零向量无方向 ⇒ 计数失真)。
|
||||
func TestMemLanceScanFiltersAndLimits(t *testing.T) {
|
||||
mlc := NewMemLanceClient(&Embedder{endpoint: "http://127.0.0.1:1/never", dim: 1024})
|
||||
mlc.memories = map[string]*memEntry{
|
||||
"m1": {ID: "m1", Namespace: "hermes-main", Category: "distilled", Content: "x"},
|
||||
"m2": {ID: "m2", Namespace: "hermes-main", Category: "distilled", Content: "x"},
|
||||
"m3": {ID: "m3", Namespace: "shared", Category: "distilled", Content: "x"},
|
||||
}
|
||||
|
||||
// ① 全表(无 ns)→ 3 条
|
||||
all, err := mlc.Scan(100, "")
|
||||
if err != nil {
|
||||
t.Fatalf("Scan 出错: %v", err)
|
||||
}
|
||||
if len(all) != 3 {
|
||||
t.Errorf("✗ 全表应得 3 条,实际 %d", len(all))
|
||||
}
|
||||
t.Logf("✓ 全表扫描 %d 条", len(all))
|
||||
|
||||
// ② namespace=hermes-main → 2 条
|
||||
hm, _ := mlc.Scan(100, "hermes-main")
|
||||
if len(hm) != 2 {
|
||||
t.Errorf("✗ hermes-main 应得 2 条,实际 %d", len(hm))
|
||||
}
|
||||
t.Logf("✓ hermes-main 过滤 %d 条", len(hm))
|
||||
|
||||
// ③ namespace=shared → 1 条
|
||||
sh, _ := mlc.Scan(100, "shared")
|
||||
if len(sh) != 1 {
|
||||
t.Errorf("✗ shared 应得 1 条,实际 %d", len(sh))
|
||||
}
|
||||
t.Logf("✓ shared 过滤 %d 条", len(sh))
|
||||
|
||||
// ④ limit 生效(limit=1 → 最多 1 条)
|
||||
one, _ := mlc.Scan(1, "")
|
||||
if len(one) > 1 {
|
||||
t.Errorf("✗ limit=1 应 ≤1 条,实际 %d", len(one))
|
||||
}
|
||||
t.Logf("✓ limit 生效 %d 条", len(one))
|
||||
|
||||
// ⑤ 空 ns(不存在的命名空间)→ 0 条
|
||||
none, _ := mlc.Scan(100, "does-not-exist")
|
||||
if len(none) != 0 {
|
||||
t.Errorf("✗ 不存在 ns 应得 0 条,实际 %d", len(none))
|
||||
}
|
||||
t.Logf("✓ 空 ns 过滤 %d 条", len(none))
|
||||
}
|
||||
@@ -330,6 +330,53 @@ func (sc *SQLiteClient) Insert(table string, record any) error {
|
||||
}
|
||||
}
|
||||
|
||||
// Scan SQLite 后端的全表扫描(is_deleted=0 + 可选 namespace,按 created_at 倒序)。
|
||||
func (sc *SQLiteClient) Scan(limit int, namespaceFilter string) ([]models.MemoryRecord, error) {
|
||||
if limit <= 0 {
|
||||
limit = 100
|
||||
}
|
||||
sc.mu.RLock()
|
||||
defer sc.mu.RUnlock()
|
||||
|
||||
q := "SELECT id, content, category, namespace, importance, quality_score, recall_count, tier, source, created_at, updated_at FROM memories WHERE is_deleted=0"
|
||||
if namespaceFilter != "" {
|
||||
q += " AND namespace='" + sanitizeIdent(namespaceFilter) + "'"
|
||||
}
|
||||
q += " ORDER BY created_at DESC LIMIT " + fmt.Sprintf("%d", limit)
|
||||
|
||||
rows, err := sc.query(q)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]models.MemoryRecord, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
rec := models.MemoryRecord{}
|
||||
rec.ID, _ = row["id"].(string)
|
||||
rec.Content, _ = row["content"].(string)
|
||||
rec.Category, _ = row["category"].(string)
|
||||
rec.Namespace, _ = row["namespace"].(string)
|
||||
rec.Importance, _ = row["importance"].(float64)
|
||||
rec.QualityScore, _ = row["quality_score"].(float64)
|
||||
rec.Tier, _ = row["tier"].(string)
|
||||
rec.Source, _ = row["source"].(string)
|
||||
if n, ok := row["recall_count"].(int64); ok {
|
||||
rec.RecallCount = int(n)
|
||||
}
|
||||
if ts, ok := row["created_at"].(string); ok {
|
||||
if t, e := time.Parse(time.RFC3339, ts); e == nil {
|
||||
rec.CreatedAt = t
|
||||
}
|
||||
}
|
||||
if ts, ok := row["updated_at"].(string); ok {
|
||||
if t, e := time.Parse(time.RFC3339, ts); e == nil {
|
||||
rec.UpdatedAt = t
|
||||
}
|
||||
}
|
||||
out = append(out, rec)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (sc *SQLiteClient) Search(table string, vector []float32, topK int, namespaceFilter string) ([]models.MemoryRecord, error) {
|
||||
sc.mu.RLock()
|
||||
defer sc.mu.RUnlock()
|
||||
|
||||
+21
-3
@@ -517,13 +517,31 @@ impl LanceDBOps {
|
||||
// 1. 强制 limit(调用方传, main.rs 钳制硬上限 5000)
|
||||
// 2. 不读 vector 列(1024 维 × 3000 条 ≈ 12MB+,IPC JSON 会卡)
|
||||
// 3. 真读 last_recalled_at 列(scan_all 旧实现 String::new() 恒空 → 遗忘判定失真)
|
||||
pub fn scan_for_forgetting(&self, limit: usize) -> Result<Vec<MemoryRecord>, Box<dyn std::error::Error>> {
|
||||
/// 全表扫描存活记忆(过滤 is_deleted,跳过 vector)。
|
||||
/// ⚠️ 名字来自历史用途(遗忘候选),语义实为「列出存活记忆」。
|
||||
/// 2026-09-21:新增 namespace 下推过滤 —— 修前 Go 侧 /memories 与 /memories/export
|
||||
/// 用「零向量 ANN」冒充列表,零向量无方向 ⇒ 返回任意前 N 行再按 ns 过滤,
|
||||
/// 计数完全失真(实测 hermes-main 报 1 条,实际 139+)。
|
||||
/// 用 only_if 下推过滤,limit 作用于**过滤后**的结果集,避免 5000 硬上限截断。
|
||||
pub fn scan_for_forgetting(
|
||||
&self,
|
||||
limit: usize,
|
||||
namespace: Option<&str>,
|
||||
) -> Result<Vec<MemoryRecord>, Box<dyn std::error::Error>> {
|
||||
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
|
||||
let tbl = rt().block_on(db.open_table("memories").execute())?;
|
||||
|
||||
let mut filter = String::from("is_deleted = false");
|
||||
if let Some(ns) = namespace {
|
||||
if !ns.is_empty() {
|
||||
let escaped = ns.replace('\\', "\\\\").replace('\'', "''");
|
||||
filter.push_str(&format!(" AND namespace = '{}'", escaped));
|
||||
}
|
||||
}
|
||||
|
||||
let mut results = Box::pin(rt().block_on(
|
||||
tbl.query()
|
||||
.only_if("is_deleted = false")
|
||||
.only_if(&filter)
|
||||
.limit(limit)
|
||||
.execute()
|
||||
)?);
|
||||
@@ -558,7 +576,7 @@ impl LanceDBOps {
|
||||
});
|
||||
}
|
||||
}
|
||||
eprintln!("[lancedb] scan_for_forgetting → {} records (limit={})", records.len(), limit);
|
||||
eprintln!("[lancedb] scan_for_forgetting → {} records (limit={}, ns={:?})", records.len(), limit, namespace);
|
||||
Ok(records)
|
||||
}
|
||||
|
||||
|
||||
+4
-2
@@ -369,8 +369,10 @@ fn handle_client(mut stream: UnixStream, args: &Args, lancedb: LanceDBOps, state
|
||||
if limit > 5000 {
|
||||
limit = 5000;
|
||||
}
|
||||
eprintln!("[lancedb] scan_for_forgetting: limit={}", limit);
|
||||
match lancedb.scan_for_forgetting(limit) {
|
||||
// 2026-09-21: 可选 namespace 下推过滤(列表/导出端点用)
|
||||
let namespace = msg["namespace"].as_str().filter(|s| !s.is_empty());
|
||||
eprintln!("[lancedb] scan_for_forgetting: limit={} namespace={:?}", limit, namespace);
|
||||
match lancedb.scan_for_forgetting(limit, namespace) {
|
||||
Ok(r) => send_ok(&mut stream, &serde_json::to_string(&r).unwrap_or_default()),
|
||||
Err(e) => send_error(&mut stream, "lancedb_scan", &e.to_string()),
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user