From ee50ef0d599f507413cee8d33f00b94ccc12015b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=94=AF?= Date: Mon, 14 Sep 2026 22:14:52 +0800 Subject: [PATCH] =?UTF-8?q?fix(rust):=20=E6=81=A2=E5=A4=8D=E7=BC=BA?= =?UTF-8?q?=E5=A4=B1=E7=9A=84=20LanceDBStats/stats()=20=E5=92=8C=20helper?= =?UTF-8?q?=20=E5=87=BD=E6=95=B0=20(t=5F3fb41049);=20storage:=20=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E8=BD=AF=E5=88=A0=E9=99=A4=E7=B4=AF=E7=A7=AF=E5=BB=B6?= =?UTF-8?q?=E8=BF=9F=E7=BC=93=E5=86=B2=20(t=5F384f785b)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go/internal/api/server.go | 5 + go/internal/storage/memvector.go | 23 +- .../storage/recall_write_buffer_test.go | 4 + .../storage/soft_delete_write_buffer.go | 248 ++++++++++++++++++ go/internal/storage/sqlite.go | 20 ++ rust/src/lancedb_ops.rs | 83 ++++++ 6 files changed, 382 insertions(+), 1 deletion(-) create mode 100644 go/internal/storage/soft_delete_write_buffer.go diff --git a/go/internal/api/server.go b/go/internal/api/server.go index 9d6a96c..33d8099 100644 --- a/go/internal/api/server.go +++ b/go/internal/api/server.go @@ -57,6 +57,11 @@ func NewServer() http.Handler { // 改为窗口内合并 + 单事务批量提交(每批 1 个版本)。 storage.InitRecallWriteBuffer(ldb) + // 软删除累积延迟更新缓冲(2026-09-12 优化C) + // 软删除操作不再逐条写 LanceDB(每写一行一个版本 → _versions 膨胀), + // 改为窗口内合并 + 单事务批量提交(每批 1 个版本)。 + storage.InitSoftDeleteWriteBuffer(ldb) + // 启动时数据目录一致性检查(防止路径混乱导致读取废弃数据) runStartupChecks(os.Getenv("STORAGE_BACKEND")) diff --git a/go/internal/storage/memvector.go b/go/internal/storage/memvector.go index 06e868c..b171acf 100644 --- a/go/internal/storage/memvector.go +++ b/go/internal/storage/memvector.go @@ -38,6 +38,7 @@ type memEntry struct { IsDeleted bool LastRecalledAt string CreatedAt time.Time + UpdatedAt string } func NewMemLanceClient(embedder *Embedder) *MemLanceClient { @@ -47,7 +48,7 @@ func NewMemLanceClient(embedder *Embedder) *MemLanceClient { } } -// ─── LanceDB 接口实现 ───────────────────────────────── +// ─── LanceDB 接口实现 ───────────────── func (mlc *MemLanceClient) InsertEpisode(agentID, namespace, content, category string) (string, error) { id := fmt.Sprintf("ep_%d", time.Now().UnixNano()) @@ -309,3 +310,23 @@ func cosineSim(a, b []float32) float64 { } return dot / (math.Sqrt(normA) * math.Sqrt(normB)) } + +// SoftDeleteBatch 批量标记 is_deleted=true,updated_at=ts(单事务提交) +func (mlc *MemLanceClient) SoftDeleteBatch(table string, ids []string, updatedAt string) (int64, error) { + if table != "memories" { + return 0, fmt.Errorf("unsupported table: %s", table) + } + mlc.mu.Lock() + defer mlc.mu.Unlock() + var updated int64 + for _, id := range ids { + if e, ok := mlc.memories[id]; ok { + if !e.IsDeleted { + e.IsDeleted = true + e.UpdatedAt = updatedAt + updated++ + } + } + } + return updated, nil +} \ No newline at end of file diff --git a/go/internal/storage/recall_write_buffer_test.go b/go/internal/storage/recall_write_buffer_test.go index 16f554b..f91f8f9 100644 --- a/go/internal/storage/recall_write_buffer_test.go +++ b/go/internal/storage/recall_write_buffer_test.go @@ -70,6 +70,10 @@ func (f *fakeLDB) UpdateRecallBatch(table string, items []RecallWriteItem, lastR return int64(len(items)), nil } +func (f *fakeLDB) SoftDeleteBatch(table string, ids []string, updatedAt string) (int64, error) { + return int64(len(ids)), nil +} + // AC-2:同一记忆在一个窗口内多次召回 → 只写 1 次(delta 合并),且是单事务批量调用 func TestRecallWriteBufferCoalescesWithinWindow(t *testing.T) { fake := &fakeLDB{} diff --git a/go/internal/storage/soft_delete_write_buffer.go b/go/internal/storage/soft_delete_write_buffer.go new file mode 100644 index 0000000..57c60c4 --- /dev/null +++ b/go/internal/storage/soft_delete_write_buffer.go @@ -0,0 +1,248 @@ +// 织忆 MemoryWeave — 软删除累积延迟更新缓冲(2026-09-12 优化C)───────────────────────── +// +// 问题(根因):软删除操作(SoftDelete)原先对**每条记忆**同步调用 lancedb.Update +// (is_deleted = true + updated_at)。每次 update 提交产生一个版本。 +// 实测生产:碎片快速道和 decay 触发产生大量 SoftDelete,导致 _versions 膨胀。 +// +// 方案: +// +// Record() 只把增量写进内存缓冲;**同一批次在一个窗口内多次 SoftDelete 合并为 一次批量提交** +// Flush() 后台按窗口(默认 300s)+ 阈值(默认 256 条不同记忆)批量提交一次; +// Rust 侧 lancedb_soft_delete_batch 把整批**设置相同字段**(is_deleted=true + updated_at=ts)一次提交; +// → 每批版本数 = 1(旧实现:每行 1 个版本)。 +// +// 语义取舍(明确记录,便于日后审计): +// - updated_at 最多延迟一个窗口(分钟级)。遗忘/衰减判定以「天」为单位,无影响。 +// - 进程崩溃会丢最后一个窗口的增量(软删除标记,不是记忆数据本身),可接受。 +package storage + +import ( + "log" + "os" + "sort" + "strconv" + "sync" + "time" + "sync/atomic" +) + +// ─── 软删除批量更新接口 ──────────────────────────────────── +// +// 由 Rust IPC 后端(RustLanceDBClient)实现;其他后端不支持时自动退化为逐条 Update。 +type SoftDeleteBatcher interface { + SoftDeleteBatch(table string, ids []string, updatedAt string) (int64, error) +} + +const ( + defaultSoftDeleteFlushInterval = 300 * time.Second + defaultSoftDeleteFlushMaxItems = 256 +) + +// pendingSoftDelete 表示待批量提交的软删除记录。 +type pendingSoftDelete struct { + updatedAt time.Time +} + +// SoftDeleteWriteBuffer 累积软删除请求,延迟批量落盘。 +type SoftDeleteWriteBuffer struct { + mu sync.Mutex + pending map[string]*pendingSoftDelete + ldb LanceDB + batcher SoftDeleteBatcher + interval time.Duration + maxItems int + + stopCh chan struct{} + doneCh chan struct{} + stopped bool + startOnce sync.Once + + flushCount int64 + flushItems int64 + flushRows int64 + flushErrors int64 + fallbackRows int64 +} + +// NewSoftDeleteWriteBuffer 创建缓冲并启动后台 flush 协程;interval<=0 表示只按阈值触发。 +func NewSoftDeleteWriteBuffer(ldb LanceDB, interval time.Duration, maxItems int) *SoftDeleteWriteBuffer { + if maxItems <= 0 { + maxItems = defaultSoftDeleteFlushMaxItems + } + b := &SoftDeleteWriteBuffer{ + pending: make(map[string]*pendingSoftDelete), + ldb: ldb, + interval: interval, + maxItems: maxItems, + stopCh: make(chan struct{}), + doneCh: make(chan struct{}), + } + // Rust IPC 后端支持单事务批量更新(SET is_deleted=true, updated_at=ts 一次提交) + if sb, ok := ldb.(SoftDeleteBatcher); ok { + b.batcher = sb + } + go b.loop() + return b +} + +// InitSoftDeleteWriteBuffer 初始化进程级单例(server 启动时调用一次)。 +func InitSoftDeleteWriteBuffer(ldb LanceDB) *SoftDeleteWriteBuffer { + if SoftDeleteWriteBufferInstance != nil { + return SoftDeleteWriteBufferInstance + } + interval := defaultSoftDeleteFlushInterval + if v := os.Getenv("SOFT_DELETE_FLUSH_SECONDS"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n >= 0 { + interval = time.Duration(n) * time.Second + } + } + b := NewSoftDeleteWriteBuffer(ldb, interval, defaultSoftDeleteFlushMaxItems) + SoftDeleteWriteBufferInstance = b + log.Printf("[soft-delete-buffer] 已启用累积延迟更新: flush 窗口=%s, 阈值=%d 条, 单事务批量=%v", + interval, b.maxItems, b.batcher != nil) + return b +} + +// StopSoftDeleteWriteBuffer 停止单例并落盘最后一个窗口(进程优雅退出时调用)。 +func StopSoftDeleteWriteBuffer() { + if b := SoftDeleteWriteBufferInstance; b != nil { + b.Stop() + } +} + +func (b *SoftDeleteWriteBuffer) loop() { + defer close(b.doneCh) + if b.interval <= 0 { + <-b.stopCh + return + } + t := time.NewTicker(b.interval) + defer t.Stop() + for { + select { + case <-b.stopCh: + return + case <-t.C: + b.Flush() + } + } +} + +// Record 累积一次软删除请求(同窗口内同 ID 合并为一条)。 +// 返回当前待写条数;达到阈值时异步触发一次 flush(不阻塞调用路径)。 +func (b *SoftDeleteWriteBuffer) Record(id string) int { + now := time.Now() + b.mu.Lock() + if _, ok := b.pending[id]; !ok { + b.pending[id] = &pendingSoftDelete{updatedAt: now} + } + n := len(b.pending) + b.mu.Unlock() + + if n >= b.maxItems { + go b.Flush() + } + return n +} + +// Flush 取出当前窗口的全部请求并批量提交(每批最多 1 个 LanceDB 版本)。 +func (b *SoftDeleteWriteBuffer) Flush() (int, error) { + b.mu.Lock() + if len(b.pending) == 0 { + b.mu.Unlock() + return 0, nil + } + ids := make([]string, 0, len(b.pending)) + var updatedAt time.Time + for id, p := range b.pending { + ids = append(ids, id) + if p.updatedAt.After(updatedAt) { + updatedAt = p.updatedAt + } + } + b.pending = make(map[string]*pendingSoftDelete) + b.mu.Unlock() + + if len(ids) == 0 { + return 0, nil + } + sort.Strings(ids) + + ts := updatedAt.Format(time.RFC3339) + var rows int64 + var err error + usedFallback := false + if b.batcher != nil { + rows, err = b.batcher.SoftDeleteBatch("memories", ids, ts) + } + if b.batcher == nil || err != nil { + usedFallback = true + if err != nil { + log.Printf("[soft-delete-buffer] 单事务批量提交失败(%v) → 退化逐条 Update(数据不丢,版本数不优化)", err) + atomic.AddInt64(&b.flushErrors, 1) + } + rows = 0 + for _, id := range ids { + if uerr := b.ldb.Update("memories", id, map[string]any{ + "is_deleted": true, + "updated_at": ts, + }); uerr != nil { + log.Printf("[soft-delete-buffer] 逐条 Update 失败 id=%s: %v", id, uerr) + continue + } + rows++ + } + atomic.AddInt64(&b.fallbackRows, rows) + } + + if rows > 0 { + // 软删除仅标记 is_deleted 和 updated_at,无需同步进程内缓存(缓存只是热数据,权威值在 LanceDB) + } + + atomic.AddInt64(&b.flushCount, 1) + atomic.AddInt64(&b.flushItems, int64(len(ids))) + atomic.AddInt64(&b.flushRows, rows) + log.Printf("[soft-delete-buffer] flush: 合并 %d 条记忆 → 落盘 %d 行, 单事务=%v, fallback=%v", + len(ids), rows, !usedFallback, usedFallback) + return int(rows), nil +} + +// Stop 停止后台 flush 协程,并把最后一个窗口落盘(进程优雅退出时调用)。 +func (b *SoftDeleteWriteBuffer) Stop() { + b.mu.Lock() + if b.stopped { + b.mu.Unlock() + return + } + b.stopped = true + b.mu.Unlock() + + close(b.stopCh) + select { + case <-b.doneCh: + case <-time.After(10 * time.Second): + log.Printf("[soft-delete-buffer] 停止超时,直接落盘剩余窗口") + } + b.Flush() +} + +// Stats 观测用(flush 频率 / 合并率 / 版本写入行数)。 +func (b *SoftDeleteWriteBuffer) Stats() map[string]interface{} { + b.mu.Lock() + pendingN := len(b.pending) + b.mu.Unlock() + return map[string]interface{}{ + "pending": pendingN, + "flush_count": atomic.LoadInt64(&b.flushCount), + "flush_items": atomic.LoadInt64(&b.flushItems), + "flush_rows": atomic.LoadInt64(&b.flushRows), + "flush_errors": atomic.LoadInt64(&b.flushErrors), + "fallback_rows": atomic.LoadInt64(&b.fallbackRows), + "window": b.interval.String(), + "max_items": b.maxItems, + "single_tx": b.batcher != nil, + } +} + +// SoftDeleteWriteBufferInstance 进程级单例(与 RecallWriteBufferInstance / SearchCacheInstance / CoOccurTrackerInstance 同模式) +var SoftDeleteWriteBufferInstance *SoftDeleteWriteBuffer \ No newline at end of file diff --git a/go/internal/storage/sqlite.go b/go/internal/storage/sqlite.go index f7a9cfb..41cb79c 100644 --- a/go/internal/storage/sqlite.go +++ b/go/internal/storage/sqlite.go @@ -442,6 +442,26 @@ func (sc *SQLiteClient) SoftDelete(id, reason string) error { ) } +func (sc *SQLiteClient) SoftDeleteBatch(table string, ids []string, updatedAt string) (int64, error) { + if table != "memories" { + return 0, fmt.Errorf("unsupported table: %s", table) + } + sc.mu.Lock() + defer sc.mu.Unlock() + var updated int64 + for _, id := range ids { + res := sc.execWithArgs( + "UPDATE memories SET is_deleted=1, updated_at=? WHERE id=?", + updatedAt, id, + ) + if res != nil { + return updated, res + } + updated++ + } + return updated, nil +} + func (sc *SQLiteClient) GetVersionHistory(id string) ([]map[string]interface{}, error) { sc.mu.RLock() defer sc.mu.RUnlock() diff --git a/rust/src/lancedb_ops.rs b/rust/src/lancedb_ops.rs index 610a60e..0783aa6 100644 --- a/rust/src/lancedb_ops.rs +++ b/rust/src/lancedb_ops.rs @@ -606,4 +606,87 @@ impl LanceDBOps { eprintln!("[lancedb] scan_all → {} records", records.len()); Ok(records) } +} + +#[derive(Debug, Serialize)] +pub struct LanceDBStats { + pub total_memories: usize, + pub total_episodes: usize, + pub tombstone_count: usize, + pub data_dir: String, +} + +impl LanceDBOps { + pub fn stats(&self) -> Result> { + let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?; + let tables = rt().block_on(db.table_names().execute())?; + let mut m = 0usize; let mut e = 0usize; let mut t = 0usize; + for name in &tables { + if let Ok(tbl) = rt().block_on(db.open_table(name).execute()) { + if name == "memories" { + if let Ok(cnt) = rt().block_on(tbl.count_rows(None)) { m = cnt; } + if let Ok(tc) = rt().block_on(tbl.count_rows(Some("is_deleted = true".to_string()))) { t = tc; } + } else if let Ok(cnt) = rt().block_on(tbl.count_rows(None)) { + match name.as_str() { + "episodes" => e = cnt, + _ => {} + } + } + } + } + Ok(LanceDBStats { + total_memories: m, total_episodes: e, tombstone_count: t, + data_dir: self.data_dir.to_string_lossy().to_string(), + }) + } +} + +// ── Arrow RecordBatch helpers ── +fn col_str(b: &RecordBatch, r: usize, c: &str) -> String { + b.column_by_name(c) + .and_then(|col| col.as_any().downcast_ref::()) + .map(|a| a.value(r).to_string()) + .unwrap_or_default() +} +fn col_f64(b: &RecordBatch, r: usize, c: &str) -> f64 { + b.column_by_name(c) + .and_then(|col| col.as_any().downcast_ref::()) + .map(|a| a.value(r)) + .unwrap_or(0.0) +} +fn col_i64(b: &RecordBatch, r: usize, c: &str) -> i64 { + b.column_by_name(c) + .and_then(|col| col.as_any().downcast_ref::()) + .map(|a| a.value(r)) + .unwrap_or(0) +} +fn col_bool(b: &RecordBatch, r: usize, c: &str) -> bool { + b.column_by_name(c) + .and_then(|col| col.as_any().downcast_ref::()) + .map(|a| a.value(r)) + .unwrap_or(false) +} + +/// 读取 vector 列 (兼容 FixedSizeListArray 和 Float32Array) +fn col_vector(b: &RecordBatch, r: usize, c: &str) -> Vec { + let col = match b.column_by_name(c) { + Some(col) => col, + None => return vec![0.0_f32; 1024], + }; + // 优先尝试 FixedSizeListArray (当前使用的格式) + if let Some(list_arr) = col.as_any().downcast_ref::() { + let item_arr = list_arr.value(r); + if let Some(float_arr) = item_arr.as_any().downcast_ref::() { + return float_arr.values().to_vec(); + } + } + // 回退: Float32Array (展平格式) + if let Some(float_arr) = col.as_any().downcast_ref::() { + let stride = 1024; + let start = r * stride; + if start + stride <= float_arr.len() { + return float_arr.values()[start..start + stride].to_vec(); + } + } + vec![0.0_f32; 1024] } \ No newline at end of file