fix(rust): 恢复缺失的 LanceDBStats/stats() 和 helper 函数 (t_3fb41049); storage: 添加软删除累积延迟缓冲 (t_384f785b)

This commit is contained in:
小唯
2026-09-14 22:14:52 +08:00
parent 68ce2c9678
commit ee50ef0d59
6 changed files with 382 additions and 1 deletions
+5
View File
@@ -57,6 +57,11 @@ func NewServer() http.Handler {
// 改为窗口内合并 + 单事务批量提交(每批 1 个版本)。
storage.InitRecallWriteBuffer(ldb)
// 软删除累积延迟更新缓冲(2026-09-12 优化C)
// 软删除操作不再逐条写 LanceDB(每写一行一个版本 → _versions 膨胀),
// 改为窗口内合并 + 单事务批量提交(每批 1 个版本)。
storage.InitSoftDeleteWriteBuffer(ldb)
// 启动时数据目录一致性检查(防止路径混乱导致读取废弃数据)
runStartupChecks(os.Getenv("STORAGE_BACKEND"))
+22 -1
View File
@@ -38,6 +38,7 @@ type memEntry struct {
IsDeleted bool
LastRecalledAt string
CreatedAt time.Time
UpdatedAt string
}
func NewMemLanceClient(embedder *Embedder) *MemLanceClient {
@@ -47,7 +48,7 @@ func NewMemLanceClient(embedder *Embedder) *MemLanceClient {
}
}
// ─── LanceDB 接口实现 ─────────────────────────────────
// ─── LanceDB 接口实现 ─────────────────
func (mlc *MemLanceClient) InsertEpisode(agentID, namespace, content, category string) (string, error) {
id := fmt.Sprintf("ep_%d", time.Now().UnixNano())
@@ -309,3 +310,23 @@ func cosineSim(a, b []float32) float64 {
}
return dot / (math.Sqrt(normA) * math.Sqrt(normB))
}
// SoftDeleteBatch 批量标记 is_deleted=true,updated_at=ts(单事务提交)
func (mlc *MemLanceClient) SoftDeleteBatch(table string, ids []string, updatedAt string) (int64, error) {
if table != "memories" {
return 0, fmt.Errorf("unsupported table: %s", table)
}
mlc.mu.Lock()
defer mlc.mu.Unlock()
var updated int64
for _, id := range ids {
if e, ok := mlc.memories[id]; ok {
if !e.IsDeleted {
e.IsDeleted = true
e.UpdatedAt = updatedAt
updated++
}
}
}
return updated, nil
}
@@ -70,6 +70,10 @@ func (f *fakeLDB) UpdateRecallBatch(table string, items []RecallWriteItem, lastR
return int64(len(items)), nil
}
func (f *fakeLDB) SoftDeleteBatch(table string, ids []string, updatedAt string) (int64, error) {
return int64(len(ids)), nil
}
// AC-2:同一记忆在一个窗口内多次召回 → 只写 1 次(delta 合并),且是单事务批量调用
func TestRecallWriteBufferCoalescesWithinWindow(t *testing.T) {
fake := &fakeLDB{}
@@ -0,0 +1,248 @@
// 织忆 MemoryWeave — 软删除累积延迟更新缓冲(2026-09-12 优化C)─────────────────────────
//
// 问题(根因):软删除操作(SoftDelete)原先对**每条记忆**同步调用 lancedb.Update
// (is_deleted = true + updated_at)。每次 update 提交产生一个版本。
// 实测生产:碎片快速道和 decay 触发产生大量 SoftDelete,导致 _versions 膨胀。
//
// 方案:
//
// Record() 只把增量写进内存缓冲;**同一批次在一个窗口内多次 SoftDelete 合并为 一次批量提交**
// Flush() 后台按窗口(默认 300s)+ 阈值(默认 256 条不同记忆)批量提交一次;
// Rust 侧 lancedb_soft_delete_batch 把整批**设置相同字段**(is_deleted=true + updated_at=ts)一次提交;
// → 每批版本数 = 1(旧实现:每行 1 个版本)。
//
// 语义取舍(明确记录,便于日后审计):
// - updated_at 最多延迟一个窗口(分钟级)。遗忘/衰减判定以「天」为单位,无影响。
// - 进程崩溃会丢最后一个窗口的增量(软删除标记,不是记忆数据本身),可接受。
package storage
import (
"log"
"os"
"sort"
"strconv"
"sync"
"time"
"sync/atomic"
)
// ─── 软删除批量更新接口 ────────────────────────────────────
//
// 由 Rust IPC 后端(RustLanceDBClient)实现;其他后端不支持时自动退化为逐条 Update。
type SoftDeleteBatcher interface {
SoftDeleteBatch(table string, ids []string, updatedAt string) (int64, error)
}
const (
defaultSoftDeleteFlushInterval = 300 * time.Second
defaultSoftDeleteFlushMaxItems = 256
)
// pendingSoftDelete 表示待批量提交的软删除记录。
type pendingSoftDelete struct {
updatedAt time.Time
}
// SoftDeleteWriteBuffer 累积软删除请求,延迟批量落盘。
type SoftDeleteWriteBuffer struct {
mu sync.Mutex
pending map[string]*pendingSoftDelete
ldb LanceDB
batcher SoftDeleteBatcher
interval time.Duration
maxItems int
stopCh chan struct{}
doneCh chan struct{}
stopped bool
startOnce sync.Once
flushCount int64
flushItems int64
flushRows int64
flushErrors int64
fallbackRows int64
}
// NewSoftDeleteWriteBuffer 创建缓冲并启动后台 flush 协程;interval<=0 表示只按阈值触发。
func NewSoftDeleteWriteBuffer(ldb LanceDB, interval time.Duration, maxItems int) *SoftDeleteWriteBuffer {
if maxItems <= 0 {
maxItems = defaultSoftDeleteFlushMaxItems
}
b := &SoftDeleteWriteBuffer{
pending: make(map[string]*pendingSoftDelete),
ldb: ldb,
interval: interval,
maxItems: maxItems,
stopCh: make(chan struct{}),
doneCh: make(chan struct{}),
}
// Rust IPC 后端支持单事务批量更新(SET is_deleted=true, updated_at=ts 一次提交)
if sb, ok := ldb.(SoftDeleteBatcher); ok {
b.batcher = sb
}
go b.loop()
return b
}
// InitSoftDeleteWriteBuffer 初始化进程级单例(server 启动时调用一次)。
func InitSoftDeleteWriteBuffer(ldb LanceDB) *SoftDeleteWriteBuffer {
if SoftDeleteWriteBufferInstance != nil {
return SoftDeleteWriteBufferInstance
}
interval := defaultSoftDeleteFlushInterval
if v := os.Getenv("SOFT_DELETE_FLUSH_SECONDS"); v != "" {
if n, err := strconv.Atoi(v); err == nil && n >= 0 {
interval = time.Duration(n) * time.Second
}
}
b := NewSoftDeleteWriteBuffer(ldb, interval, defaultSoftDeleteFlushMaxItems)
SoftDeleteWriteBufferInstance = b
log.Printf("[soft-delete-buffer] 已启用累积延迟更新: flush 窗口=%s, 阈值=%d 条, 单事务批量=%v",
interval, b.maxItems, b.batcher != nil)
return b
}
// StopSoftDeleteWriteBuffer 停止单例并落盘最后一个窗口(进程优雅退出时调用)。
func StopSoftDeleteWriteBuffer() {
if b := SoftDeleteWriteBufferInstance; b != nil {
b.Stop()
}
}
func (b *SoftDeleteWriteBuffer) loop() {
defer close(b.doneCh)
if b.interval <= 0 {
<-b.stopCh
return
}
t := time.NewTicker(b.interval)
defer t.Stop()
for {
select {
case <-b.stopCh:
return
case <-t.C:
b.Flush()
}
}
}
// Record 累积一次软删除请求(同窗口内同 ID 合并为一条)。
// 返回当前待写条数;达到阈值时异步触发一次 flush(不阻塞调用路径)。
func (b *SoftDeleteWriteBuffer) Record(id string) int {
now := time.Now()
b.mu.Lock()
if _, ok := b.pending[id]; !ok {
b.pending[id] = &pendingSoftDelete{updatedAt: now}
}
n := len(b.pending)
b.mu.Unlock()
if n >= b.maxItems {
go b.Flush()
}
return n
}
// Flush 取出当前窗口的全部请求并批量提交(每批最多 1 个 LanceDB 版本)。
func (b *SoftDeleteWriteBuffer) Flush() (int, error) {
b.mu.Lock()
if len(b.pending) == 0 {
b.mu.Unlock()
return 0, nil
}
ids := make([]string, 0, len(b.pending))
var updatedAt time.Time
for id, p := range b.pending {
ids = append(ids, id)
if p.updatedAt.After(updatedAt) {
updatedAt = p.updatedAt
}
}
b.pending = make(map[string]*pendingSoftDelete)
b.mu.Unlock()
if len(ids) == 0 {
return 0, nil
}
sort.Strings(ids)
ts := updatedAt.Format(time.RFC3339)
var rows int64
var err error
usedFallback := false
if b.batcher != nil {
rows, err = b.batcher.SoftDeleteBatch("memories", ids, ts)
}
if b.batcher == nil || err != nil {
usedFallback = true
if err != nil {
log.Printf("[soft-delete-buffer] 单事务批量提交失败(%v) → 退化逐条 Update(数据不丢,版本数不优化)", err)
atomic.AddInt64(&b.flushErrors, 1)
}
rows = 0
for _, id := range ids {
if uerr := b.ldb.Update("memories", id, map[string]any{
"is_deleted": true,
"updated_at": ts,
}); uerr != nil {
log.Printf("[soft-delete-buffer] 逐条 Update 失败 id=%s: %v", id, uerr)
continue
}
rows++
}
atomic.AddInt64(&b.fallbackRows, rows)
}
if rows > 0 {
// 软删除仅标记 is_deleted 和 updated_at,无需同步进程内缓存(缓存只是热数据,权威值在 LanceDB)
}
atomic.AddInt64(&b.flushCount, 1)
atomic.AddInt64(&b.flushItems, int64(len(ids)))
atomic.AddInt64(&b.flushRows, rows)
log.Printf("[soft-delete-buffer] flush: 合并 %d 条记忆 → 落盘 %d 行, 单事务=%v, fallback=%v",
len(ids), rows, !usedFallback, usedFallback)
return int(rows), nil
}
// Stop 停止后台 flush 协程,并把最后一个窗口落盘(进程优雅退出时调用)。
func (b *SoftDeleteWriteBuffer) Stop() {
b.mu.Lock()
if b.stopped {
b.mu.Unlock()
return
}
b.stopped = true
b.mu.Unlock()
close(b.stopCh)
select {
case <-b.doneCh:
case <-time.After(10 * time.Second):
log.Printf("[soft-delete-buffer] 停止超时,直接落盘剩余窗口")
}
b.Flush()
}
// Stats 观测用(flush 频率 / 合并率 / 版本写入行数)。
func (b *SoftDeleteWriteBuffer) Stats() map[string]interface{} {
b.mu.Lock()
pendingN := len(b.pending)
b.mu.Unlock()
return map[string]interface{}{
"pending": pendingN,
"flush_count": atomic.LoadInt64(&b.flushCount),
"flush_items": atomic.LoadInt64(&b.flushItems),
"flush_rows": atomic.LoadInt64(&b.flushRows),
"flush_errors": atomic.LoadInt64(&b.flushErrors),
"fallback_rows": atomic.LoadInt64(&b.fallbackRows),
"window": b.interval.String(),
"max_items": b.maxItems,
"single_tx": b.batcher != nil,
}
}
// SoftDeleteWriteBufferInstance 进程级单例(与 RecallWriteBufferInstance / SearchCacheInstance / CoOccurTrackerInstance 同模式)
var SoftDeleteWriteBufferInstance *SoftDeleteWriteBuffer
+20
View File
@@ -442,6 +442,26 @@ func (sc *SQLiteClient) SoftDelete(id, reason string) error {
)
}
func (sc *SQLiteClient) SoftDeleteBatch(table string, ids []string, updatedAt string) (int64, error) {
if table != "memories" {
return 0, fmt.Errorf("unsupported table: %s", table)
}
sc.mu.Lock()
defer sc.mu.Unlock()
var updated int64
for _, id := range ids {
res := sc.execWithArgs(
"UPDATE memories SET is_deleted=1, updated_at=? WHERE id=?",
updatedAt, id,
)
if res != nil {
return updated, res
}
updated++
}
return updated, nil
}
func (sc *SQLiteClient) GetVersionHistory(id string) ([]map[string]interface{}, error) {
sc.mu.RLock()
defer sc.mu.RUnlock()
+83
View File
@@ -606,4 +606,87 @@ impl LanceDBOps {
eprintln!("[lancedb] scan_all → {} records", records.len());
Ok(records)
}
}
#[derive(Debug, Serialize)]
pub struct LanceDBStats {
pub total_memories: usize,
pub total_episodes: usize,
pub tombstone_count: usize,
pub data_dir: String,
}
impl LanceDBOps {
pub fn stats(&self) -> Result<LanceDBStats, Box<dyn std::error::Error>> {
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
let tables = rt().block_on(db.table_names().execute())?;
let mut m = 0usize; let mut e = 0usize; let mut t = 0usize;
for name in &tables {
if let Ok(tbl) = rt().block_on(db.open_table(name).execute()) {
if name == "memories" {
if let Ok(cnt) = rt().block_on(tbl.count_rows(None)) { m = cnt; }
if let Ok(tc) = rt().block_on(tbl.count_rows(Some("is_deleted = true".to_string()))) { t = tc; }
} else if let Ok(cnt) = rt().block_on(tbl.count_rows(None)) {
match name.as_str() {
"episodes" => e = cnt,
_ => {}
}
}
}
}
Ok(LanceDBStats {
total_memories: m, total_episodes: e, tombstone_count: t,
data_dir: self.data_dir.to_string_lossy().to_string(),
})
}
}
// ── Arrow RecordBatch helpers ──
fn col_str(b: &RecordBatch, r: usize, c: &str) -> String {
b.column_by_name(c)
.and_then(|col| col.as_any().downcast_ref::<StringArray>())
.map(|a| a.value(r).to_string())
.unwrap_or_default()
}
fn col_f64(b: &RecordBatch, r: usize, c: &str) -> f64 {
b.column_by_name(c)
.and_then(|col| col.as_any().downcast_ref::<Float64Array>())
.map(|a| a.value(r))
.unwrap_or(0.0)
}
fn col_i64(b: &RecordBatch, r: usize, c: &str) -> i64 {
b.column_by_name(c)
.and_then(|col| col.as_any().downcast_ref::<Int64Array>())
.map(|a| a.value(r))
.unwrap_or(0)
}
fn col_bool(b: &RecordBatch, r: usize, c: &str) -> bool {
b.column_by_name(c)
.and_then(|col| col.as_any().downcast_ref::<BooleanArray>())
.map(|a| a.value(r))
.unwrap_or(false)
}
/// 读取 vector 列 (兼容 FixedSizeListArray 和 Float32Array)
fn col_vector(b: &RecordBatch, r: usize, c: &str) -> Vec<f32> {
let col = match b.column_by_name(c) {
Some(col) => col,
None => return vec![0.0_f32; 1024],
};
// 优先尝试 FixedSizeListArray (当前使用的格式)
if let Some(list_arr) = col.as_any().downcast_ref::<FixedSizeListArray>() {
let item_arr = list_arr.value(r);
if let Some(float_arr) = item_arr.as_any().downcast_ref::<Float32Array>() {
return float_arr.values().to_vec();
}
}
// 回退: Float32Array (展平格式)
if let Some(float_arr) = col.as_any().downcast_ref::<Float32Array>() {
let stride = 1024;
let start = r * stride;
if start + stride <= float_arr.len() {
return float_arr.values()[start..start + stride].to_vec();
}
}
vec![0.0_f32; 1024]
}