fix(rust): 恢复缺失的 LanceDBStats/stats() 和 helper 函数 (t_3fb41049); storage: 添加软删除累积延迟缓冲 (t_384f785b)
This commit is contained in:
@@ -57,6 +57,11 @@ func NewServer() http.Handler {
|
||||
// 改为窗口内合并 + 单事务批量提交(每批 1 个版本)。
|
||||
storage.InitRecallWriteBuffer(ldb)
|
||||
|
||||
// 软删除累积延迟更新缓冲(2026-09-12 优化C)
|
||||
// 软删除操作不再逐条写 LanceDB(每写一行一个版本 → _versions 膨胀),
|
||||
// 改为窗口内合并 + 单事务批量提交(每批 1 个版本)。
|
||||
storage.InitSoftDeleteWriteBuffer(ldb)
|
||||
|
||||
// 启动时数据目录一致性检查(防止路径混乱导致读取废弃数据)
|
||||
runStartupChecks(os.Getenv("STORAGE_BACKEND"))
|
||||
|
||||
|
||||
@@ -38,6 +38,7 @@ type memEntry struct {
|
||||
IsDeleted bool
|
||||
LastRecalledAt string
|
||||
CreatedAt time.Time
|
||||
UpdatedAt string
|
||||
}
|
||||
|
||||
func NewMemLanceClient(embedder *Embedder) *MemLanceClient {
|
||||
@@ -47,7 +48,7 @@ func NewMemLanceClient(embedder *Embedder) *MemLanceClient {
|
||||
}
|
||||
}
|
||||
|
||||
// ─── LanceDB 接口实现 ─────────────────────────────────
|
||||
// ─── LanceDB 接口实现 ─────────────────
|
||||
|
||||
func (mlc *MemLanceClient) InsertEpisode(agentID, namespace, content, category string) (string, error) {
|
||||
id := fmt.Sprintf("ep_%d", time.Now().UnixNano())
|
||||
@@ -309,3 +310,23 @@ func cosineSim(a, b []float32) float64 {
|
||||
}
|
||||
return dot / (math.Sqrt(normA) * math.Sqrt(normB))
|
||||
}
|
||||
|
||||
// SoftDeleteBatch 批量标记 is_deleted=true,updated_at=ts(单事务提交)
|
||||
func (mlc *MemLanceClient) SoftDeleteBatch(table string, ids []string, updatedAt string) (int64, error) {
|
||||
if table != "memories" {
|
||||
return 0, fmt.Errorf("unsupported table: %s", table)
|
||||
}
|
||||
mlc.mu.Lock()
|
||||
defer mlc.mu.Unlock()
|
||||
var updated int64
|
||||
for _, id := range ids {
|
||||
if e, ok := mlc.memories[id]; ok {
|
||||
if !e.IsDeleted {
|
||||
e.IsDeleted = true
|
||||
e.UpdatedAt = updatedAt
|
||||
updated++
|
||||
}
|
||||
}
|
||||
}
|
||||
return updated, nil
|
||||
}
|
||||
@@ -70,6 +70,10 @@ func (f *fakeLDB) UpdateRecallBatch(table string, items []RecallWriteItem, lastR
|
||||
return int64(len(items)), nil
|
||||
}
|
||||
|
||||
func (f *fakeLDB) SoftDeleteBatch(table string, ids []string, updatedAt string) (int64, error) {
|
||||
return int64(len(ids)), nil
|
||||
}
|
||||
|
||||
// AC-2:同一记忆在一个窗口内多次召回 → 只写 1 次(delta 合并),且是单事务批量调用
|
||||
func TestRecallWriteBufferCoalescesWithinWindow(t *testing.T) {
|
||||
fake := &fakeLDB{}
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
// 织忆 MemoryWeave — 软删除累积延迟更新缓冲(2026-09-12 优化C)─────────────────────────
|
||||
//
|
||||
// 问题(根因):软删除操作(SoftDelete)原先对**每条记忆**同步调用 lancedb.Update
|
||||
// (is_deleted = true + updated_at)。每次 update 提交产生一个版本。
|
||||
// 实测生产:碎片快速道和 decay 触发产生大量 SoftDelete,导致 _versions 膨胀。
|
||||
//
|
||||
// 方案:
|
||||
//
|
||||
// Record() 只把增量写进内存缓冲;**同一批次在一个窗口内多次 SoftDelete 合并为 一次批量提交**
|
||||
// Flush() 后台按窗口(默认 300s)+ 阈值(默认 256 条不同记忆)批量提交一次;
|
||||
// Rust 侧 lancedb_soft_delete_batch 把整批**设置相同字段**(is_deleted=true + updated_at=ts)一次提交;
|
||||
// → 每批版本数 = 1(旧实现:每行 1 个版本)。
|
||||
//
|
||||
// 语义取舍(明确记录,便于日后审计):
|
||||
// - updated_at 最多延迟一个窗口(分钟级)。遗忘/衰减判定以「天」为单位,无影响。
|
||||
// - 进程崩溃会丢最后一个窗口的增量(软删除标记,不是记忆数据本身),可接受。
|
||||
package storage
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
// ─── 软删除批量更新接口 ────────────────────────────────────
|
||||
//
|
||||
// 由 Rust IPC 后端(RustLanceDBClient)实现;其他后端不支持时自动退化为逐条 Update。
|
||||
type SoftDeleteBatcher interface {
|
||||
SoftDeleteBatch(table string, ids []string, updatedAt string) (int64, error)
|
||||
}
|
||||
|
||||
const (
|
||||
defaultSoftDeleteFlushInterval = 300 * time.Second
|
||||
defaultSoftDeleteFlushMaxItems = 256
|
||||
)
|
||||
|
||||
// pendingSoftDelete 表示待批量提交的软删除记录。
|
||||
type pendingSoftDelete struct {
|
||||
updatedAt time.Time
|
||||
}
|
||||
|
||||
// SoftDeleteWriteBuffer 累积软删除请求,延迟批量落盘。
|
||||
type SoftDeleteWriteBuffer struct {
|
||||
mu sync.Mutex
|
||||
pending map[string]*pendingSoftDelete
|
||||
ldb LanceDB
|
||||
batcher SoftDeleteBatcher
|
||||
interval time.Duration
|
||||
maxItems int
|
||||
|
||||
stopCh chan struct{}
|
||||
doneCh chan struct{}
|
||||
stopped bool
|
||||
startOnce sync.Once
|
||||
|
||||
flushCount int64
|
||||
flushItems int64
|
||||
flushRows int64
|
||||
flushErrors int64
|
||||
fallbackRows int64
|
||||
}
|
||||
|
||||
// NewSoftDeleteWriteBuffer 创建缓冲并启动后台 flush 协程;interval<=0 表示只按阈值触发。
|
||||
func NewSoftDeleteWriteBuffer(ldb LanceDB, interval time.Duration, maxItems int) *SoftDeleteWriteBuffer {
|
||||
if maxItems <= 0 {
|
||||
maxItems = defaultSoftDeleteFlushMaxItems
|
||||
}
|
||||
b := &SoftDeleteWriteBuffer{
|
||||
pending: make(map[string]*pendingSoftDelete),
|
||||
ldb: ldb,
|
||||
interval: interval,
|
||||
maxItems: maxItems,
|
||||
stopCh: make(chan struct{}),
|
||||
doneCh: make(chan struct{}),
|
||||
}
|
||||
// Rust IPC 后端支持单事务批量更新(SET is_deleted=true, updated_at=ts 一次提交)
|
||||
if sb, ok := ldb.(SoftDeleteBatcher); ok {
|
||||
b.batcher = sb
|
||||
}
|
||||
go b.loop()
|
||||
return b
|
||||
}
|
||||
|
||||
// InitSoftDeleteWriteBuffer 初始化进程级单例(server 启动时调用一次)。
|
||||
func InitSoftDeleteWriteBuffer(ldb LanceDB) *SoftDeleteWriteBuffer {
|
||||
if SoftDeleteWriteBufferInstance != nil {
|
||||
return SoftDeleteWriteBufferInstance
|
||||
}
|
||||
interval := defaultSoftDeleteFlushInterval
|
||||
if v := os.Getenv("SOFT_DELETE_FLUSH_SECONDS"); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil && n >= 0 {
|
||||
interval = time.Duration(n) * time.Second
|
||||
}
|
||||
}
|
||||
b := NewSoftDeleteWriteBuffer(ldb, interval, defaultSoftDeleteFlushMaxItems)
|
||||
SoftDeleteWriteBufferInstance = b
|
||||
log.Printf("[soft-delete-buffer] 已启用累积延迟更新: flush 窗口=%s, 阈值=%d 条, 单事务批量=%v",
|
||||
interval, b.maxItems, b.batcher != nil)
|
||||
return b
|
||||
}
|
||||
|
||||
// StopSoftDeleteWriteBuffer 停止单例并落盘最后一个窗口(进程优雅退出时调用)。
|
||||
func StopSoftDeleteWriteBuffer() {
|
||||
if b := SoftDeleteWriteBufferInstance; b != nil {
|
||||
b.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
func (b *SoftDeleteWriteBuffer) loop() {
|
||||
defer close(b.doneCh)
|
||||
if b.interval <= 0 {
|
||||
<-b.stopCh
|
||||
return
|
||||
}
|
||||
t := time.NewTicker(b.interval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-b.stopCh:
|
||||
return
|
||||
case <-t.C:
|
||||
b.Flush()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Record 累积一次软删除请求(同窗口内同 ID 合并为一条)。
|
||||
// 返回当前待写条数;达到阈值时异步触发一次 flush(不阻塞调用路径)。
|
||||
func (b *SoftDeleteWriteBuffer) Record(id string) int {
|
||||
now := time.Now()
|
||||
b.mu.Lock()
|
||||
if _, ok := b.pending[id]; !ok {
|
||||
b.pending[id] = &pendingSoftDelete{updatedAt: now}
|
||||
}
|
||||
n := len(b.pending)
|
||||
b.mu.Unlock()
|
||||
|
||||
if n >= b.maxItems {
|
||||
go b.Flush()
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// Flush 取出当前窗口的全部请求并批量提交(每批最多 1 个 LanceDB 版本)。
|
||||
func (b *SoftDeleteWriteBuffer) Flush() (int, error) {
|
||||
b.mu.Lock()
|
||||
if len(b.pending) == 0 {
|
||||
b.mu.Unlock()
|
||||
return 0, nil
|
||||
}
|
||||
ids := make([]string, 0, len(b.pending))
|
||||
var updatedAt time.Time
|
||||
for id, p := range b.pending {
|
||||
ids = append(ids, id)
|
||||
if p.updatedAt.After(updatedAt) {
|
||||
updatedAt = p.updatedAt
|
||||
}
|
||||
}
|
||||
b.pending = make(map[string]*pendingSoftDelete)
|
||||
b.mu.Unlock()
|
||||
|
||||
if len(ids) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
sort.Strings(ids)
|
||||
|
||||
ts := updatedAt.Format(time.RFC3339)
|
||||
var rows int64
|
||||
var err error
|
||||
usedFallback := false
|
||||
if b.batcher != nil {
|
||||
rows, err = b.batcher.SoftDeleteBatch("memories", ids, ts)
|
||||
}
|
||||
if b.batcher == nil || err != nil {
|
||||
usedFallback = true
|
||||
if err != nil {
|
||||
log.Printf("[soft-delete-buffer] 单事务批量提交失败(%v) → 退化逐条 Update(数据不丢,版本数不优化)", err)
|
||||
atomic.AddInt64(&b.flushErrors, 1)
|
||||
}
|
||||
rows = 0
|
||||
for _, id := range ids {
|
||||
if uerr := b.ldb.Update("memories", id, map[string]any{
|
||||
"is_deleted": true,
|
||||
"updated_at": ts,
|
||||
}); uerr != nil {
|
||||
log.Printf("[soft-delete-buffer] 逐条 Update 失败 id=%s: %v", id, uerr)
|
||||
continue
|
||||
}
|
||||
rows++
|
||||
}
|
||||
atomic.AddInt64(&b.fallbackRows, rows)
|
||||
}
|
||||
|
||||
if rows > 0 {
|
||||
// 软删除仅标记 is_deleted 和 updated_at,无需同步进程内缓存(缓存只是热数据,权威值在 LanceDB)
|
||||
}
|
||||
|
||||
atomic.AddInt64(&b.flushCount, 1)
|
||||
atomic.AddInt64(&b.flushItems, int64(len(ids)))
|
||||
atomic.AddInt64(&b.flushRows, rows)
|
||||
log.Printf("[soft-delete-buffer] flush: 合并 %d 条记忆 → 落盘 %d 行, 单事务=%v, fallback=%v",
|
||||
len(ids), rows, !usedFallback, usedFallback)
|
||||
return int(rows), nil
|
||||
}
|
||||
|
||||
// Stop 停止后台 flush 协程,并把最后一个窗口落盘(进程优雅退出时调用)。
|
||||
func (b *SoftDeleteWriteBuffer) Stop() {
|
||||
b.mu.Lock()
|
||||
if b.stopped {
|
||||
b.mu.Unlock()
|
||||
return
|
||||
}
|
||||
b.stopped = true
|
||||
b.mu.Unlock()
|
||||
|
||||
close(b.stopCh)
|
||||
select {
|
||||
case <-b.doneCh:
|
||||
case <-time.After(10 * time.Second):
|
||||
log.Printf("[soft-delete-buffer] 停止超时,直接落盘剩余窗口")
|
||||
}
|
||||
b.Flush()
|
||||
}
|
||||
|
||||
// Stats 观测用(flush 频率 / 合并率 / 版本写入行数)。
|
||||
func (b *SoftDeleteWriteBuffer) Stats() map[string]interface{} {
|
||||
b.mu.Lock()
|
||||
pendingN := len(b.pending)
|
||||
b.mu.Unlock()
|
||||
return map[string]interface{}{
|
||||
"pending": pendingN,
|
||||
"flush_count": atomic.LoadInt64(&b.flushCount),
|
||||
"flush_items": atomic.LoadInt64(&b.flushItems),
|
||||
"flush_rows": atomic.LoadInt64(&b.flushRows),
|
||||
"flush_errors": atomic.LoadInt64(&b.flushErrors),
|
||||
"fallback_rows": atomic.LoadInt64(&b.fallbackRows),
|
||||
"window": b.interval.String(),
|
||||
"max_items": b.maxItems,
|
||||
"single_tx": b.batcher != nil,
|
||||
}
|
||||
}
|
||||
|
||||
// SoftDeleteWriteBufferInstance 进程级单例(与 RecallWriteBufferInstance / SearchCacheInstance / CoOccurTrackerInstance 同模式)
|
||||
var SoftDeleteWriteBufferInstance *SoftDeleteWriteBuffer
|
||||
@@ -442,6 +442,26 @@ func (sc *SQLiteClient) SoftDelete(id, reason string) error {
|
||||
)
|
||||
}
|
||||
|
||||
func (sc *SQLiteClient) SoftDeleteBatch(table string, ids []string, updatedAt string) (int64, error) {
|
||||
if table != "memories" {
|
||||
return 0, fmt.Errorf("unsupported table: %s", table)
|
||||
}
|
||||
sc.mu.Lock()
|
||||
defer sc.mu.Unlock()
|
||||
var updated int64
|
||||
for _, id := range ids {
|
||||
res := sc.execWithArgs(
|
||||
"UPDATE memories SET is_deleted=1, updated_at=? WHERE id=?",
|
||||
updatedAt, id,
|
||||
)
|
||||
if res != nil {
|
||||
return updated, res
|
||||
}
|
||||
updated++
|
||||
}
|
||||
return updated, nil
|
||||
}
|
||||
|
||||
func (sc *SQLiteClient) GetVersionHistory(id string) ([]map[string]interface{}, error) {
|
||||
sc.mu.RLock()
|
||||
defer sc.mu.RUnlock()
|
||||
|
||||
@@ -606,4 +606,87 @@ impl LanceDBOps {
|
||||
eprintln!("[lancedb] scan_all → {} records", records.len());
|
||||
Ok(records)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct LanceDBStats {
|
||||
pub total_memories: usize,
|
||||
pub total_episodes: usize,
|
||||
pub tombstone_count: usize,
|
||||
pub data_dir: String,
|
||||
}
|
||||
|
||||
impl LanceDBOps {
|
||||
pub fn stats(&self) -> Result<LanceDBStats, Box<dyn std::error::Error>> {
|
||||
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
|
||||
let tables = rt().block_on(db.table_names().execute())?;
|
||||
let mut m = 0usize; let mut e = 0usize; let mut t = 0usize;
|
||||
for name in &tables {
|
||||
if let Ok(tbl) = rt().block_on(db.open_table(name).execute()) {
|
||||
if name == "memories" {
|
||||
if let Ok(cnt) = rt().block_on(tbl.count_rows(None)) { m = cnt; }
|
||||
if let Ok(tc) = rt().block_on(tbl.count_rows(Some("is_deleted = true".to_string()))) { t = tc; }
|
||||
} else if let Ok(cnt) = rt().block_on(tbl.count_rows(None)) {
|
||||
match name.as_str() {
|
||||
"episodes" => e = cnt,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(LanceDBStats {
|
||||
total_memories: m, total_episodes: e, tombstone_count: t,
|
||||
data_dir: self.data_dir.to_string_lossy().to_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ── Arrow RecordBatch helpers ──
|
||||
fn col_str(b: &RecordBatch, r: usize, c: &str) -> String {
|
||||
b.column_by_name(c)
|
||||
.and_then(|col| col.as_any().downcast_ref::<StringArray>())
|
||||
.map(|a| a.value(r).to_string())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
fn col_f64(b: &RecordBatch, r: usize, c: &str) -> f64 {
|
||||
b.column_by_name(c)
|
||||
.and_then(|col| col.as_any().downcast_ref::<Float64Array>())
|
||||
.map(|a| a.value(r))
|
||||
.unwrap_or(0.0)
|
||||
}
|
||||
fn col_i64(b: &RecordBatch, r: usize, c: &str) -> i64 {
|
||||
b.column_by_name(c)
|
||||
.and_then(|col| col.as_any().downcast_ref::<Int64Array>())
|
||||
.map(|a| a.value(r))
|
||||
.unwrap_or(0)
|
||||
}
|
||||
fn col_bool(b: &RecordBatch, r: usize, c: &str) -> bool {
|
||||
b.column_by_name(c)
|
||||
.and_then(|col| col.as_any().downcast_ref::<BooleanArray>())
|
||||
.map(|a| a.value(r))
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// 读取 vector 列 (兼容 FixedSizeListArray 和 Float32Array)
|
||||
fn col_vector(b: &RecordBatch, r: usize, c: &str) -> Vec<f32> {
|
||||
let col = match b.column_by_name(c) {
|
||||
Some(col) => col,
|
||||
None => return vec![0.0_f32; 1024],
|
||||
};
|
||||
// 优先尝试 FixedSizeListArray (当前使用的格式)
|
||||
if let Some(list_arr) = col.as_any().downcast_ref::<FixedSizeListArray>() {
|
||||
let item_arr = list_arr.value(r);
|
||||
if let Some(float_arr) = item_arr.as_any().downcast_ref::<Float32Array>() {
|
||||
return float_arr.values().to_vec();
|
||||
}
|
||||
}
|
||||
// 回退: Float32Array (展平格式)
|
||||
if let Some(float_arr) = col.as_any().downcast_ref::<Float32Array>() {
|
||||
let stride = 1024;
|
||||
let start = r * stride;
|
||||
if start + stride <= float_arr.len() {
|
||||
return float_arr.values()[start..start + stride].to_vec();
|
||||
}
|
||||
}
|
||||
vec![0.0_f32; 1024]
|
||||
}
|
||||
Reference in New Issue
Block a user