Files
memoryweave/rust/src/lancedb_ops.rs
T
小唯 d0558e1fd5 fix(zhiyi): t_763e8701 返修 bi-temporal 三列未落 LanceDB — 补齐 Go payload+读回 & Rust schema+builder+读侧 5 层
根因:t_3ada332b 只改了 models 结构体 + server.go 算值,LanceDB 边界三层缺失
(Go lancedb_ipc.go 写 payload + 读回解包、Rust arrow_schema + struct + builder + col_str)
⇒ valid_to 永不持久化 ⇒ validityExpired("") 恒 false ⇒ 失效降权生产 no-op。

本次(照 t_3e98b65a 给 superseded_by 的 5 层模式逐一对齐):
1. Go lancedb_ipc.go 写侧 payload 补 valid_from/valid_to/provenance 三键
2. Go 读侧补三处:Search 解包struct+输出映射 / Scan 解包struct+输出映射 / GetSkillCandidates mapStr 路径
3. Rust lancedb_ops.rs struct 字段(#[serde(default)],存量行不报错)
4. Rust arrow_schema Field::new 三列(nullable,存量由 open() 迁移补列)
5. Rust 写 builder col vec + push + StringArray + 5 处读侧 col_str

新增 e2e 回归闸门 bitemporal_e2e_test.go:经真实 LanceDB sidecar 写一条 valid_to 已过期记忆
→ 读回断言三列非空且等于写入值(ZHIYI_BITEMPORAL_SOCK 门控,未设跳过)。
单测(AC1/AC2/AC3/AC4)+e2e 全 PASS;全量 go test 仅既有 flaky TestCommitConflictGate_E2E(clean baseline 同败)。
2026-10-14 05:01:27 +08:00

1545 lines
81 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
use std::sync::OnceLock;
fn rt() -> &'static tokio::runtime::Runtime {
static RT: OnceLock<tokio::runtime::Runtime> = OnceLock::new();
RT.get_or_init(|| tokio::runtime::Runtime::new().unwrap())
}
// LanceDB 原生读写 — lancedb crate 0.15
// 使用 pollster::block_on 处理 async API
// API: connect(path) → create_empty_table/table_names → vector_search → add(arrow_data)
use std::path::PathBuf;
use std::sync::Arc;
use serde::{Deserialize, Serialize};
use arrow::array::{Array, ArrayRef, BooleanArray, FixedSizeListArray, Float32Array, Float64Array, Int64Array, StringArray};
use arrow::datatypes::{DataType, Field, Schema};
use arrow::record_batch::{RecordBatch, RecordBatchIterator};
use futures::StreamExt;
use lancedb::query::{ExecutableQuery, QueryBase, Select};
use lancedb::table::NewColumnTransform;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MemoryRecord {
pub id: String, pub agent_id: String, pub namespace: String,
pub content: String, pub category: String, pub vector: Vec<f32>,
pub tier: String, pub importance: f64, pub quality_score: f64,
pub recall_count: i64, pub useful_count: i64, pub not_useful_count: i64,
// GA-P0-2 (t_c6d41ac6):拆分 recall_count 的两种语义(「出现在结果里」/「被验证使用」)。
// recall_count 曾把两种语义混写 —— 每次召回命中都 ++,被当成 V 值喂给衰减/遗忘逻辑,
// 造成「被召回 500+ 次所以永不遗忘」的虚高保命。现拆分为:
// recall_seen_count 出现即 ++(只用于观测/排序去偏,不参与 V 值/衰减权重)
// recall_verified_count有正反馈(IncrementUseful)或下游引用才 ++(唯一参与 V 值/衰减)
// recall_count 保留(deprecated),冻结不再增长,仅向后兼容读取历史值。
pub recall_seen_count: i64, pub recall_verified_count: i64,
// GA-P0-1 (t_718d8707):拆分 not_useful_count 的两种语义(人工「没用」/ dedup 自动降权)。
// not_useful_count 保留(deprecated),新代码不再读它;human_not_useful_count 与
// dedup_downweight_count 是新写入点。arrow_schema 尾部新增列(nullable,存量补 NULL)。
pub human_not_useful_count: i64, pub dedup_downweight_count: i64,
pub freshness: String, pub version: i64,
pub version_history: String, pub source: String,
// 2026-10-08 (t_7e3577c5):血缘字段
pub embedding_model: String, pub script_version: String,
// 2026-10-08 (Gap#2):信任等级。Option<f64> —— 存量行是 NULL,必须与「显式 0.0」区分。
// 漏了这里 ⇒ IPC 响应无该字段 ⇒ Go 侧解包恒 nil ⇒ 加列也白加(静默 no-op)。
#[serde(default)]
pub confidence: Option<f64>,
#[serde(default)]
pub confidence_source: String,
pub volatile_flag: bool, pub is_deleted: bool,
// GA-P0-3 (t_f785d7f2):拆分 is_deleted 的删除原因。is_deleted=true 不再「不可分辨」——
// 坏记忆被删(contradiction/manual/retention) 与 合并/整理被删(merge_dup) 现可分列。
// 固定枚举: merge_dup | contradiction | manual | retention | unknown。additive 新列,
// 存量行是 NULL(读路径 col_str 兜底 ""),回填脚本以 tombstones 为据尽力判定。
pub delete_reason: String,
pub depends_on: String, pub derived_from: String,
// 2026-10-13 (t_e9a4d18f P1) 证据链:全量来源 episode ID 的 JSON 数组字符串。
// 与 depends_on 同款(JSON 串列),additive 新列,存量由 schema 演进补 NULL。
pub source_events: String,
// 2026-10-13 (t_e9a4d18f P1) 矛盾保留:被取代的旧记忆 ID(JSON 数组字符串)
pub supersedes: String,
// 2026-10-14 (t_3e98b65a) 被取代方标记:反向指针(JSON 数组字符串,additive 新列)
#[serde(default)]
pub superseded_by: String,
// 2026-10-14 (t_3ada332b P3) bi-temporal 有效期三列:RFC3339;空 = 永久有效。
// #[serde(default)] ⇒ 存量行/缺列读回 "",降权永不触发 ⇒ 必须 5 层齐,缺 Rust 这层即 no-op。
#[serde(default)]
pub valid_from: String,
#[serde(default)]
pub valid_to: String,
#[serde(default)]
pub provenance: String,
pub last_recalled_at: String, pub created_at: String, pub updated_at: String,
}
#[derive(Clone)]
pub struct LanceDBOps {
data_dir: PathBuf,
}
/// 遗忘候选「旋转窗口」返回(方案 c′ 2026-10-11 t_a4b8fa52)。
/// - records: 本窗口全字段记录(已按 id 排序)
/// - next_cursor: 新游标 = 窗口末 id;wrapped=true 时为空串(下轮从 0 起)
/// - wrapped: 窗口不足 limit(取到存活集合末尾)⇒ 本轮遍历结束
/// - live_total: 存活 mem_ id 总数(排除 episode),供覆盖度量与 AC-7 吞吐判据
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ForgetWindow {
pub records: Vec<MemoryRecord>,
pub next_cursor: String,
pub wrapped: bool,
pub live_total: usize,
}
fn arrow_schema() -> Schema {
Schema::new(vec![
Field::new("id", DataType::Utf8, false),
Field::new("agent_id", DataType::Utf8, false),
Field::new("namespace", DataType::Utf8, false),
Field::new("content", DataType::Utf8, false),
Field::new("category", DataType::Utf8, false),
Field::new("vector", DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 1024), true),
Field::new("tier", DataType::Utf8, false),
Field::new("importance", DataType::Float64, false),
Field::new("quality_score", DataType::Float64, false),
Field::new("recall_count", DataType::Int64, false),
Field::new("useful_count", DataType::Int64, false),
Field::new("not_useful_count", DataType::Int64, false),
Field::new("freshness", DataType::Utf8, false),
Field::new("version", DataType::Int64, false),
Field::new("version_history", DataType::Utf8, true),
Field::new("source", DataType::Utf8, true),
Field::new("embedding_model", DataType::Utf8, true),
Field::new("script_version", DataType::Utf8, true),
Field::new("volatile_flag", DataType::Boolean, false),
Field::new("is_deleted", DataType::Boolean, false),
Field::new("depends_on", DataType::Utf8, true),
Field::new("derived_from", DataType::Utf8, true),
Field::new("last_recalled_at", DataType::Utf8, true),
Field::new("created_at", DataType::Utf8, false),
Field::new("updated_at", DataType::Utf8, false),
// 2026-10-08 (Gap#2):来源加权。confidence=写入方显式声明的信任等级(不从 source 反推);
// confidence_source=authoritative|estimated,用于区分「写入时声明」与「事后回填猜测」,
// 是日后批量纠偏的唯一手段(否则无法定向打 estimated 行)。
// 均 nullable:存量行保持 NULL,读路径 COALESCE→1.0,使加权上线首日即 no-op。
Field::new("confidence", DataType::Float64, true),
Field::new("confidence_source", DataType::Utf8, true),
// GA-P0-1 (t_718d8707):拆分 not_useful_count 的两种语义。additive 新列;
// 存量表由 open() 的 schema 演进迁移自动补列(nullable 容纳存量 NULL,
// 与 t_3df364b8 血缘列 / Gap#2 confidence 的迁移模式一致)。
Field::new("human_not_useful_count", DataType::Int64, true),
Field::new("dedup_downweight_count", DataType::Int64, true),
// GA-P0-2 (t_c6d41ac6):拆分 recall_count 的两种语义。additive 新列;
// 存量表由 open() 的 schema 演进自动补列(nullable 容纳存量 NULL)。
Field::new("recall_seen_count", DataType::Int64, true),
Field::new("recall_verified_count", DataType::Int64, true),
// GA-P0-3 (t_f785d7f2):is_deleted 的删除原因(固定枚举)。additive 新列,
// nullable 容纳存量 NULL;回填脚本以 tombstones 为据尽力判定,判不出写 unknown。
Field::new("delete_reason", DataType::Utf8, true),
// 2026-10-13 (t_e9a4d18f P1) 证据链:全量来源 episode ID(JSON 数组字符串)。
// additive 新列,存量由 schema 演进补 NULL(与 depends_on 同款)。
Field::new("source_events", DataType::Utf8, true),
// 2026-10-13 (t_e9a4d18f P1) 矛盾保留(additive)
Field::new("supersedes", DataType::Utf8, true),
// 2026-10-14 (t_3e98b65a) 被取代方标记(additive,反向指针)
Field::new("superseded_by", DataType::Utf8, true),
// 2026-10-14 (t_3ada332b P3) bi-temporal 有效期三列(additive,nullable 容纳存量 NULL)
Field::new("valid_from", DataType::Utf8, true),
Field::new("valid_to", DataType::Utf8, true),
Field::new("provenance", DataType::Utf8, true),
])
}
// GA-P0-4 (t_bb10e8bc):hits+/hits− outcome 事件记录(append-only 事件表)。
// 与 MemoryRecord 计数列不同:这是「按时间戳追加」的事件,**不覆盖式累加**,
// 便于 P2 自适应阈值按周窗口聚合。语义定义见 Go 侧 internal/storage/hit_event.go。
// 只由真实 outcome 写入(human_feedback|contradiction|replacement|downstream_use);
// dedup 降权 / 单纯召回出现禁止写入。
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct HitEventRecord {
pub id: String,
pub memory_id: String,
pub outcome: String, // pos | neg
pub reason: String, // human_feedback|contradiction|replacement|downstream_use
pub source: String,
pub created_at: String, // RFC3339
}
fn hit_event_arrow_schema() -> Schema {
Schema::new(vec![
Field::new("id", DataType::Utf8, false),
Field::new("memory_id", DataType::Utf8, false),
Field::new("outcome", DataType::Utf8, false),
Field::new("reason", DataType::Utf8, false),
Field::new("source", DataType::Utf8, true),
Field::new("created_at", DataType::Utf8, false),
])
}
// 2026-10-13 (t_e9a4d18f P2) 常设答案表(mental models / knowledge pages)。
// question_id 主键;answer 为生成的常设答案;source_memories 为可追溯的证据记忆
// ID 列表(JSON 数组字符串);version 每次改写 +1。读取 = 一次主键查询,无检索无 LLM。
fn standing_answer_arrow_schema() -> Schema {
Schema::new(vec![
Field::new("question_id", DataType::Utf8, false),
Field::new("question", DataType::Utf8, false),
Field::new("answer", DataType::Utf8, false),
Field::new("source_memories", DataType::Utf8, true),
Field::new("version", DataType::Int64, false),
Field::new("created_at", DataType::Utf8, false),
Field::new("updated_at", DataType::Utf8, false),
])
}
/// 生成 hit_event 行的唯一 id(memory_id + 微秒时间戳;同微秒追加几乎不可能)。
fn hit_event_id(memory_id: &str) -> String {
let micros = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_micros())
.unwrap_or(0);
format!("{}-{}", memory_id, micros)
}
/// SQL 单引号转义(用于 hit_events 过滤条件的字符串字面量)。
fn sql_escape(s: &str) -> String {
s.replace('\'', "''")
}
impl LanceDBOps {
pub fn open(data_dir: &str) -> Result<Self, Box<dyn std::error::Error>> {
let path = PathBuf::from(data_dir);
std::fs::create_dir_all(&path)?;
let db = rt().block_on(lancedb::connect(path.to_str().unwrap()).execute())?;
let tables = rt().block_on(db.table_names().execute())?;
let target = arrow_schema();
for name in ["memories", "episodes", "tombstones"] {
if !tables.iter().any(|t| t == name) {
rt().block_on(db.create_empty_table(name, Arc::new(arrow_schema())).execute()).ok();
eprintln!("[lancedb] created table: {}", name);
continue;
}
// 已存在表 → schema 演进迁移:补齐 arrow_schema() 中有而实际表缺失的列
// (2026-10-08 t_3df364b8:commit 5038d19 加 2 血缘列后 episodes 仍 23 列,
// 每次 Append 25 列 ⇒ schema 不匹配 ⇒ 静默丢数据。这里统一补列,三表共用一套 schema。)
let tbl = rt().block_on(db.open_table(name).execute())?;
let tbl_schema = rt().block_on(tbl.schema())?;
let mut missing: Vec<(String, String)> = Vec::new();
for f in target.fields() {
if tbl_schema.field_with_name(f.name()).is_err() {
// 缺失列一律补**可空**(2026-10-08 修正:原先字符串给 '',
// 而 add_columns 的 SQL 字面量默认值会把列标成 **non-null** ——
// 与 arrow_schema 的 nullable=true 冲突,写入 NULL 会被 lance 拒绝:
// "contained null values even though the field is marked non-null"
// 实测对照:embedding_model/script_version(历史迁移列)都是 nullable=true。
let default = match f.data_type() {
DataType::Float64 => "CAST(NULL AS DOUBLE)",
DataType::Float32 => "CAST(NULL AS FLOAT)",
DataType::Int64 => "CAST(NULL AS BIGINT)",
DataType::Boolean => "CAST(NULL AS BOOLEAN)",
_ => "CAST(NULL AS STRING)",
};
missing.push((f.name().to_string(), default.to_string()));
}
}
if !missing.is_empty() {
let cols: Vec<String> = missing.iter().map(|(c, _)| c.clone()).collect();
eprintln!("[lancedb] migrating {}: adding missing cols {:?}", name, cols);
rt().block_on(tbl.add_columns(
NewColumnTransform::SqlExpressions(missing),
None,
))?;
eprintln!("[lancedb] migrated {}: added {:?}", name, cols);
}
}
// GA-P0-4 (t_bb10e8bc):hits+/hits− outcome 事件表(append-only,独立 schema)。
if !tables.iter().any(|t| t == "hit_events") {
rt().block_on(db.create_empty_table("hit_events", Arc::new(hit_event_arrow_schema())).execute()).ok();
eprintln!("[lancedb] created table: hit_events");
}
// 2026-10-13 (t_e9a4d18f P2):常设答案表(mental models)。独立 schema。
if !tables.iter().any(|t| t == "standing_answers") {
rt().block_on(db.create_empty_table("standing_answers", Arc::new(standing_answer_arrow_schema())).execute()).ok();
eprintln!("[lancedb] created table: standing_answers");
}
eprintln!("[lancedb] opened at {:?} ({} tables)", path, tables.len());
Ok(Self { data_dir: path })
}
pub fn init_tables(&self) -> Result<(), Box<dyn std::error::Error>> {
Ok(())
}
pub fn search(&self, query_vec: &[f32], top_k: usize, namespace: Option<&str>) -> Result<Vec<MemoryRecord>, Box<dyn std::error::Error>> {
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
let tbl = rt().block_on(db.open_table("memories").execute())?;
let mut q = tbl.vector_search(query_vec.to_vec())?.limit(top_k);
if let Some(ns) = namespace {
q = q.only_if(&format!("namespace = '{}'", ns));
}
q = q.only_if("is_deleted = false");
let mut results = Box::pin(rt().block_on(q.execute())?);
let mut records = Vec::new();
while let Some(Ok(batch)) = rt().block_on(results.next()) {
for i in 0..batch.num_rows() {
records.push(MemoryRecord {
id: col_str(&batch, i, "id"),
agent_id: col_str(&batch, i, "agent_id"),
namespace: col_str(&batch, i, "namespace"),
content: col_str(&batch, i, "content"),
category: col_str(&batch, i, "category"),
vector: col_vector(&batch, i, "vector"),
tier: col_str(&batch, i, "tier"),
importance: col_f64(&batch, i, "importance"),
quality_score: col_f64(&batch, i, "quality_score"),
recall_count: col_i64(&batch, i, "recall_count"),
// GA-P0-2 (t_c6d41ac6):拆分字段读侧接线
recall_seen_count: col_i64(&batch, i, "recall_seen_count"),
recall_verified_count: col_i64(&batch, i, "recall_verified_count"),
useful_count: col_i64(&batch, i, "useful_count"),
not_useful_count: col_i64(&batch, i, "not_useful_count"),
// GA-P0-1 (t_718d8707):拆分字段读侧接线
human_not_useful_count: col_i64(&batch, i, "human_not_useful_count"),
dedup_downweight_count: col_i64(&batch, i, "dedup_downweight_count"),
freshness: col_str(&batch, i, "freshness"),
version: col_i64(&batch, i, "version"),
version_history: String::new(),
source: col_str(&batch, i, "source"),
embedding_model: col_str(&batch, i, "embedding_model"),
script_version: col_str(&batch, i, "script_version"),
volatile_flag: col_bool(&batch, i, "volatile_flag"),
is_deleted: col_bool(&batch, i, "is_deleted"),
delete_reason: col_str(&batch, i, "delete_reason"),
depends_on: String::new(),
derived_from: String::new(),
source_events: col_str(&batch, i, "source_events"), // 2026-10-13 P1
supersedes: col_str(&batch, i, "supersedes"), // 2026-10-13 P1
superseded_by: col_str(&batch, i, "superseded_by"), // 2026-10-14 t_3e98b65a
valid_from: col_str(&batch, i, "valid_from"), // 2026-10-14 t_3ada332b P3
valid_to: col_str(&batch, i, "valid_to"), // 2026-10-14 t_3ada332b P3
provenance: col_str(&batch, i, "provenance"), // 2026-10-14 t_3ada332b P3
last_recalled_at: String::new(),
created_at: col_str(&batch, i, "created_at"),
updated_at: col_str(&batch, i, "updated_at"),
// 2026-10-08 (Gap#2):读侧接线。col_f64_opt 保留 NULL 语义
confidence: col_f64_opt(&batch, i, "confidence"),
confidence_source: col_str(&batch, i, "confidence_source"),
});
}
}
eprintln!("[lancedb] search top_k={} → {} results", top_k, records.len());
Ok(records)
}
/// 查询记忆(支持 min_recall_count 过滤),供 GetSkillCandidates 使用
pub fn query_memories(&self, min_recall: i64, limit: usize) -> Result<Vec<MemoryRecord>, Box<dyn std::error::Error>> {
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
let tbl = rt().block_on(db.open_table("memories").execute())?;
// GA-P0-2 (t_c6d41ac6):技能候选 = 观测/选择(被召回多次)→ 用 recall_seen_count。
// recall_count 已冻结,不再增长,不能再当选择依据。
let filter = format!("recall_seen_count >= {} AND is_deleted = false AND tier != 'core'", min_recall);
let mut results = Box::pin(rt().block_on(
tbl.query()
.only_if(&filter)
.limit(limit)
.execute()
)?);
let mut records = Vec::new();
while let Some(Ok(batch)) = rt().block_on(results.next()) {
for i in 0..batch.num_rows() {
records.push(MemoryRecord {
id: col_str(&batch, i, "id"),
agent_id: col_str(&batch, i, "agent_id"),
namespace: col_str(&batch, i, "namespace"),
content: col_str(&batch, i, "content"),
category: col_str(&batch, i, "category"),
vector: col_vector(&batch, i, "vector"),
tier: col_str(&batch, i, "tier"),
importance: col_f64(&batch, i, "importance"),
quality_score: col_f64(&batch, i, "quality_score"),
recall_count: col_i64(&batch, i, "recall_count"),
// GA-P0-2 (t_c6d41ac6):拆分字段读侧接线
recall_seen_count: col_i64(&batch, i, "recall_seen_count"),
recall_verified_count: col_i64(&batch, i, "recall_verified_count"),
useful_count: col_i64(&batch, i, "useful_count"),
not_useful_count: col_i64(&batch, i, "not_useful_count"),
// GA-P0-1 (t_718d8707):拆分字段读侧接线
human_not_useful_count: col_i64(&batch, i, "human_not_useful_count"),
dedup_downweight_count: col_i64(&batch, i, "dedup_downweight_count"),
freshness: col_str(&batch, i, "freshness"),
version: col_i64(&batch, i, "version"),
version_history: String::new(),
source: col_str(&batch, i, "source"),
embedding_model: col_str(&batch, i, "embedding_model"),
script_version: col_str(&batch, i, "script_version"),
volatile_flag: col_bool(&batch, i, "volatile_flag"),
is_deleted: col_bool(&batch, i, "is_deleted"),
delete_reason: col_str(&batch, i, "delete_reason"),
depends_on: String::new(),
derived_from: String::new(),
source_events: col_str(&batch, i, "source_events"), // 2026-10-13 P1
supersedes: col_str(&batch, i, "supersedes"), // 2026-10-13 P1
superseded_by: col_str(&batch, i, "superseded_by"), // 2026-10-14 t_3e98b65a
valid_from: col_str(&batch, i, "valid_from"), // 2026-10-14 t_3ada332b P3
valid_to: col_str(&batch, i, "valid_to"), // 2026-10-14 t_3ada332b P3
provenance: col_str(&batch, i, "provenance"), // 2026-10-14 t_3ada332b P3
last_recalled_at: String::new(),
created_at: col_str(&batch, i, "created_at"),
updated_at: col_str(&batch, i, "updated_at"),
// 2026-10-08 (Gap#2):读侧接线。col_f64_opt 保留 NULL 语义
confidence: col_f64_opt(&batch, i, "confidence"),
confidence_source: col_str(&batch, i, "confidence_source"),
});
}
}
Ok(records)
}
pub fn insert_batch(&self, table: &str, records_json: &str) -> Result<usize, Box<dyn std::error::Error>> {
let values: serde_json::Value = serde_json::from_str(records_json)?;
let records: Vec<serde_json::Value> = values.as_array().ok_or("expected JSON array")?.to_vec();
let count = records.len();
if count == 0 {
return Ok(0);
}
// 构建 Arrow RecordBatch
let schema = Arc::new(arrow_schema());
let mut ids = Vec::new(); let mut agent_ids = Vec::new(); let mut namespaces = Vec::new();
let mut contents = Vec::new(); let mut categories = Vec::new(); let mut vectors = Vec::new();
let mut tiers = Vec::new(); let mut importances = Vec::new(); let mut quality_scores = Vec::new();
let mut recall_counts = Vec::new(); let mut recall_seen_counts = Vec::new(); let mut recall_verified_counts = Vec::new();
let mut useful_counts = Vec::new(); let mut not_useful_counts = Vec::new();
// GA-P0-1 (t_718d8707):拆分字段
let mut human_not_useful_counts = Vec::new(); let mut dedup_downweight_counts = Vec::new();
let mut freshnesses = Vec::new(); let mut versions = Vec::new();
let mut version_histories = Vec::new(); let mut sources = Vec::new();
let mut embedding_models = Vec::new(); let mut script_versions = Vec::new();
let mut volatile_flags = Vec::new(); let mut is_deleteds = Vec::new();
// GA-P0-3 (t_f785d7f2):删除原因(additive)
let mut delete_reasons: Vec<&str> = Vec::new();
let mut depends_ons = Vec::new(); let mut derived_froms = Vec::new();
// 2026-10-13 (t_e9a4d18f P1) 证据链:全量来源 episode ID(JSON 数组字符串)
let mut source_events_col = Vec::new();
let mut supersedes_col = Vec::new();
let mut superseded_by_col = Vec::new(); // 2026-10-14 t_3e98b65a
// 2026-10-14 (t_3ada332b P3) bi-temporal 有效期三列
let mut valid_froms = Vec::new(); let mut valid_tos = Vec::new(); let mut provenances = Vec::new();
let mut last_recalled_ats = Vec::new(); let mut created_ats = Vec::new(); let mut updated_ats = Vec::new();
// 2026-10-08 (Gap#2):nullable 列用 Option,未声明时写 None → 库里是 NULL
let mut confidences: Vec<Option<f64>> = Vec::new();
let mut confidence_sources: Vec<&str> = Vec::new();
for r in &records {
ids.push(r["id"].as_str().unwrap_or(""));
agent_ids.push(r["agent_id"].as_str().unwrap_or(""));
namespaces.push(r["namespace"].as_str().unwrap_or(""));
contents.push(r["content"].as_str().unwrap_or(""));
categories.push(r["category"].as_str().unwrap_or(""));
let v: Vec<f32> = if let Some(arr) = r["vector"].as_array() {
arr.iter().filter_map(|v| v.as_f64()).map(|v| v as f32).collect()
} else { vec![0.0_f32; 1024] };
vectors.push(v);
tiers.push(r["tier"].as_str().unwrap_or("normal"));
importances.push(r["importance"].as_f64().unwrap_or(1.0));
quality_scores.push(r["quality_score"].as_f64().unwrap_or(0.0));
recall_counts.push(r["recall_count"].as_i64().unwrap_or(0));
// GA-P0-2 (t_c6d41ac6):拆分字段写入
recall_seen_counts.push(r["recall_seen_count"].as_i64().unwrap_or(0));
recall_verified_counts.push(r["recall_verified_count"].as_i64().unwrap_or(0));
useful_counts.push(r["useful_count"].as_i64().unwrap_or(0));
not_useful_counts.push(r["not_useful_count"].as_i64().unwrap_or(0));
human_not_useful_counts.push(r["human_not_useful_count"].as_i64().unwrap_or(0)); // GA-P0-1
dedup_downweight_counts.push(r["dedup_downweight_count"].as_i64().unwrap_or(0)); // GA-P0-1
freshnesses.push(r["freshness"].as_str().unwrap_or("fresh"));
versions.push(r["version"].as_i64().unwrap_or(1));
version_histories.push(r["version_history"].as_str().unwrap_or("[]"));
sources.push(r["source"].as_str().unwrap_or(""));
embedding_models.push(r["embedding_model"].as_str().unwrap_or(""));
script_versions.push(r["script_version"].as_str().unwrap_or(""));
volatile_flags.push(r["volatile_flag"].as_bool().unwrap_or(false));
is_deleteds.push(r["is_deleted"].as_bool().unwrap_or(false));
delete_reasons.push(r["delete_reason"].as_str().unwrap_or("")); // GA-P0-3
depends_ons.push(r["depends_on"].as_str().unwrap_or("[]"));
derived_froms.push(r["derived_from"].as_str().unwrap_or(""));
source_events_col.push(r["source_events"].as_str().unwrap_or("[]")); // 2026-10-13 P1
supersedes_col.push(r["supersedes"].as_str().unwrap_or("[]")); // 2026-10-13 P1
superseded_by_col.push(r["superseded_by"].as_str().unwrap_or("[]")); // 2026-10-14 t_3e98b65a
valid_froms.push(r["valid_from"].as_str().unwrap_or("")); // 2026-10-14 t_3ada332b P3
valid_tos.push(r["valid_to"].as_str().unwrap_or("")); // 2026-10-14 t_3ada332b P3
provenances.push(r["provenance"].as_str().unwrap_or("")); // 2026-10-14 t_3ada332b P3
last_recalled_ats.push(r["last_recalled_at"].as_str().unwrap_or(""));
created_ats.push(r["created_at"].as_str().unwrap_or(""));
updated_ats.push(r["updated_at"].as_str().unwrap_or(""));
// confidence:显式声明才写,否则 NULL(不从 source 反推)——
// **NULL 本身就是「未声明」的语义信号**,confidence_source 只是标签。
confidences.push(r["confidence"].as_f64());
// confidence_source 恒给字符串(未声明 ⇒ ""):迁移来的列可能是 non-null,
// 送 None 会被 lance 拒收。语义仍由 confidence=NULL 承担。
confidence_sources.push(r["confidence_source"].as_str().unwrap_or(""));
}
// 构建 vector 列 (FixedSizeList<Float32, 1024>)
let flat: Vec<f32> = vectors.iter().flatten().copied().collect();
let values_array = Float32Array::from_iter_values(flat.iter().cloned());
let vec_array = FixedSizeListArray::try_new(
Arc::new(Field::new("item", DataType::Float32, true)),
1024,
Arc::new(values_array),
None,
)?;
let columns: Vec<ArrayRef> = vec![
Arc::new(StringArray::from(ids)),
Arc::new(StringArray::from(agent_ids)),
Arc::new(StringArray::from(namespaces)),
Arc::new(StringArray::from(contents)),
Arc::new(StringArray::from(categories)),
Arc::new(vec_array),
Arc::new(StringArray::from(tiers)),
Arc::new(Float64Array::from(importances)),
Arc::new(Float64Array::from(quality_scores)),
Arc::new(Int64Array::from(recall_counts)),
Arc::new(Int64Array::from(useful_counts)),
Arc::new(Int64Array::from(not_useful_counts)),
Arc::new(StringArray::from(freshnesses)),
Arc::new(Int64Array::from(versions)),
Arc::new(StringArray::from(version_histories)),
Arc::new(StringArray::from(sources)),
Arc::new(StringArray::from(embedding_models)),
Arc::new(StringArray::from(script_versions)),
Arc::new(BooleanArray::from(volatile_flags)),
Arc::new(BooleanArray::from(is_deleteds)),
Arc::new(StringArray::from(depends_ons)),
Arc::new(StringArray::from(derived_froms)),
Arc::new(StringArray::from(last_recalled_ats)),
Arc::new(StringArray::from(created_ats)),
Arc::new(StringArray::from(updated_ats)),
// 2026-10-08 (Gap#2):顺序必须与 arrow_schema() 严格一致
Arc::new(Float64Array::from(confidences)),
Arc::new(StringArray::from(confidence_sources)),
// GA-P0-1 (t_718d8707):顺序必须与 arrow_schema() 尾部新增列严格一致
Arc::new(Int64Array::from(human_not_useful_counts)),
Arc::new(Int64Array::from(dedup_downweight_counts)),
// GA-P0-2 (t_c6d41ac6):拆分字段列追加在末尾,与 arrow_schema 顺序一致
Arc::new(Int64Array::from(recall_seen_counts)),
Arc::new(Int64Array::from(recall_verified_counts)),
// GA-P0-3 (t_f785d7f2):delete_reason 列,与 arrow_schema 顺序一致(additive)
Arc::new(StringArray::from(delete_reasons)),
// 2026-10-13 (t_e9a4d18f P1):source_events 列,与 arrow_schema 顺序一致(additive)
Arc::new(StringArray::from(source_events_col)),
// 2026-10-13 (t_e9a4d18f P1):supersedes 列,与 arrow_schema 顺序一致(additive)
Arc::new(StringArray::from(supersedes_col)),
// 2026-10-14 (t_3e98b65a):superseded_by 列,与 arrow_schema 顺序一致(additive)
Arc::new(StringArray::from(superseded_by_col)),
// 2026-10-14 (t_3ada332b P3):bi-temporal 三列,与 arrow_schema 顺序一致(additive)
Arc::new(StringArray::from(valid_froms)),
Arc::new(StringArray::from(valid_tos)),
Arc::new(StringArray::from(provenances)),
];
let batch = RecordBatch::try_new(schema.clone(), columns)?;
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
let tbl = rt().block_on(db.open_table(table).execute())?;
let batches = RecordBatchIterator::new(
vec![batch].into_iter().map(Ok),
Arc::clone(&schema),
);
rt().block_on(tbl.add(Box::new(batches)).execute())?;
eprintln!("[lancedb] {}: inserted {} records", table, count);
Ok(count)
}
/// 更新记忆记录的字段(通过 SQL UPDATE)
/// fields_json 格式: [{"column":"recall_count","value":"recall_count + 1"}, ...]
/// 支持算术表达式(value 包含 + 或 - 时),先查当前值再计算
pub fn update(&self, table: &str, id: &str, fields_json: &str) -> Result<u64, Box<dyn std::error::Error>> {
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
let tbl = rt().block_on(db.open_table(table).execute())?;
let fields: Vec<serde_json::Value> = serde_json::from_str(fields_json)?;
let mut op = tbl.update().only_if(format!("id = '{}'", id.replace("\\'", "''")));
// 如果有算术表达式,先查当前值
let has_arith = fields.iter().any(|f| {
f["value"].as_str().map(|v| v.contains('+') || v.contains('-')).unwrap_or(false)
});
let mut current_values: std::collections::HashMap<String, i64> = std::collections::HashMap::new();
if has_arith {
// 查询当前值
let mut results = Box::pin(rt().block_on(
tbl.query()
.only_if(&format!("id = '{}'", id.replace("\\'", "''")))
.limit(1)
.execute(),
)?);
while let Some(Ok(batch)) = rt().block_on(results.next()) {
for i in 0..batch.num_rows() {
for f in &fields {
let col = f["column"].as_str().unwrap_or("");
if !col.is_empty() {
let val = col_i64(&batch, i, col);
current_values.insert(col.to_string(), val);
}
}
}
}
}
for f in &fields {
let col = f["column"].as_str().unwrap_or("");
let val = f["value"].as_str().unwrap_or("");
if col.is_empty() { continue; }
// 处理算术表达式: "column + N" 或 "column - N"
let final_val = if (val.contains('+') || val.contains('-')) && !val.starts_with("\\'") {
let parts: Vec<&str> = val.split_whitespace().collect();
if parts.len() == 3 {
let operand_col = parts[0];
let op_char = parts[1].chars().next().unwrap_or('+');
if let Ok(delta) = parts[2].parse::<i64>() {
let current = *current_values.get(operand_col).unwrap_or(&0);
let new_val = match op_char {
'+' => current.saturating_add(delta),
'-' => current.saturating_sub(delta),
_ => current,
};
eprintln!("[lancedb] update {}: col={} expr={} current={} -> {}", id, col, val, current, new_val);
new_val.to_string()
} else {
val.to_string()
}
} else {
val.to_string()
}
} else {
val.to_string()
};
eprintln!("[lancedb] UPDATE col={} val='{}'", col, final_val);
op = op.column(col, &final_val);
}
let updated = rt().block_on(op.execute())?;
eprintln!("[lancedb] UPDATE DONE: id={} fields={} rows={}", id, fields.len(), updated);
Ok(updated)
}
/// 累积召回批量更新(2026-09-12 优化B)。
///
/// 背景:recall 读路径旧实现对每条结果调一次 update,而 LanceDB 是 MVCC——
/// **每次 update 提交 = 一个版本**,实测 15 版本/分钟 → _versions 膨胀到 17.7G。
///
/// 做法(实测约束:lance 的 update 只支持基础 SQL 表达式,`CASE WHEN` 会被
/// planner 拒绝 —— 见 2026-09-12 实测 `Expression 'CASE WHEN ...' is not supported SQL in lance`):
/// 把整批**按 delta 分组**,每组用一次 `recall_count + delta` 表达式 + `id IN (...)` 谓词提交。
/// 真实召回流里 delta 绝大多数是 1(同一窗口内多次命中才 >1),所以
/// N 条记忆的批量 ≈ **1 个版本**(旧实现:N 个版本)。
///
/// items_json: [{"id":"mem_xxx","delta":2}],ts: RFC3339(last_recalled_at)
pub fn update_recall_batch(&self, items_json: &str, ts: &str) -> Result<u64, Box<dyn std::error::Error>> {
#[derive(serde::Deserialize)]
struct Item {
id: String,
delta: i64,
}
let items: Vec<Item> = serde_json::from_str(items_json)?;
if items.is_empty() {
return Ok(0);
}
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
let tbl = rt().block_on(db.open_table("memories").execute())?;
let esc = |s: &str| s.replace("\\'", "''");
let ts_lit = format!("'{}'", esc(ts));
// 按 delta 分组(BTreeMap 保证顺序稳定,便于日志/排障)
let mut groups: std::collections::BTreeMap<i64, Vec<String>> = std::collections::BTreeMap::new();
for it in &items {
groups.entry(it.delta).or_default().push(it.id.clone());
}
let mut total: u64 = 0;
for (delta, ids) in &groups {
let predicate = format!(
"id IN ({})",
ids.iter().map(|id| format!("'{}'", esc(id))).collect::<Vec<_>>().join(",")
);
let op = tbl
.update()
.only_if(&predicate)
.column("recall_seen_count", &format!("recall_seen_count + {}", delta))
.column("last_recalled_at", &ts_lit);
// GA-P0-2 (t_c6d41ac6):召回命中写 recall_seen_count(出现即 ++),
// 不再写 recall_count(冻结)。recall_count 只保留历史值,不再增长。
// t_268b0353(治根):召回只更新 seen/last_recalled_at,
// 不再写 freshness='verified'。freshness=verified 语义是「经过校验/用户
// 反馈」;「被召回」≠「被验证」。此前每次召回都盖 verified → 越常被召回
// 的旧快照越「可信」,时效判断全被污染(过期快照被当现状的机制根因之一)。
// 这是主写路径(zhiyi-consolidate sidecar)——Go 侧 write_buffer 只是兜底。
match rt().block_on(op.execute()) {
Ok(n) => {
eprintln!(
"[lancedb] BATCH UPDATE recall DONE: delta={} ids={} rows={} (单事务)",
delta,
ids.len(),
n
);
total += n;
}
Err(e) => {
// 兜底:谓词/表达式被拒时逐条更新(版本数退化为 N,但数据不丢、计数正确)
eprintln!(
"[lancedb] BATCH UPDATE(delta={}) failed ({}), fallback per-item ({} ids)",
delta,
e,
ids.len()
);
for id in ids {
// GA-P0-2 (t_c6d41ac6):召回命中写 recall_seen_count;fallback 同主路径。
let fields = format!(
r#"[{{"column":"recall_seen_count","value":"recall_seen_count + {}"}},{{"column":"last_recalled_at","value":"{}"}}]"#,
delta, ts_lit
);
match self.update("memories", id, &fields) {
Ok(n) => total += n,
Err(e2) => eprintln!("[lancedb] fallback update {} failed: {}", id, e2),
}
}
}
}
}
Ok(total)
}
/// 累积软删除批量更新(2026-09-12 优化C)。
///
/// 背景:软删除操作(SoftDelete)原先对**每条记忆**同步调用 lancedb.Update
/// (is_deleted = true + updated_at)。每次 update 提交产生一个版本。
/// 实测生产:碎片快速道和 decay 触发产生大量 SoftDelete,导致 _versions 膨胀。
///
/// 方案:
///
/// Record() 只把增量写进内存缓冲;**同一批次在一个窗口内多次 SoftDelete 合并为 一次批量提交**
/// Flush() 后台按窗口(默认 300s)+ 阈值(默认 256 条不同记忆)批量提交一次;
/// Rust 侧 lancedb_soft_delete_batch 把整批**设置相同字段**(is_deleted=true + updated_at=ts)一次提交;
/// → 每批版本数 = 1(旧实现:每行 1 个版本)。
///
/// 语义取舍(明确记录,便于日后审计):
/// - updated_at 最多延迟一个窗口(分钟级)。遗忘/衰减判定以「天」为单位,无影响。
/// - 进程崩溃会丢最后一个窗口的增量(软删除标记,不是记忆数据本身),可接受。
///
/// items_json: [{"id":"mem_xxx"}],ts: RFC3339(updated_at)
pub fn soft_delete_batch(&self, items_json: &str, ts: &str) -> Result<u64, Box<dyn std::error::Error>> {
#[derive(serde::Deserialize)]
struct Item {
id: String,
}
let items: Vec<Item> = serde_json::from_str(items_json)?;
if items.is_empty() {
return Ok(0);
}
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
let tbl = rt().block_on(db.open_table("memories").execute())?;
let esc = |s: &str| s.replace("\\'", "''");
let ts_lit = format!("'{}'", esc(ts));
// 按 id 分组(其实每个 id 只出现一次,但为了统一接口仍保持分组逻辑)
let mut groups: std::collections::BTreeMap<String, Vec<String>> = std::collections::BTreeMap::new();
for it in &items {
groups.entry(it.id.clone()).or_default().push(it.id.clone());
}
let mut total: u64 = 0;
for (_key, ids) in &groups {
let predicate = format!(
"id IN ({})",
ids.iter().map(|id| format!("'{}'", esc(id))).collect::<Vec<_>>().join(",")
);
let op = tbl
.update()
.only_if(&predicate)
.column("is_deleted", "true")
// GA-P0-3 (t_f785d7f2):批量软删不携带逐条原因 ⇒ delete_reason=unknown(兜底)。
.column("delete_reason", "'unknown'")
.column("updated_at", &ts_lit);
match rt().block_on(op.execute()) {
Ok(n) => {
eprintln!(
"[lancedb] BATCH SOFT DELETE DONE: ids={} rows={} (单事务)",
ids.len(),
n
);
total += n;
}
Err(e) => {
// 兜底:谓词/表达式被拒时逐条更新(版本数退化为 N,但数据不丢)
eprintln!(
"[lancedb] BATCH SOFT DELETE failed ({}), fallback per-item ({} ids)",
e,
ids.len()
);
for id in ids {
let fields = format!(
r#"[{{"column":"is_deleted","value":"true"}},{{"column":"updated_at","value":"{}"}}]"#,
ts_lit
);
match self.update("memories", id, &fields) {
Ok(n) => total += n,
Err(e2) => eprintln!("[lancedb] fallback soft delete {} failed: {}", id, e2),
}
}
}
}
}
Ok(total)
}
/// 清理 id 为空的垃圾行(P1-1, 2026-10-08)。
///
/// 背景:DB 存在 id/content/source 全空的垃圾行 ⇒ 命中「<30 字碎片快速道」
/// ⇒ SoftDelete("") 被 Rust IPC 拒绝("id required")⇒ 每轮刷错误日志 + 遗忘计数虚高。
/// 这些行无 id 无法按 id 软删,只能按「id 为空」条件一次性软删(软删非硬删,is_deleted=true)。
///
/// 做法:一次 update 提交把 id 为空的存活行标记 is_deleted=true + updated_at=ts。
/// 只匹配精确空串(id = '');不匹配纯空白(LanceDB SQL 的 trim 支持未验证,
/// 宁缺勿滥 —— 防御/计数代码已能安全跳过任何残留空 id,不会刷日志)。
/// ts 由 Go 侧传入(与 soft_delete_batch 同构,避免 Rust 侧依赖时钟库)。
pub fn purge_invalid(&self, ts: &str) -> Result<u64, Box<dyn std::error::Error>> {
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
let tbl = rt().block_on(db.open_table("memories").execute())?;
let esc = |s: &str| s.replace("\\'", "''");
let ts_lit = format!("'{}'", esc(ts));
let op = tbl
.update()
.only_if("id = ''")
.column("is_deleted", "true")
.column("delete_reason", "'retention'") // GA-P0-3:id 为空垃圾行 = 保留策略清理
.column("updated_at", &ts_lit);
let updated = rt().block_on(op.execute())?;
eprintln!("[lancedb] PURGE INVALID DONE: rows={}", updated);
Ok(updated)
}
/// 全量扫描 memories 表(用于深整),上限 10000 条
// P2 2026-09-06 安全版遗忘候选全表扫描(替代 febc2c9 风暴版):
// 1. 强制 limit(调用方传, main.rs 钳制硬上限 5000)
// 2. 不读 vector 列(1024 维 × 3000 条 ≈ 12MB+,IPC JSON 会卡)
// 3. 真读 last_recalled_at 列(scan_all 旧实现 String::new() 恒空 → 遗忘判定失真)
/// 全表扫描存活记忆(过滤 is_deleted,跳过 vector)。
/// ⚠️ 名字来自历史用途(遗忘候选),语义实为「列出存活记忆」。
/// 2026-09-21:新增 namespace 下推过滤 —— 修前 Go 侧 /memories 与 /memories/export
/// 用「零向量 ANN」冒充列表,零向量无方向 ⇒ 返回任意前 N 行再按 ns 过滤,
/// 计数完全失真(实测 hermes-main 报 1 条,实际 139+)。
/// 用 only_if 下推过滤,limit 作用于**过滤后**的结果集,避免 5000 硬上限截断。
pub fn scan_for_forgetting(
&self,
limit: usize,
namespace: Option<&str>,
) -> Result<Vec<MemoryRecord>, Box<dyn std::error::Error>> {
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
let tbl = rt().block_on(db.open_table("memories").execute())?;
let mut filter = String::from("is_deleted = false");
if let Some(ns) = namespace {
if !ns.is_empty() {
let escaped = ns.replace('\\', "\\\\").replace('\'', "''");
filter.push_str(&format!(" AND namespace = '{}'", escaped));
}
}
let mut results = Box::pin(rt().block_on(
tbl.query()
.only_if(&filter)
.limit(limit)
.execute()
)?);
let mut records = Vec::new();
while let Some(Ok(batch)) = rt().block_on(results.next()) {
for i in 0..batch.num_rows() {
records.push(Self::record_from_batch(&batch, i));
}
}
eprintln!("[lancedb] scan_forgetting → {} records (limit={}, ns={:?})", records.len(), limit, namespace);
Ok(records)
}
/// 遗忘候选「旋转窗口」扫描(2026-10-11 t_a4b8fa52 方案 c′)。
///
/// 修复「固定 2000 窗口不轮转」:LanceDB 0.15 查询**无 order_by**,`.limit(N)`
/// 返回的是**物理存储序**前 N 条(compact/merge 后重排,非创建序)⇒ 新记忆落物理序
/// 后部 ⇒ 永不进遗忘候选池。方案 c′ 不依赖任何排序下推,只靠「列投影 + 集合成员」:
/// 1. 只读 id 列取**全部存活 id**:`select(["id"])` + `only_if("is_deleted = false")`
/// 2. Rust 内按字符串排序(id = mem_<unixnano>,定宽 ⇒ 字符串序 == 创建序严格单调)
/// 3. 取**连续窗口** `ids[start : start+limit]`,start 由游标 after_id 定位
/// 4. 区间过滤把窗口还原成**全字段记录**:`only_if("is_deleted = false AND id >= lo AND id <= hi")`
/// 5. 新游标 = hi(窗口末 id);窗口不足 limit(取到末尾)⇒ 回绕,下次从 0 起
/// 游标持久化在 Go 侧(/var/lib/memoryweave/forget-cursor.json,复用 conflict-seen 原子写),
/// Rust 无状态、只接收 after_id。
pub fn scan_forgetting_window(
&self,
limit: usize,
after_id: Option<&str>,
) -> Result<ForgetWindow, Box<dyn std::error::Error>> {
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
let tbl = rt().block_on(db.open_table("memories").execute())?;
// ── 1. 列投影:只读 id,取全部存活 id ──
// ⚠️ LanceDB 0.15 query() 默认 limit=10 —— 必须显式给大上限,否则只取 10 条
// (spike 实测:无 limit ⇒ live_total=10;加 limit(200_000) ⇒ 22398)
let mut ids: Vec<String> = Vec::new();
{
let mut it = Box::pin(rt().block_on(
tbl.query()
.only_if("is_deleted = false")
.select(Select::columns(&["id"]))
.limit(200_000)
.execute()
)?);
while let Some(Ok(batch)) = rt().block_on(it.next()) {
for i in 0..batch.num_rows() {
let id = col_str(&batch, i, "id");
// 只保留 mem_ 形态(排除 episode `ep_` 行 —— 本卡审定:episode 原始轮次不参与遗忘)
if id.starts_with("mem_") {
ids.push(id);
}
}
}
}
let live_total = ids.len();
// ── 2. 排序 + 去重断言(AC-9:过滤后 id 必须唯一,防未来重复 id 静默跳行)──
ids.sort();
let before_dedup = ids.len();
ids.dedup();
if ids.len() != before_dedup {
eprintln!(
"[lancedb] WARN: forget window 过滤后出现 {} 条重复 id(已去重,不应发生)",
before_dedup - ids.len()
);
}
// ── 3. 游标定位窗口起点(纯逻辑,见 compute_forget_window)──
let (start, end, wrapped) = compute_forget_window(&ids, limit, after_id);
let window = &ids[start..end];
if window.is_empty() {
return Ok(ForgetWindow {
records: Vec::new(),
next_cursor: String::new(),
wrapped: true,
live_total,
});
}
let lo = &window[0];
let hi = &window[window.len() - 1];
let esc = |s: &str| s.replace('\'', "''");
let interval = format!(
"is_deleted = false AND id >= '{}' AND id <= '{}'",
esc(lo),
esc(hi)
);
// ── 4. 区间过滤还原全字段记录 ──
// ⚠️ 同样必须显式 limit(LanceDB 默认 10),否则窗口被截断
let mut records: Vec<MemoryRecord> = Vec::new();
{
let mut it = Box::pin(rt().block_on(
tbl.query()
.only_if(&interval)
.limit(200_000)
.execute()
)?);
while let Some(Ok(batch)) = rt().block_on(it.next()) {
for i in 0..batch.num_rows() {
records.push(Self::record_from_batch(&batch, i));
}
}
}
// 还原记录按 id 排序,与窗口切片一致(确定性输出)
records.sort_by(|a, b| a.id.cmp(&b.id));
let next_cursor = if wrapped { String::new() } else { hi.clone() };
eprintln!(
"[lancedb] scan_forgetting_window: limit={} after={:?} window=[{}-{}] n={} live_total={} wrapped={}",
limit, after_id, lo, hi, records.len(), live_total, wrapped
);
Ok(ForgetWindow {
records,
next_cursor,
wrapped,
live_total,
})
}
/// 从 RecordBatch 的一行构造 MemoryRecord(scan_forgetting / scan_forgetting_window 共用)。
fn record_from_batch(batch: &RecordBatch, i: usize) -> MemoryRecord {
MemoryRecord {
id: col_str(batch, i, "id"),
agent_id: col_str(batch, i, "agent_id"),
namespace: col_str(batch, i, "namespace"),
content: col_str(batch, i, "content"),
category: col_str(batch, i, "category"),
vector: Vec::new(), // 不读 vector — 遗忘候选不需要向量
tier: col_str(batch, i, "tier"),
importance: col_f64(batch, i, "importance"),
quality_score: col_f64(batch, i, "quality_score"),
recall_count: col_i64(batch, i, "recall_count"),
// GA-P0-2 (t_c6d41ac6):拆分字段读侧接线
recall_seen_count: col_i64(batch, i, "recall_seen_count"),
recall_verified_count: col_i64(batch, i, "recall_verified_count"),
useful_count: col_i64(batch, i, "useful_count"),
not_useful_count: col_i64(batch, i, "not_useful_count"),
human_not_useful_count: col_i64(batch, i, "human_not_useful_count"),
dedup_downweight_count: col_i64(batch, i, "dedup_downweight_count"),
freshness: col_str(batch, i, "freshness"),
version: col_i64(batch, i, "version"),
version_history: String::new(),
source: col_str(batch, i, "source"),
embedding_model: col_str(batch, i, "embedding_model"),
script_version: col_str(batch, i, "script_version"),
volatile_flag: col_bool(batch, i, "volatile_flag"),
is_deleted: col_bool(batch, i, "is_deleted"),
delete_reason: col_str(batch, i, "delete_reason"),
depends_on: String::new(),
derived_from: String::new(),
source_events: col_str(batch, i, "source_events"), // 2026-10-13 P1
supersedes: col_str(batch, i, "supersedes"), // 2026-10-13 P1
superseded_by: col_str(batch, i, "superseded_by"), // 2026-10-14 t_3e98b65a
valid_from: col_str(batch, i, "valid_from"), // 2026-10-14 t_3ada332b P3
valid_to: col_str(batch, i, "valid_to"), // 2026-10-14 t_3ada332b P3
provenance: col_str(batch, i, "provenance"), // 2026-10-14 t_3ada332b P3
last_recalled_at: col_str(batch, i, "last_recalled_at"),
created_at: col_str(batch, i, "created_at"),
updated_at: col_str(batch, i, "updated_at"),
confidence: col_f64_opt(batch, i, "confidence"),
confidence_source: col_str(batch, i, "confidence_source"),
}
}
/// 全表扫描(元数据;2026-09-21 治本改造)
/// ① 用 Select 投影跳过 vector 列(1024 维 = 4KB/条,Step 3/4 都不需要)
/// ② 上限从 10000 提到 200000(原值会静默丢记录:库已有 10065 条)
pub fn scan_all(&self) -> Result<Vec<MemoryRecord>, Box<dyn std::error::Error>> {
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
let tbl = rt().block_on(db.open_table("memories").execute())?;
const META_COLUMNS: &[&str] = &[
"id", "agent_id", "namespace", "content", "category",
"tier", "importance", "quality_score", "recall_count",
"useful_count", "not_useful_count", "freshness", "version",
"source", "volatile_flag", "is_deleted",
// GA-P0-1 (t_718d8707):scan_all 投影裁剪,漏加 ⇒ col_i64 取不到列 ⇒ 恒 0
"human_not_useful_count", "dedup_downweight_count",
// GA-P0-2 (t_c6d41ac6):拆分字段投影,漏加 ⇒ col_i64 取不到列 ⇒ 恒 0
"recall_seen_count", "recall_verified_count",
"last_recalled_at", "created_at", "updated_at",
// 2026-10-08 (Gap#2):scan_all 用投影裁剪,漏加 ⇒ col_f64_opt 取不到列 ⇒ 恒 None
"confidence", "confidence_source",
];
let mut results = Box::pin(rt().block_on(
tbl.query()
.only_if("is_deleted = false")
.select(Select::columns(META_COLUMNS))
.limit(200_000)
.execute()
)?);
let mut records = Vec::new();
while let Some(Ok(batch)) = rt().block_on(results.next()) {
for i in 0..batch.num_rows() {
records.push(MemoryRecord {
id: col_str(&batch, i, "id"),
agent_id: col_str(&batch, i, "agent_id"),
namespace: col_str(&batch, i, "namespace"),
content: col_str(&batch, i, "content"),
category: col_str(&batch, i, "category"),
vector: Vec::new(), // 投影扫描:不读 vector(Step 3/4 不需要,省 4KB/条)
tier: col_str(&batch, i, "tier"),
importance: col_f64(&batch, i, "importance"),
quality_score: col_f64(&batch, i, "quality_score"),
recall_count: col_i64(&batch, i, "recall_count"),
// GA-P0-2 (t_c6d41ac6):拆分字段读侧接线
recall_seen_count: col_i64(&batch, i, "recall_seen_count"),
recall_verified_count: col_i64(&batch, i, "recall_verified_count"),
useful_count: col_i64(&batch, i, "useful_count"),
not_useful_count: col_i64(&batch, i, "not_useful_count"),
// GA-P0-1 (t_718d8707):拆分字段读侧接线
human_not_useful_count: col_i64(&batch, i, "human_not_useful_count"),
dedup_downweight_count: col_i64(&batch, i, "dedup_downweight_count"),
freshness: col_str(&batch, i, "freshness"),
version: col_i64(&batch, i, "version"),
version_history: String::new(),
source: col_str(&batch, i, "source"),
embedding_model: col_str(&batch, i, "embedding_model"),
script_version: col_str(&batch, i, "script_version"),
volatile_flag: col_bool(&batch, i, "volatile_flag"),
is_deleted: col_bool(&batch, i, "is_deleted"),
delete_reason: col_str(&batch, i, "delete_reason"),
depends_on: String::new(),
derived_from: String::new(),
source_events: col_str(&batch, i, "source_events"), // 2026-10-13 P1
supersedes: col_str(&batch, i, "supersedes"), // 2026-10-13 P1
superseded_by: col_str(&batch, i, "superseded_by"), // 2026-10-14 t_3e98b65a
valid_from: col_str(&batch, i, "valid_from"), // 2026-10-14 t_3ada332b P3
valid_to: col_str(&batch, i, "valid_to"), // 2026-10-14 t_3ada332b P3
provenance: col_str(&batch, i, "provenance"), // 2026-10-14 t_3ada332b P3
last_recalled_at: String::new(),
created_at: col_str(&batch, i, "created_at"),
updated_at: col_str(&batch, i, "updated_at"),
// 2026-10-08 (Gap#2):读侧接线。col_f64_opt 保留 NULL 语义
confidence: col_f64_opt(&batch, i, "confidence"),
confidence_source: col_str(&batch, i, "confidence_source"),
});
}
}
eprintln!("[lancedb] scan_all → {} records", records.len());
Ok(records)
}
/// 全量扫描**已删除**行(is_deleted=true)(GA-P0-3 t_f785d7f2)。
///
/// 背景:所有 scan 路径(scan / scan_all / scan_forgetting / scan_forgetting_window)
/// 都过滤 is_deleted=false —— 存量 11228 条 is_deleted=true 的行**没有任何读路径能看**
/// (唯独 stats 能 count)。回填脚本需要枚举已删除行以回填 delete_reason 并出直方图。
///
/// 只读元数据(跳 vector),上限 200000。带 delete_reason 列(供回填时跳过已回填的)。
pub fn scan_deleted(&self) -> Result<Vec<MemoryRecord>, Box<dyn std::error::Error>> {
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
let tbl = rt().block_on(db.open_table("memories").execute())?;
const META_COLUMNS: &[&str] = &[
"id", "agent_id", "namespace", "content", "category",
"tier", "importance", "quality_score", "recall_count",
"useful_count", "not_useful_count", "freshness", "version",
"source", "volatile_flag", "is_deleted",
"human_not_useful_count", "dedup_downweight_count",
"recall_seen_count", "recall_verified_count",
"delete_reason", // GA-P0-3
"last_recalled_at", "created_at", "updated_at",
"confidence", "confidence_source",
];
let mut results = Box::pin(rt().block_on(
tbl.query()
.only_if("is_deleted = true")
.select(Select::columns(META_COLUMNS))
.limit(200_000)
.execute()
)?);
let mut records = Vec::new();
while let Some(Ok(batch)) = rt().block_on(results.next()) {
for i in 0..batch.num_rows() {
records.push(MemoryRecord {
id: col_str(&batch, i, "id"),
agent_id: col_str(&batch, i, "agent_id"),
namespace: col_str(&batch, i, "namespace"),
content: col_str(&batch, i, "content"),
category: col_str(&batch, i, "category"),
vector: Vec::new(),
tier: col_str(&batch, i, "tier"),
importance: col_f64(&batch, i, "importance"),
quality_score: col_f64(&batch, i, "quality_score"),
recall_count: col_i64(&batch, i, "recall_count"),
recall_seen_count: col_i64(&batch, i, "recall_seen_count"),
recall_verified_count: col_i64(&batch, i, "recall_verified_count"),
useful_count: col_i64(&batch, i, "useful_count"),
not_useful_count: col_i64(&batch, i, "not_useful_count"),
human_not_useful_count: col_i64(&batch, i, "human_not_useful_count"),
dedup_downweight_count: col_i64(&batch, i, "dedup_downweight_count"),
freshness: col_str(&batch, i, "freshness"),
version: col_i64(&batch, i, "version"),
version_history: String::new(),
source: col_str(&batch, i, "source"),
embedding_model: col_str(&batch, i, "embedding_model"),
script_version: col_str(&batch, i, "script_version"),
volatile_flag: col_bool(&batch, i, "volatile_flag"),
is_deleted: col_bool(&batch, i, "is_deleted"),
delete_reason: col_str(&batch, i, "delete_reason"),
depends_on: String::new(),
derived_from: String::new(),
source_events: col_str(&batch, i, "source_events"), // 2026-10-13 P1
supersedes: col_str(&batch, i, "supersedes"), // 2026-10-13 P1
superseded_by: col_str(&batch, i, "superseded_by"), // 2026-10-14 t_3e98b65a
valid_from: col_str(&batch, i, "valid_from"), // 2026-10-14 t_3ada332b P3
valid_to: col_str(&batch, i, "valid_to"), // 2026-10-14 t_3ada332b P3
provenance: col_str(&batch, i, "provenance"), // 2026-10-14 t_3ada332b P3
last_recalled_at: col_str(&batch, i, "last_recalled_at"),
created_at: col_str(&batch, i, "created_at"),
updated_at: col_str(&batch, i, "updated_at"),
confidence: col_f64_opt(&batch, i, "confidence"),
confidence_source: col_str(&batch, i, "confidence_source"),
});
}
}
eprintln!("[lancedb] scan_deleted → {} records", records.len());
Ok(records)
}
}
#[derive(Debug, Serialize)]
pub struct LanceDBStats {
pub total_memories: usize,
pub total_episodes: usize,
pub tombstone_count: usize,
pub data_dir: String,
}
impl LanceDBOps {
/// 追加一条真实 outcome 事件到 hit_events 表(GA-P0-4 t_bb10e8bc)。
/// append-only:每次真实 outcome 追加一行(带 created_at),**不覆盖式累加**。
pub fn append_hit_event(&self, memory_id: &str, outcome: &str, reason: &str, source: &str, ts: &str) -> Result<u64, Box<dyn std::error::Error>> {
let id = hit_event_id(memory_id);
let schema = Arc::new(hit_event_arrow_schema());
let columns: Vec<ArrayRef> = vec![
Arc::new(StringArray::from(vec![id])),
Arc::new(StringArray::from(vec![memory_id.to_string()])),
Arc::new(StringArray::from(vec![outcome.to_string()])),
Arc::new(StringArray::from(vec![reason.to_string()])),
Arc::new(StringArray::from(vec![source.to_string()])),
Arc::new(StringArray::from(vec![ts.to_string()])),
];
let batch = RecordBatch::try_new(schema.clone(), columns)?;
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
let tbl = rt().block_on(db.open_table("hit_events").execute())?;
let batches = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), Arc::clone(&schema));
rt().block_on(tbl.add(Box::new(batches)).execute())?;
eprintln!("[lancedb] hit_events: appended id={} outcome={} reason={}", memory_id, outcome, reason);
Ok(1)
}
/// 统计某 memory 在某时间窗起点(since, RFC3339)之后的 outcome 事件数(P2 自适应阈值按周窗口用)。
pub fn count_hit_events(&self, memory_id: &str, outcome: &str, since: &str) -> Result<u64, Box<dyn std::error::Error>> {
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
let tbl = rt().block_on(db.open_table("hit_events").execute())?;
// created_at 恒以 UTC(RFC3339 "Z") 存储(append 侧 chrono::Utc 生成)。
// since 可能是本地时区("+08:00")——字符串比较跨时区会错位,必须先归一到 UTC。
let since_utc = chrono::DateTime::parse_from_rfc3339(since)
.map(|dt| dt.with_timezone(&chrono::Utc).to_rfc3339_opts(chrono::SecondsFormat::Secs, true))
.unwrap_or_else(|_| since.to_string());
let filter = format!(
"memory_id = '{}' AND outcome = '{}' AND created_at >= '{}'",
sql_escape(memory_id), sql_escape(outcome), sql_escape(&since_utc)
);
let mut results = Box::pin(rt().block_on(tbl.query().only_if(&filter).execute())?);
let mut n: u64 = 0;
while let Some(Ok(batch)) = rt().block_on(results.next()) {
n += batch.num_rows() as u64;
}
Ok(n)
}
/// 读回某 memory 的事件明细(read-back 取证,非只看函数返回)。
pub fn get_hit_events(&self, memory_id: &str, limit: usize) -> Result<Vec<HitEventRecord>, Box<dyn std::error::Error>> {
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
let tbl = rt().block_on(db.open_table("hit_events").execute())?;
let filter = format!("memory_id = '{}'", sql_escape(memory_id));
let mut results = Box::pin(rt().block_on(
tbl.query().only_if(&filter).limit(limit).execute(),
)?);
let mut events = Vec::new();
while let Some(Ok(batch)) = rt().block_on(results.next()) {
for i in 0..batch.num_rows() {
events.push(HitEventRecord {
id: col_str(&batch, i, "id"),
memory_id: col_str(&batch, i, "memory_id"),
outcome: col_str(&batch, i, "outcome"),
reason: col_str(&batch, i, "reason"),
source: col_str(&batch, i, "source"),
created_at: col_str(&batch, i, "created_at"),
});
}
}
Ok(events)
}
pub fn stats(&self) -> Result<LanceDBStats, Box<dyn std::error::Error>> {
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
let tables = rt().block_on(db.table_names().execute())?;
let mut m = 0usize; let mut e = 0usize; let mut t = 0usize;
for name in &tables {
if let Ok(tbl) = rt().block_on(db.open_table(name).execute()) {
if name == "memories" {
if let Ok(cnt) = rt().block_on(tbl.count_rows(None)) { m = cnt; }
if let Ok(tc) = rt().block_on(tbl.count_rows(Some("is_deleted = true".to_string()))) { t = tc; }
} else if let Ok(cnt) = rt().block_on(tbl.count_rows(None)) {
match name.as_str() {
"episodes" => e = cnt,
_ => {}
}
}
}
}
Ok(LanceDBStats {
total_memories: m, total_episodes: e, tombstone_count: t,
data_dir: self.data_dir.to_string_lossy().to_string(),
})
}
}
// ── Arrow RecordBatch helpers ──
fn col_str(b: &RecordBatch, r: usize, c: &str) -> String {
b.column_by_name(c)
.and_then(|col| col.as_any().downcast_ref::<StringArray>())
.map(|a| a.value(r).to_string())
.unwrap_or_default()
}
fn col_f64(b: &RecordBatch, r: usize, c: &str) -> f64 {
b.column_by_name(c)
.and_then(|col| col.as_any().downcast_ref::<Float64Array>())
.map(|a| a.value(r))
.unwrap_or(0.0)
}
/// col_f64_opt 2026-10-08 (Gap#2):可空浮点读法。
/// 与 col_f64 的关键差别:**列不存在**或**该行是 NULL** 都返回 None,而不是 0.0。
/// 对 confidence 这是必须的 —— 0.0 会被读成「极不信任」,而 NULL 表示「未声明」。
fn col_f64_opt(b: &RecordBatch, r: usize, c: &str) -> Option<f64> {
b.column_by_name(c)
.and_then(|col| col.as_any().downcast_ref::<Float64Array>())
.and_then(|a| if a.is_null(r) { None } else { Some(a.value(r)) })
}
fn col_i64(b: &RecordBatch, r: usize, c: &str) -> i64 {
b.column_by_name(c)
.and_then(|col| col.as_any().downcast_ref::<Int64Array>())
.map(|a| a.value(r))
.unwrap_or(0)
}
fn col_bool(b: &RecordBatch, r: usize, c: &str) -> bool {
b.column_by_name(c)
.and_then(|col| col.as_any().downcast_ref::<BooleanArray>())
.map(|a| a.value(r))
.unwrap_or(false)
}
/// 读取 vector 列 (兼容 FixedSizeListArray 和 Float32Array)
fn col_vector(b: &RecordBatch, r: usize, c: &str) -> Vec<f32> {
let col = match b.column_by_name(c) {
Some(col) => col,
None => return vec![0.0_f32; 1024],
};
// 优先尝试 FixedSizeListArray (当前使用的格式)
if let Some(list_arr) = col.as_any().downcast_ref::<FixedSizeListArray>() {
let item_arr = list_arr.value(r);
if let Some(float_arr) = item_arr.as_any().downcast_ref::<Float32Array>() {
return float_arr.values().to_vec();
}
}
// 回退: Float32Array (展平格式)
if let Some(float_arr) = col.as_any().downcast_ref::<Float32Array>() {
let stride = 1024;
let start = r * stride;
if start + stride <= float_arr.len() {
return float_arr.values()[start..start + stride].to_vec();
}
}
vec![0.0_f32; 1024]
}
/// 纯逻辑:计算遗忘候选旋转窗口的下标区间。
/// - 输入:已排序去重的存活 id 集合(仅 mem_ 形态)、limit、after_id(游标)
/// - 返回:(start, end, wrapped)
/// - 窗口 = ids[start..end]
/// - wrapped=true 表示**游标回绕回 0**(本次传了游标,且游标 ≥ 全部 id 或已到存活集合末尾)
/// ⇒ 一轮完整遍历结束,下次从 0 起。无论 live 是否能被 limit 整除都能正确发出信号
/// (整除时靠「游标到末尾→下一轮回绕」触发,非整除时末窗口不足 limit 也回绕)。
/// 不依赖 LanceDB 排序下推,纯字符串集合成员计算(方案 c′ 核心,可单测)。
fn compute_forget_window(
ids: &[String],
limit: usize,
after_id: Option<&str>,
) -> (usize, usize, bool) {
let has_cursor = matches!(after_id, Some(a) if !a.is_empty());
if ids.is_empty() {
// 空库/全删:无可扫,直接标记一轮遍历结束(回绕语义,使 Go 侧游标复位不卡)
return (0, 0, true);
}
let mut start = if has_cursor {
// 第一个 > after_id 的下标;after_id 已删除/不在集合中 → 仍续扫之后(不丢、不崩)
ids.partition_point(|x| x.as_str() <= after_id.unwrap())
} else {
0
};
// 游标 ≥ 全部 id(已到末尾)→ 回绕到 0,并标记「一轮遍历结束」
let mut wrapped = false;
if start >= ids.len() {
start = 0;
wrapped = has_cursor;
}
let end = (start + limit).min(ids.len());
(start, end, wrapped)
}
#[cfg(test)]
mod tests {
use super::*;
/// 构造 N 个连续 mem_<n> 形态 id(定宽前缀保证字符串序 == 数字序)。
fn mk_ids(n: u64) -> Vec<String> {
(0..n).map(|i| format!("mem_{:019}", i)).collect()
}
/// 断言:连续 N 轮窗口的并集 == 全部存活 id(全覆盖判据,AC-1)。
/// 轮数上界 = ceil(live/limit),模拟真实调度(每 6h 一轮)。
fn union_covers_all(live: usize, limit: usize) -> bool {
let ids = mk_ids(live as u64);
let mut cursor = String::new();
let mut union = std::collections::HashSet::new();
let mut rounds = 0usize;
let max_rounds = live.div_ceil(limit) + 2; // 上界 + 余量
loop {
let after = if cursor.is_empty() { None } else { Some(cursor.as_str()) };
let (s, e, wrapped) = compute_forget_window(&ids, limit, after);
for x in &ids[s..e] {
union.insert(x.clone());
}
rounds += 1;
if wrapped {
break;
}
cursor = ids[e - 1].clone(); // 窗口末 id = 新游标
if rounds > max_rounds {
break; // 防死循环(理论不会触发)
}
}
union.len() == live && rounds <= max_rounds
}
/// 断言:窗口内全部「不满足遗忘条件」时,下一轮窗口仍与上一轮不同(不卡死,AC-2)。
fn not_stuck(live: usize, limit: usize) -> bool {
let ids = mk_ids(live as u64);
let after = Some(ids[0].as_str()); // 假设第 0 条已在窗口
let (s0, e0, _) = compute_forget_window(&ids, limit, None);
let w0: Vec<String> = ids[s0..e0].to_vec();
let (s1, e1, _) = compute_forget_window(&ids, limit, after);
let w1: Vec<String> = ids[s1..e1].to_vec();
// 窗口前进 → 集合不同(不卡死)
w0 != w1 && !w1.is_empty()
}
#[test]
fn test_window_full_coverage() {
// 小表(能整除):9 存活 / 3 窗口 → 3 轮全覆盖
assert!(union_covers_all(9, 3), "9/3 应 3 轮全覆盖");
// 非整除:10 存活 / 3 窗口 → 最后一轮不足 → 仍全覆盖
assert!(union_covers_all(10, 3), "10/3 应 ceil=4 轮全覆盖");
// 大表:22447 存活 / 2000 窗口 → ceil=12 轮全覆盖(贴近生产量级)
assert!(union_covers_all(22447, 2000), "22447/2000 应 12 轮全覆盖");
// 单条 / 空边界不崩
assert!(union_covers_all(1, 2000), "1 条应 1 轮全覆盖");
}
#[test]
fn test_window_not_stuck() {
// 窗口全不满足遗忘条件(模拟前 2000 条被高 recall 保护)→ 下轮仍换一批
assert!(not_stuck(10, 3), "前 3 条保护 → 下轮应换一批");
assert!(not_stuck(10000, 2000), "大表不卡死");
}
#[test]
fn test_window_wrap_boundary() {
// 空集合:window 为空,wrapped=true,不 panic
let empty: Vec<String> = vec![];
let (s, e, wrapped) = compute_forget_window(&empty, 2000, None);
assert_eq!(s, 0);
assert_eq!(e, 0);
assert!(wrapped);
// 单条:首轮返回该条(未 wrapped,游标=该 id);次轮游标回绕才 wrapped
let ids = mk_ids(1);
let (s, e, wrapped) = compute_forget_window(&ids, 2000, None);
assert_eq!(s, 0);
assert_eq!(e, 1);
assert!(!wrapped, "单条首轮覆盖完成,但游标尚未回绕 → 不标记");
let only = ids[0].clone();
let (s2, _e2, wrapped2) = compute_forget_window(&ids, 2000, Some(&only));
assert_eq!(s2, 0);
assert!(wrapped2, "单条次轮游标回绕 → 标记一轮遍历结束");
// 游标已到末尾(= 最后一个 id)→ 下轮回绕从 0 起,并标记一轮遍历结束
let ids = mk_ids(10);
let last = ids.last().unwrap().clone();
let (s, e, wrapped) = compute_forget_window(&ids, 3, Some(&last));
assert_eq!(s, 0, "游标在末尾 → 应回绕到 0");
assert_eq!(e, 3);
assert!(wrapped, "回绕到 0 → 标记一轮遍历结束");
}
#[test]
fn test_window_cursor_deleted_resilience() {
// 游标 id 已被删除(不在集合中)→ 降级续扫其后,不 panic(AC-8)
let mut ids = mk_ids(10);
// 真删掉 id=5(模拟 soft-delete 后该 id 不在存活集合)
ids.retain(|x| x != "mem_0000000000000000005");
let gone = format!("mem_{:019}", 5); // 游标指向已删的 5
let (s, e, _) = compute_forget_window(&ids, 3, Some(&gone));
assert_eq!(s, 5, "游标=已删的5 → 应从第一个>5的下标续扫(不丢不崩)");
assert_eq!(e, 8);
// 游标大于全部 id → 回绕从头
let big = format!("mem_{:019}", 999);
let (s, e, wrapped) = compute_forget_window(&ids, 3, Some(&big));
assert_eq!(s, 0, "游标大于全部 → 回绕从头");
assert_eq!(e, 3);
assert!(wrapped, "游标越界 → 标记一轮遍历结束");
}
#[test]
fn test_window_dedup_assertion() {
// 排序后去重:重复 id 应被去重且不崩(AC-9)
let mut ids = mk_ids(5);
ids.push("mem_0000000000000000002".to_string()); // 重复
ids.sort();
let before = ids.len();
ids.dedup();
assert_eq!(ids.len(), before - 1, "重复 id 应去重");
// 去重后仍能正确轮转
let (s, e, _) = compute_forget_window(&ids, 2000, None);
assert_eq!(s, 0);
assert_eq!(e, ids.len());
}
/// 真实库只读覆盖率测试(AC-1/2,需 LIVE_DB_WINDOW=1 才跑,避免污染常规 cargo test)。
/// 用生产代码路径 scan_forgetting_window 循环取窗直到 wrapped,验证:
/// ① 连续窗口两两不同(不卡死)
/// ② 并集最终覆盖全部存活 mem_ 记忆(覆盖率→100%)
/// ③ 池内 created_at>10-07 条数从旧实现~2 显著上升
#[test]
fn test_live_db_full_coverage() {
if std::env::var("LIVE_DB_WINDOW").as_deref() != Ok("1") {
eprintln!("[skip] LIVE_DB_WINDOW!=1");
return;
}
let ldb = LanceDBOps::open("/var/lib/memoryweave").expect("open live db");
let mut union = std::collections::HashSet::new();
let mut cursor = String::new();
let mut round = 0usize;
let mut live_total = 0usize;
let mut max_recent = 0usize;
let mut rounds_with_recent = 0usize;
loop {
let after = if cursor.is_empty() { None } else { Some(cursor.as_str()) };
let w = ldb.scan_forgetting_window(2000, after).expect("window");
if round == 0 { live_total = w.live_total; }
let recent = w.records.iter().filter(|r| r.created_at > "2026-10-07T00:00:00Z".to_string()).count();
max_recent = max_recent.max(recent);
if recent > 0 { rounds_with_recent += 1; }
for r in &w.records { union.insert(r.id.clone()); }
round += 1;
println!("round{} n={} live_total={} wrapped={} recent>10-07={} union={}/{}",
round, w.records.len(), w.live_total, w.wrapped, recent, union.len(), live_total);
if w.wrapped { break; }
cursor = w.next_cursor;
if round > live_total.div_ceil(2000) + 5 { println!("guard break"); break; }
}
println!("RESULT: rounds={} union={} live_total={} coverage={:.4} max_recent_in_pool={} rounds_with_recent={}",
round, union.len(), live_total, union.len() as f64 / live_total as f64, max_recent, rounds_with_recent);
assert_eq!(union.len(), live_total, "全库存活 mem_ 记忆应被全覆盖");
}
}