根因:t_3ada332b 只改了 models 结构体 + server.go 算值,LanceDB 边界三层缺失
(Go lancedb_ipc.go 写 payload + 读回解包、Rust arrow_schema + struct + builder + col_str)
⇒ valid_to 永不持久化 ⇒ validityExpired("") 恒 false ⇒ 失效降权生产 no-op。
本次(照 t_3e98b65a 给 superseded_by 的 5 层模式逐一对齐):
1. Go lancedb_ipc.go 写侧 payload 补 valid_from/valid_to/provenance 三键
2. Go 读侧补三处:Search 解包struct+输出映射 / Scan 解包struct+输出映射 / GetSkillCandidates mapStr 路径
3. Rust lancedb_ops.rs struct 字段(#[serde(default)],存量行不报错)
4. Rust arrow_schema Field::new 三列(nullable,存量由 open() 迁移补列)
5. Rust 写 builder col vec + push + StringArray + 5 处读侧 col_str
新增 e2e 回归闸门 bitemporal_e2e_test.go:经真实 LanceDB sidecar 写一条 valid_to 已过期记忆
→ 读回断言三列非空且等于写入值(ZHIYI_BITEMPORAL_SOCK 门控,未设跳过)。
单测(AC1/AC2/AC3/AC4)+e2e 全 PASS;全量 go test 仅既有 flaky TestCommitConflictGate_E2E(clean baseline 同败)。
1545 lines
81 KiB
Rust
1545 lines
81 KiB
Rust
use std::sync::OnceLock;
|
||
|
||
fn rt() -> &'static tokio::runtime::Runtime {
|
||
static RT: OnceLock<tokio::runtime::Runtime> = OnceLock::new();
|
||
RT.get_or_init(|| tokio::runtime::Runtime::new().unwrap())
|
||
}
|
||
|
||
// LanceDB 原生读写 — lancedb crate 0.15
|
||
// 使用 pollster::block_on 处理 async API
|
||
// API: connect(path) → create_empty_table/table_names → vector_search → add(arrow_data)
|
||
|
||
use std::path::PathBuf;
|
||
use std::sync::Arc;
|
||
|
||
use serde::{Deserialize, Serialize};
|
||
|
||
use arrow::array::{Array, ArrayRef, BooleanArray, FixedSizeListArray, Float32Array, Float64Array, Int64Array, StringArray};
|
||
use arrow::datatypes::{DataType, Field, Schema};
|
||
use arrow::record_batch::{RecordBatch, RecordBatchIterator};
|
||
use futures::StreamExt;
|
||
use lancedb::query::{ExecutableQuery, QueryBase, Select};
|
||
use lancedb::table::NewColumnTransform;
|
||
|
||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||
pub struct MemoryRecord {
|
||
pub id: String, pub agent_id: String, pub namespace: String,
|
||
pub content: String, pub category: String, pub vector: Vec<f32>,
|
||
pub tier: String, pub importance: f64, pub quality_score: f64,
|
||
pub recall_count: i64, pub useful_count: i64, pub not_useful_count: i64,
|
||
// GA-P0-2 (t_c6d41ac6):拆分 recall_count 的两种语义(「出现在结果里」/「被验证使用」)。
|
||
// recall_count 曾把两种语义混写 —— 每次召回命中都 ++,被当成 V 值喂给衰减/遗忘逻辑,
|
||
// 造成「被召回 500+ 次所以永不遗忘」的虚高保命。现拆分为:
|
||
// recall_seen_count 出现即 ++(只用于观测/排序去偏,不参与 V 值/衰减权重)
|
||
// recall_verified_count有正反馈(IncrementUseful)或下游引用才 ++(唯一参与 V 值/衰减)
|
||
// recall_count 保留(deprecated),冻结不再增长,仅向后兼容读取历史值。
|
||
pub recall_seen_count: i64, pub recall_verified_count: i64,
|
||
// GA-P0-1 (t_718d8707):拆分 not_useful_count 的两种语义(人工「没用」/ dedup 自动降权)。
|
||
// not_useful_count 保留(deprecated),新代码不再读它;human_not_useful_count 与
|
||
// dedup_downweight_count 是新写入点。arrow_schema 尾部新增列(nullable,存量补 NULL)。
|
||
pub human_not_useful_count: i64, pub dedup_downweight_count: i64,
|
||
pub freshness: String, pub version: i64,
|
||
pub version_history: String, pub source: String,
|
||
// 2026-10-08 (t_7e3577c5):血缘字段
|
||
pub embedding_model: String, pub script_version: String,
|
||
// 2026-10-08 (Gap#2):信任等级。Option<f64> —— 存量行是 NULL,必须与「显式 0.0」区分。
|
||
// 漏了这里 ⇒ IPC 响应无该字段 ⇒ Go 侧解包恒 nil ⇒ 加列也白加(静默 no-op)。
|
||
#[serde(default)]
|
||
pub confidence: Option<f64>,
|
||
#[serde(default)]
|
||
pub confidence_source: String,
|
||
pub volatile_flag: bool, pub is_deleted: bool,
|
||
// GA-P0-3 (t_f785d7f2):拆分 is_deleted 的删除原因。is_deleted=true 不再「不可分辨」——
|
||
// 坏记忆被删(contradiction/manual/retention) 与 合并/整理被删(merge_dup) 现可分列。
|
||
// 固定枚举: merge_dup | contradiction | manual | retention | unknown。additive 新列,
|
||
// 存量行是 NULL(读路径 col_str 兜底 ""),回填脚本以 tombstones 为据尽力判定。
|
||
pub delete_reason: String,
|
||
pub depends_on: String, pub derived_from: String,
|
||
// 2026-10-13 (t_e9a4d18f P1) 证据链:全量来源 episode ID 的 JSON 数组字符串。
|
||
// 与 depends_on 同款(JSON 串列),additive 新列,存量由 schema 演进补 NULL。
|
||
pub source_events: String,
|
||
// 2026-10-13 (t_e9a4d18f P1) 矛盾保留:被取代的旧记忆 ID(JSON 数组字符串)
|
||
pub supersedes: String,
|
||
// 2026-10-14 (t_3e98b65a) 被取代方标记:反向指针(JSON 数组字符串,additive 新列)
|
||
#[serde(default)]
|
||
pub superseded_by: String,
|
||
// 2026-10-14 (t_3ada332b P3) bi-temporal 有效期三列:RFC3339;空 = 永久有效。
|
||
// #[serde(default)] ⇒ 存量行/缺列读回 "",降权永不触发 ⇒ 必须 5 层齐,缺 Rust 这层即 no-op。
|
||
#[serde(default)]
|
||
pub valid_from: String,
|
||
#[serde(default)]
|
||
pub valid_to: String,
|
||
#[serde(default)]
|
||
pub provenance: String,
|
||
pub last_recalled_at: String, pub created_at: String, pub updated_at: String,
|
||
}
|
||
|
||
#[derive(Clone)]
|
||
pub struct LanceDBOps {
|
||
data_dir: PathBuf,
|
||
}
|
||
|
||
/// 遗忘候选「旋转窗口」返回(方案 c′ 2026-10-11 t_a4b8fa52)。
|
||
/// - records: 本窗口全字段记录(已按 id 排序)
|
||
/// - next_cursor: 新游标 = 窗口末 id;wrapped=true 时为空串(下轮从 0 起)
|
||
/// - wrapped: 窗口不足 limit(取到存活集合末尾)⇒ 本轮遍历结束
|
||
/// - live_total: 存活 mem_ id 总数(排除 episode),供覆盖度量与 AC-7 吞吐判据
|
||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||
pub struct ForgetWindow {
|
||
pub records: Vec<MemoryRecord>,
|
||
pub next_cursor: String,
|
||
pub wrapped: bool,
|
||
pub live_total: usize,
|
||
}
|
||
|
||
fn arrow_schema() -> Schema {
|
||
Schema::new(vec![
|
||
Field::new("id", DataType::Utf8, false),
|
||
Field::new("agent_id", DataType::Utf8, false),
|
||
Field::new("namespace", DataType::Utf8, false),
|
||
Field::new("content", DataType::Utf8, false),
|
||
Field::new("category", DataType::Utf8, false),
|
||
Field::new("vector", DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 1024), true),
|
||
Field::new("tier", DataType::Utf8, false),
|
||
Field::new("importance", DataType::Float64, false),
|
||
Field::new("quality_score", DataType::Float64, false),
|
||
Field::new("recall_count", DataType::Int64, false),
|
||
Field::new("useful_count", DataType::Int64, false),
|
||
Field::new("not_useful_count", DataType::Int64, false),
|
||
Field::new("freshness", DataType::Utf8, false),
|
||
Field::new("version", DataType::Int64, false),
|
||
Field::new("version_history", DataType::Utf8, true),
|
||
Field::new("source", DataType::Utf8, true),
|
||
Field::new("embedding_model", DataType::Utf8, true),
|
||
Field::new("script_version", DataType::Utf8, true),
|
||
Field::new("volatile_flag", DataType::Boolean, false),
|
||
Field::new("is_deleted", DataType::Boolean, false),
|
||
Field::new("depends_on", DataType::Utf8, true),
|
||
Field::new("derived_from", DataType::Utf8, true),
|
||
Field::new("last_recalled_at", DataType::Utf8, true),
|
||
Field::new("created_at", DataType::Utf8, false),
|
||
Field::new("updated_at", DataType::Utf8, false),
|
||
// 2026-10-08 (Gap#2):来源加权。confidence=写入方显式声明的信任等级(不从 source 反推);
|
||
// confidence_source=authoritative|estimated,用于区分「写入时声明」与「事后回填猜测」,
|
||
// 是日后批量纠偏的唯一手段(否则无法定向打 estimated 行)。
|
||
// 均 nullable:存量行保持 NULL,读路径 COALESCE→1.0,使加权上线首日即 no-op。
|
||
Field::new("confidence", DataType::Float64, true),
|
||
Field::new("confidence_source", DataType::Utf8, true),
|
||
// GA-P0-1 (t_718d8707):拆分 not_useful_count 的两种语义。additive 新列;
|
||
// 存量表由 open() 的 schema 演进迁移自动补列(nullable 容纳存量 NULL,
|
||
// 与 t_3df364b8 血缘列 / Gap#2 confidence 的迁移模式一致)。
|
||
Field::new("human_not_useful_count", DataType::Int64, true),
|
||
Field::new("dedup_downweight_count", DataType::Int64, true),
|
||
// GA-P0-2 (t_c6d41ac6):拆分 recall_count 的两种语义。additive 新列;
|
||
// 存量表由 open() 的 schema 演进自动补列(nullable 容纳存量 NULL)。
|
||
Field::new("recall_seen_count", DataType::Int64, true),
|
||
Field::new("recall_verified_count", DataType::Int64, true),
|
||
// GA-P0-3 (t_f785d7f2):is_deleted 的删除原因(固定枚举)。additive 新列,
|
||
// nullable 容纳存量 NULL;回填脚本以 tombstones 为据尽力判定,判不出写 unknown。
|
||
Field::new("delete_reason", DataType::Utf8, true),
|
||
// 2026-10-13 (t_e9a4d18f P1) 证据链:全量来源 episode ID(JSON 数组字符串)。
|
||
// additive 新列,存量由 schema 演进补 NULL(与 depends_on 同款)。
|
||
Field::new("source_events", DataType::Utf8, true),
|
||
// 2026-10-13 (t_e9a4d18f P1) 矛盾保留(additive)
|
||
Field::new("supersedes", DataType::Utf8, true),
|
||
// 2026-10-14 (t_3e98b65a) 被取代方标记(additive,反向指针)
|
||
Field::new("superseded_by", DataType::Utf8, true),
|
||
// 2026-10-14 (t_3ada332b P3) bi-temporal 有效期三列(additive,nullable 容纳存量 NULL)
|
||
Field::new("valid_from", DataType::Utf8, true),
|
||
Field::new("valid_to", DataType::Utf8, true),
|
||
Field::new("provenance", DataType::Utf8, true),
|
||
])
|
||
}
|
||
|
||
// GA-P0-4 (t_bb10e8bc):hits+/hits− outcome 事件记录(append-only 事件表)。
|
||
// 与 MemoryRecord 计数列不同:这是「按时间戳追加」的事件,**不覆盖式累加**,
|
||
// 便于 P2 自适应阈值按周窗口聚合。语义定义见 Go 侧 internal/storage/hit_event.go。
|
||
// 只由真实 outcome 写入(human_feedback|contradiction|replacement|downstream_use);
|
||
// dedup 降权 / 单纯召回出现禁止写入。
|
||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||
pub struct HitEventRecord {
|
||
pub id: String,
|
||
pub memory_id: String,
|
||
pub outcome: String, // pos | neg
|
||
pub reason: String, // human_feedback|contradiction|replacement|downstream_use
|
||
pub source: String,
|
||
pub created_at: String, // RFC3339
|
||
}
|
||
|
||
fn hit_event_arrow_schema() -> Schema {
|
||
Schema::new(vec![
|
||
Field::new("id", DataType::Utf8, false),
|
||
Field::new("memory_id", DataType::Utf8, false),
|
||
Field::new("outcome", DataType::Utf8, false),
|
||
Field::new("reason", DataType::Utf8, false),
|
||
Field::new("source", DataType::Utf8, true),
|
||
Field::new("created_at", DataType::Utf8, false),
|
||
])
|
||
}
|
||
|
||
// 2026-10-13 (t_e9a4d18f P2) 常设答案表(mental models / knowledge pages)。
|
||
// question_id 主键;answer 为生成的常设答案;source_memories 为可追溯的证据记忆
|
||
// ID 列表(JSON 数组字符串);version 每次改写 +1。读取 = 一次主键查询,无检索无 LLM。
|
||
fn standing_answer_arrow_schema() -> Schema {
|
||
Schema::new(vec![
|
||
Field::new("question_id", DataType::Utf8, false),
|
||
Field::new("question", DataType::Utf8, false),
|
||
Field::new("answer", DataType::Utf8, false),
|
||
Field::new("source_memories", DataType::Utf8, true),
|
||
Field::new("version", DataType::Int64, false),
|
||
Field::new("created_at", DataType::Utf8, false),
|
||
Field::new("updated_at", DataType::Utf8, false),
|
||
])
|
||
}
|
||
|
||
/// 生成 hit_event 行的唯一 id(memory_id + 微秒时间戳;同微秒追加几乎不可能)。
|
||
fn hit_event_id(memory_id: &str) -> String {
|
||
let micros = std::time::SystemTime::now()
|
||
.duration_since(std::time::UNIX_EPOCH)
|
||
.map(|d| d.as_micros())
|
||
.unwrap_or(0);
|
||
format!("{}-{}", memory_id, micros)
|
||
}
|
||
|
||
/// SQL 单引号转义(用于 hit_events 过滤条件的字符串字面量)。
|
||
fn sql_escape(s: &str) -> String {
|
||
s.replace('\'', "''")
|
||
}
|
||
|
||
impl LanceDBOps {
|
||
pub fn open(data_dir: &str) -> Result<Self, Box<dyn std::error::Error>> {
|
||
let path = PathBuf::from(data_dir);
|
||
std::fs::create_dir_all(&path)?;
|
||
let db = rt().block_on(lancedb::connect(path.to_str().unwrap()).execute())?;
|
||
let tables = rt().block_on(db.table_names().execute())?;
|
||
let target = arrow_schema();
|
||
for name in ["memories", "episodes", "tombstones"] {
|
||
if !tables.iter().any(|t| t == name) {
|
||
rt().block_on(db.create_empty_table(name, Arc::new(arrow_schema())).execute()).ok();
|
||
eprintln!("[lancedb] created table: {}", name);
|
||
continue;
|
||
}
|
||
// 已存在表 → schema 演进迁移:补齐 arrow_schema() 中有而实际表缺失的列
|
||
// (2026-10-08 t_3df364b8:commit 5038d19 加 2 血缘列后 episodes 仍 23 列,
|
||
// 每次 Append 25 列 ⇒ schema 不匹配 ⇒ 静默丢数据。这里统一补列,三表共用一套 schema。)
|
||
let tbl = rt().block_on(db.open_table(name).execute())?;
|
||
let tbl_schema = rt().block_on(tbl.schema())?;
|
||
let mut missing: Vec<(String, String)> = Vec::new();
|
||
for f in target.fields() {
|
||
if tbl_schema.field_with_name(f.name()).is_err() {
|
||
// 缺失列一律补**可空**(2026-10-08 修正:原先字符串给 '',
|
||
// 而 add_columns 的 SQL 字面量默认值会把列标成 **non-null** ——
|
||
// 与 arrow_schema 的 nullable=true 冲突,写入 NULL 会被 lance 拒绝:
|
||
// "contained null values even though the field is marked non-null"
|
||
// 实测对照:embedding_model/script_version(历史迁移列)都是 nullable=true。
|
||
let default = match f.data_type() {
|
||
DataType::Float64 => "CAST(NULL AS DOUBLE)",
|
||
DataType::Float32 => "CAST(NULL AS FLOAT)",
|
||
DataType::Int64 => "CAST(NULL AS BIGINT)",
|
||
DataType::Boolean => "CAST(NULL AS BOOLEAN)",
|
||
_ => "CAST(NULL AS STRING)",
|
||
};
|
||
missing.push((f.name().to_string(), default.to_string()));
|
||
}
|
||
}
|
||
if !missing.is_empty() {
|
||
let cols: Vec<String> = missing.iter().map(|(c, _)| c.clone()).collect();
|
||
eprintln!("[lancedb] migrating {}: adding missing cols {:?}", name, cols);
|
||
rt().block_on(tbl.add_columns(
|
||
NewColumnTransform::SqlExpressions(missing),
|
||
None,
|
||
))?;
|
||
eprintln!("[lancedb] migrated {}: added {:?}", name, cols);
|
||
}
|
||
}
|
||
// GA-P0-4 (t_bb10e8bc):hits+/hits− outcome 事件表(append-only,独立 schema)。
|
||
if !tables.iter().any(|t| t == "hit_events") {
|
||
rt().block_on(db.create_empty_table("hit_events", Arc::new(hit_event_arrow_schema())).execute()).ok();
|
||
eprintln!("[lancedb] created table: hit_events");
|
||
}
|
||
// 2026-10-13 (t_e9a4d18f P2):常设答案表(mental models)。独立 schema。
|
||
if !tables.iter().any(|t| t == "standing_answers") {
|
||
rt().block_on(db.create_empty_table("standing_answers", Arc::new(standing_answer_arrow_schema())).execute()).ok();
|
||
eprintln!("[lancedb] created table: standing_answers");
|
||
}
|
||
eprintln!("[lancedb] opened at {:?} ({} tables)", path, tables.len());
|
||
Ok(Self { data_dir: path })
|
||
}
|
||
|
||
pub fn init_tables(&self) -> Result<(), Box<dyn std::error::Error>> {
|
||
Ok(())
|
||
}
|
||
|
||
pub fn search(&self, query_vec: &[f32], top_k: usize, namespace: Option<&str>) -> Result<Vec<MemoryRecord>, Box<dyn std::error::Error>> {
|
||
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
|
||
let tbl = rt().block_on(db.open_table("memories").execute())?;
|
||
|
||
let mut q = tbl.vector_search(query_vec.to_vec())?.limit(top_k);
|
||
if let Some(ns) = namespace {
|
||
q = q.only_if(&format!("namespace = '{}'", ns));
|
||
}
|
||
q = q.only_if("is_deleted = false");
|
||
|
||
let mut results = Box::pin(rt().block_on(q.execute())?);
|
||
let mut records = Vec::new();
|
||
while let Some(Ok(batch)) = rt().block_on(results.next()) {
|
||
for i in 0..batch.num_rows() {
|
||
records.push(MemoryRecord {
|
||
id: col_str(&batch, i, "id"),
|
||
agent_id: col_str(&batch, i, "agent_id"),
|
||
namespace: col_str(&batch, i, "namespace"),
|
||
content: col_str(&batch, i, "content"),
|
||
category: col_str(&batch, i, "category"),
|
||
vector: col_vector(&batch, i, "vector"),
|
||
tier: col_str(&batch, i, "tier"),
|
||
importance: col_f64(&batch, i, "importance"),
|
||
quality_score: col_f64(&batch, i, "quality_score"),
|
||
recall_count: col_i64(&batch, i, "recall_count"),
|
||
// GA-P0-2 (t_c6d41ac6):拆分字段读侧接线
|
||
recall_seen_count: col_i64(&batch, i, "recall_seen_count"),
|
||
recall_verified_count: col_i64(&batch, i, "recall_verified_count"),
|
||
useful_count: col_i64(&batch, i, "useful_count"),
|
||
not_useful_count: col_i64(&batch, i, "not_useful_count"),
|
||
// GA-P0-1 (t_718d8707):拆分字段读侧接线
|
||
human_not_useful_count: col_i64(&batch, i, "human_not_useful_count"),
|
||
dedup_downweight_count: col_i64(&batch, i, "dedup_downweight_count"),
|
||
freshness: col_str(&batch, i, "freshness"),
|
||
version: col_i64(&batch, i, "version"),
|
||
version_history: String::new(),
|
||
source: col_str(&batch, i, "source"),
|
||
embedding_model: col_str(&batch, i, "embedding_model"),
|
||
script_version: col_str(&batch, i, "script_version"),
|
||
volatile_flag: col_bool(&batch, i, "volatile_flag"),
|
||
is_deleted: col_bool(&batch, i, "is_deleted"),
|
||
delete_reason: col_str(&batch, i, "delete_reason"),
|
||
depends_on: String::new(),
|
||
derived_from: String::new(),
|
||
source_events: col_str(&batch, i, "source_events"), // 2026-10-13 P1
|
||
supersedes: col_str(&batch, i, "supersedes"), // 2026-10-13 P1
|
||
superseded_by: col_str(&batch, i, "superseded_by"), // 2026-10-14 t_3e98b65a
|
||
valid_from: col_str(&batch, i, "valid_from"), // 2026-10-14 t_3ada332b P3
|
||
valid_to: col_str(&batch, i, "valid_to"), // 2026-10-14 t_3ada332b P3
|
||
provenance: col_str(&batch, i, "provenance"), // 2026-10-14 t_3ada332b P3
|
||
last_recalled_at: String::new(),
|
||
created_at: col_str(&batch, i, "created_at"),
|
||
updated_at: col_str(&batch, i, "updated_at"),
|
||
// 2026-10-08 (Gap#2):读侧接线。col_f64_opt 保留 NULL 语义
|
||
confidence: col_f64_opt(&batch, i, "confidence"),
|
||
confidence_source: col_str(&batch, i, "confidence_source"),
|
||
});
|
||
}
|
||
}
|
||
eprintln!("[lancedb] search top_k={} → {} results", top_k, records.len());
|
||
Ok(records)
|
||
}
|
||
|
||
/// 查询记忆(支持 min_recall_count 过滤),供 GetSkillCandidates 使用
|
||
pub fn query_memories(&self, min_recall: i64, limit: usize) -> Result<Vec<MemoryRecord>, Box<dyn std::error::Error>> {
|
||
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
|
||
let tbl = rt().block_on(db.open_table("memories").execute())?;
|
||
|
||
// GA-P0-2 (t_c6d41ac6):技能候选 = 观测/选择(被召回多次)→ 用 recall_seen_count。
|
||
// recall_count 已冻结,不再增长,不能再当选择依据。
|
||
let filter = format!("recall_seen_count >= {} AND is_deleted = false AND tier != 'core'", min_recall);
|
||
let mut results = Box::pin(rt().block_on(
|
||
tbl.query()
|
||
.only_if(&filter)
|
||
.limit(limit)
|
||
.execute()
|
||
)?);
|
||
|
||
let mut records = Vec::new();
|
||
while let Some(Ok(batch)) = rt().block_on(results.next()) {
|
||
for i in 0..batch.num_rows() {
|
||
records.push(MemoryRecord {
|
||
id: col_str(&batch, i, "id"),
|
||
agent_id: col_str(&batch, i, "agent_id"),
|
||
namespace: col_str(&batch, i, "namespace"),
|
||
content: col_str(&batch, i, "content"),
|
||
category: col_str(&batch, i, "category"),
|
||
vector: col_vector(&batch, i, "vector"),
|
||
tier: col_str(&batch, i, "tier"),
|
||
importance: col_f64(&batch, i, "importance"),
|
||
quality_score: col_f64(&batch, i, "quality_score"),
|
||
recall_count: col_i64(&batch, i, "recall_count"),
|
||
// GA-P0-2 (t_c6d41ac6):拆分字段读侧接线
|
||
recall_seen_count: col_i64(&batch, i, "recall_seen_count"),
|
||
recall_verified_count: col_i64(&batch, i, "recall_verified_count"),
|
||
useful_count: col_i64(&batch, i, "useful_count"),
|
||
not_useful_count: col_i64(&batch, i, "not_useful_count"),
|
||
// GA-P0-1 (t_718d8707):拆分字段读侧接线
|
||
human_not_useful_count: col_i64(&batch, i, "human_not_useful_count"),
|
||
dedup_downweight_count: col_i64(&batch, i, "dedup_downweight_count"),
|
||
freshness: col_str(&batch, i, "freshness"),
|
||
version: col_i64(&batch, i, "version"),
|
||
version_history: String::new(),
|
||
source: col_str(&batch, i, "source"),
|
||
embedding_model: col_str(&batch, i, "embedding_model"),
|
||
script_version: col_str(&batch, i, "script_version"),
|
||
volatile_flag: col_bool(&batch, i, "volatile_flag"),
|
||
is_deleted: col_bool(&batch, i, "is_deleted"),
|
||
delete_reason: col_str(&batch, i, "delete_reason"),
|
||
depends_on: String::new(),
|
||
derived_from: String::new(),
|
||
source_events: col_str(&batch, i, "source_events"), // 2026-10-13 P1
|
||
supersedes: col_str(&batch, i, "supersedes"), // 2026-10-13 P1
|
||
superseded_by: col_str(&batch, i, "superseded_by"), // 2026-10-14 t_3e98b65a
|
||
valid_from: col_str(&batch, i, "valid_from"), // 2026-10-14 t_3ada332b P3
|
||
valid_to: col_str(&batch, i, "valid_to"), // 2026-10-14 t_3ada332b P3
|
||
provenance: col_str(&batch, i, "provenance"), // 2026-10-14 t_3ada332b P3
|
||
last_recalled_at: String::new(),
|
||
created_at: col_str(&batch, i, "created_at"),
|
||
updated_at: col_str(&batch, i, "updated_at"),
|
||
// 2026-10-08 (Gap#2):读侧接线。col_f64_opt 保留 NULL 语义
|
||
confidence: col_f64_opt(&batch, i, "confidence"),
|
||
confidence_source: col_str(&batch, i, "confidence_source"),
|
||
});
|
||
}
|
||
}
|
||
Ok(records)
|
||
}
|
||
|
||
pub fn insert_batch(&self, table: &str, records_json: &str) -> Result<usize, Box<dyn std::error::Error>> {
|
||
let values: serde_json::Value = serde_json::from_str(records_json)?;
|
||
let records: Vec<serde_json::Value> = values.as_array().ok_or("expected JSON array")?.to_vec();
|
||
let count = records.len();
|
||
if count == 0 {
|
||
return Ok(0);
|
||
}
|
||
|
||
// 构建 Arrow RecordBatch
|
||
let schema = Arc::new(arrow_schema());
|
||
let mut ids = Vec::new(); let mut agent_ids = Vec::new(); let mut namespaces = Vec::new();
|
||
let mut contents = Vec::new(); let mut categories = Vec::new(); let mut vectors = Vec::new();
|
||
let mut tiers = Vec::new(); let mut importances = Vec::new(); let mut quality_scores = Vec::new();
|
||
let mut recall_counts = Vec::new(); let mut recall_seen_counts = Vec::new(); let mut recall_verified_counts = Vec::new();
|
||
let mut useful_counts = Vec::new(); let mut not_useful_counts = Vec::new();
|
||
// GA-P0-1 (t_718d8707):拆分字段
|
||
let mut human_not_useful_counts = Vec::new(); let mut dedup_downweight_counts = Vec::new();
|
||
let mut freshnesses = Vec::new(); let mut versions = Vec::new();
|
||
let mut version_histories = Vec::new(); let mut sources = Vec::new();
|
||
let mut embedding_models = Vec::new(); let mut script_versions = Vec::new();
|
||
let mut volatile_flags = Vec::new(); let mut is_deleteds = Vec::new();
|
||
// GA-P0-3 (t_f785d7f2):删除原因(additive)
|
||
let mut delete_reasons: Vec<&str> = Vec::new();
|
||
let mut depends_ons = Vec::new(); let mut derived_froms = Vec::new();
|
||
// 2026-10-13 (t_e9a4d18f P1) 证据链:全量来源 episode ID(JSON 数组字符串)
|
||
let mut source_events_col = Vec::new();
|
||
let mut supersedes_col = Vec::new();
|
||
let mut superseded_by_col = Vec::new(); // 2026-10-14 t_3e98b65a
|
||
// 2026-10-14 (t_3ada332b P3) bi-temporal 有效期三列
|
||
let mut valid_froms = Vec::new(); let mut valid_tos = Vec::new(); let mut provenances = Vec::new();
|
||
let mut last_recalled_ats = Vec::new(); let mut created_ats = Vec::new(); let mut updated_ats = Vec::new();
|
||
// 2026-10-08 (Gap#2):nullable 列用 Option,未声明时写 None → 库里是 NULL
|
||
let mut confidences: Vec<Option<f64>> = Vec::new();
|
||
let mut confidence_sources: Vec<&str> = Vec::new();
|
||
|
||
for r in &records {
|
||
ids.push(r["id"].as_str().unwrap_or(""));
|
||
agent_ids.push(r["agent_id"].as_str().unwrap_or(""));
|
||
namespaces.push(r["namespace"].as_str().unwrap_or(""));
|
||
contents.push(r["content"].as_str().unwrap_or(""));
|
||
categories.push(r["category"].as_str().unwrap_or(""));
|
||
let v: Vec<f32> = if let Some(arr) = r["vector"].as_array() {
|
||
arr.iter().filter_map(|v| v.as_f64()).map(|v| v as f32).collect()
|
||
} else { vec![0.0_f32; 1024] };
|
||
vectors.push(v);
|
||
tiers.push(r["tier"].as_str().unwrap_or("normal"));
|
||
importances.push(r["importance"].as_f64().unwrap_or(1.0));
|
||
quality_scores.push(r["quality_score"].as_f64().unwrap_or(0.0));
|
||
recall_counts.push(r["recall_count"].as_i64().unwrap_or(0));
|
||
// GA-P0-2 (t_c6d41ac6):拆分字段写入
|
||
recall_seen_counts.push(r["recall_seen_count"].as_i64().unwrap_or(0));
|
||
recall_verified_counts.push(r["recall_verified_count"].as_i64().unwrap_or(0));
|
||
useful_counts.push(r["useful_count"].as_i64().unwrap_or(0));
|
||
not_useful_counts.push(r["not_useful_count"].as_i64().unwrap_or(0));
|
||
human_not_useful_counts.push(r["human_not_useful_count"].as_i64().unwrap_or(0)); // GA-P0-1
|
||
dedup_downweight_counts.push(r["dedup_downweight_count"].as_i64().unwrap_or(0)); // GA-P0-1
|
||
freshnesses.push(r["freshness"].as_str().unwrap_or("fresh"));
|
||
versions.push(r["version"].as_i64().unwrap_or(1));
|
||
version_histories.push(r["version_history"].as_str().unwrap_or("[]"));
|
||
sources.push(r["source"].as_str().unwrap_or(""));
|
||
embedding_models.push(r["embedding_model"].as_str().unwrap_or(""));
|
||
script_versions.push(r["script_version"].as_str().unwrap_or(""));
|
||
volatile_flags.push(r["volatile_flag"].as_bool().unwrap_or(false));
|
||
is_deleteds.push(r["is_deleted"].as_bool().unwrap_or(false));
|
||
delete_reasons.push(r["delete_reason"].as_str().unwrap_or("")); // GA-P0-3
|
||
depends_ons.push(r["depends_on"].as_str().unwrap_or("[]"));
|
||
derived_froms.push(r["derived_from"].as_str().unwrap_or(""));
|
||
source_events_col.push(r["source_events"].as_str().unwrap_or("[]")); // 2026-10-13 P1
|
||
supersedes_col.push(r["supersedes"].as_str().unwrap_or("[]")); // 2026-10-13 P1
|
||
superseded_by_col.push(r["superseded_by"].as_str().unwrap_or("[]")); // 2026-10-14 t_3e98b65a
|
||
valid_froms.push(r["valid_from"].as_str().unwrap_or("")); // 2026-10-14 t_3ada332b P3
|
||
valid_tos.push(r["valid_to"].as_str().unwrap_or("")); // 2026-10-14 t_3ada332b P3
|
||
provenances.push(r["provenance"].as_str().unwrap_or("")); // 2026-10-14 t_3ada332b P3
|
||
last_recalled_ats.push(r["last_recalled_at"].as_str().unwrap_or(""));
|
||
created_ats.push(r["created_at"].as_str().unwrap_or(""));
|
||
updated_ats.push(r["updated_at"].as_str().unwrap_or(""));
|
||
// confidence:显式声明才写,否则 NULL(不从 source 反推)——
|
||
// **NULL 本身就是「未声明」的语义信号**,confidence_source 只是标签。
|
||
confidences.push(r["confidence"].as_f64());
|
||
// confidence_source 恒给字符串(未声明 ⇒ ""):迁移来的列可能是 non-null,
|
||
// 送 None 会被 lance 拒收。语义仍由 confidence=NULL 承担。
|
||
confidence_sources.push(r["confidence_source"].as_str().unwrap_or(""));
|
||
}
|
||
|
||
// 构建 vector 列 (FixedSizeList<Float32, 1024>)
|
||
let flat: Vec<f32> = vectors.iter().flatten().copied().collect();
|
||
let values_array = Float32Array::from_iter_values(flat.iter().cloned());
|
||
let vec_array = FixedSizeListArray::try_new(
|
||
Arc::new(Field::new("item", DataType::Float32, true)),
|
||
1024,
|
||
Arc::new(values_array),
|
||
None,
|
||
)?;
|
||
|
||
let columns: Vec<ArrayRef> = vec![
|
||
Arc::new(StringArray::from(ids)),
|
||
Arc::new(StringArray::from(agent_ids)),
|
||
Arc::new(StringArray::from(namespaces)),
|
||
Arc::new(StringArray::from(contents)),
|
||
Arc::new(StringArray::from(categories)),
|
||
Arc::new(vec_array),
|
||
Arc::new(StringArray::from(tiers)),
|
||
Arc::new(Float64Array::from(importances)),
|
||
Arc::new(Float64Array::from(quality_scores)),
|
||
Arc::new(Int64Array::from(recall_counts)),
|
||
Arc::new(Int64Array::from(useful_counts)),
|
||
Arc::new(Int64Array::from(not_useful_counts)),
|
||
Arc::new(StringArray::from(freshnesses)),
|
||
Arc::new(Int64Array::from(versions)),
|
||
Arc::new(StringArray::from(version_histories)),
|
||
Arc::new(StringArray::from(sources)),
|
||
Arc::new(StringArray::from(embedding_models)),
|
||
Arc::new(StringArray::from(script_versions)),
|
||
Arc::new(BooleanArray::from(volatile_flags)),
|
||
Arc::new(BooleanArray::from(is_deleteds)),
|
||
Arc::new(StringArray::from(depends_ons)),
|
||
Arc::new(StringArray::from(derived_froms)),
|
||
Arc::new(StringArray::from(last_recalled_ats)),
|
||
Arc::new(StringArray::from(created_ats)),
|
||
Arc::new(StringArray::from(updated_ats)),
|
||
// 2026-10-08 (Gap#2):顺序必须与 arrow_schema() 严格一致
|
||
Arc::new(Float64Array::from(confidences)),
|
||
Arc::new(StringArray::from(confidence_sources)),
|
||
// GA-P0-1 (t_718d8707):顺序必须与 arrow_schema() 尾部新增列严格一致
|
||
Arc::new(Int64Array::from(human_not_useful_counts)),
|
||
Arc::new(Int64Array::from(dedup_downweight_counts)),
|
||
// GA-P0-2 (t_c6d41ac6):拆分字段列追加在末尾,与 arrow_schema 顺序一致
|
||
Arc::new(Int64Array::from(recall_seen_counts)),
|
||
Arc::new(Int64Array::from(recall_verified_counts)),
|
||
// GA-P0-3 (t_f785d7f2):delete_reason 列,与 arrow_schema 顺序一致(additive)
|
||
Arc::new(StringArray::from(delete_reasons)),
|
||
// 2026-10-13 (t_e9a4d18f P1):source_events 列,与 arrow_schema 顺序一致(additive)
|
||
Arc::new(StringArray::from(source_events_col)),
|
||
// 2026-10-13 (t_e9a4d18f P1):supersedes 列,与 arrow_schema 顺序一致(additive)
|
||
Arc::new(StringArray::from(supersedes_col)),
|
||
// 2026-10-14 (t_3e98b65a):superseded_by 列,与 arrow_schema 顺序一致(additive)
|
||
Arc::new(StringArray::from(superseded_by_col)),
|
||
// 2026-10-14 (t_3ada332b P3):bi-temporal 三列,与 arrow_schema 顺序一致(additive)
|
||
Arc::new(StringArray::from(valid_froms)),
|
||
Arc::new(StringArray::from(valid_tos)),
|
||
Arc::new(StringArray::from(provenances)),
|
||
];
|
||
|
||
let batch = RecordBatch::try_new(schema.clone(), columns)?;
|
||
|
||
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
|
||
let tbl = rt().block_on(db.open_table(table).execute())?;
|
||
let batches = RecordBatchIterator::new(
|
||
vec![batch].into_iter().map(Ok),
|
||
Arc::clone(&schema),
|
||
);
|
||
rt().block_on(tbl.add(Box::new(batches)).execute())?;
|
||
|
||
eprintln!("[lancedb] {}: inserted {} records", table, count);
|
||
Ok(count)
|
||
}
|
||
|
||
/// 更新记忆记录的字段(通过 SQL UPDATE)
|
||
/// fields_json 格式: [{"column":"recall_count","value":"recall_count + 1"}, ...]
|
||
/// 支持算术表达式(value 包含 + 或 - 时),先查当前值再计算
|
||
pub fn update(&self, table: &str, id: &str, fields_json: &str) -> Result<u64, Box<dyn std::error::Error>> {
|
||
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
|
||
let tbl = rt().block_on(db.open_table(table).execute())?;
|
||
let fields: Vec<serde_json::Value> = serde_json::from_str(fields_json)?;
|
||
let mut op = tbl.update().only_if(format!("id = '{}'", id.replace("\\'", "''")));
|
||
|
||
// 如果有算术表达式,先查当前值
|
||
let has_arith = fields.iter().any(|f| {
|
||
f["value"].as_str().map(|v| v.contains('+') || v.contains('-')).unwrap_or(false)
|
||
});
|
||
|
||
let mut current_values: std::collections::HashMap<String, i64> = std::collections::HashMap::new();
|
||
if has_arith {
|
||
// 查询当前值
|
||
let mut results = Box::pin(rt().block_on(
|
||
tbl.query()
|
||
.only_if(&format!("id = '{}'", id.replace("\\'", "''")))
|
||
.limit(1)
|
||
.execute(),
|
||
)?);
|
||
while let Some(Ok(batch)) = rt().block_on(results.next()) {
|
||
for i in 0..batch.num_rows() {
|
||
for f in &fields {
|
||
let col = f["column"].as_str().unwrap_or("");
|
||
if !col.is_empty() {
|
||
let val = col_i64(&batch, i, col);
|
||
current_values.insert(col.to_string(), val);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
for f in &fields {
|
||
let col = f["column"].as_str().unwrap_or("");
|
||
let val = f["value"].as_str().unwrap_or("");
|
||
if col.is_empty() { continue; }
|
||
|
||
// 处理算术表达式: "column + N" 或 "column - N"
|
||
let final_val = if (val.contains('+') || val.contains('-')) && !val.starts_with("\\'") {
|
||
let parts: Vec<&str> = val.split_whitespace().collect();
|
||
if parts.len() == 3 {
|
||
let operand_col = parts[0];
|
||
let op_char = parts[1].chars().next().unwrap_or('+');
|
||
if let Ok(delta) = parts[2].parse::<i64>() {
|
||
let current = *current_values.get(operand_col).unwrap_or(&0);
|
||
let new_val = match op_char {
|
||
'+' => current.saturating_add(delta),
|
||
'-' => current.saturating_sub(delta),
|
||
_ => current,
|
||
};
|
||
eprintln!("[lancedb] update {}: col={} expr={} current={} -> {}", id, col, val, current, new_val);
|
||
new_val.to_string()
|
||
} else {
|
||
val.to_string()
|
||
}
|
||
} else {
|
||
val.to_string()
|
||
}
|
||
} else {
|
||
val.to_string()
|
||
};
|
||
|
||
eprintln!("[lancedb] UPDATE col={} val='{}'", col, final_val);
|
||
op = op.column(col, &final_val);
|
||
}
|
||
let updated = rt().block_on(op.execute())?;
|
||
eprintln!("[lancedb] UPDATE DONE: id={} fields={} rows={}", id, fields.len(), updated);
|
||
Ok(updated)
|
||
}
|
||
|
||
/// 累积召回批量更新(2026-09-12 优化B)。
|
||
///
|
||
/// 背景:recall 读路径旧实现对每条结果调一次 update,而 LanceDB 是 MVCC——
|
||
/// **每次 update 提交 = 一个版本**,实测 15 版本/分钟 → _versions 膨胀到 17.7G。
|
||
///
|
||
/// 做法(实测约束:lance 的 update 只支持基础 SQL 表达式,`CASE WHEN` 会被
|
||
/// planner 拒绝 —— 见 2026-09-12 实测 `Expression 'CASE WHEN ...' is not supported SQL in lance`):
|
||
/// 把整批**按 delta 分组**,每组用一次 `recall_count + delta` 表达式 + `id IN (...)` 谓词提交。
|
||
/// 真实召回流里 delta 绝大多数是 1(同一窗口内多次命中才 >1),所以
|
||
/// N 条记忆的批量 ≈ **1 个版本**(旧实现:N 个版本)。
|
||
///
|
||
/// items_json: [{"id":"mem_xxx","delta":2}],ts: RFC3339(last_recalled_at)
|
||
pub fn update_recall_batch(&self, items_json: &str, ts: &str) -> Result<u64, Box<dyn std::error::Error>> {
|
||
#[derive(serde::Deserialize)]
|
||
struct Item {
|
||
id: String,
|
||
delta: i64,
|
||
}
|
||
let items: Vec<Item> = serde_json::from_str(items_json)?;
|
||
if items.is_empty() {
|
||
return Ok(0);
|
||
}
|
||
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
|
||
let tbl = rt().block_on(db.open_table("memories").execute())?;
|
||
|
||
let esc = |s: &str| s.replace("\\'", "''");
|
||
let ts_lit = format!("'{}'", esc(ts));
|
||
|
||
// 按 delta 分组(BTreeMap 保证顺序稳定,便于日志/排障)
|
||
let mut groups: std::collections::BTreeMap<i64, Vec<String>> = std::collections::BTreeMap::new();
|
||
for it in &items {
|
||
groups.entry(it.delta).or_default().push(it.id.clone());
|
||
}
|
||
|
||
let mut total: u64 = 0;
|
||
for (delta, ids) in &groups {
|
||
let predicate = format!(
|
||
"id IN ({})",
|
||
ids.iter().map(|id| format!("'{}'", esc(id))).collect::<Vec<_>>().join(",")
|
||
);
|
||
let op = tbl
|
||
.update()
|
||
.only_if(&predicate)
|
||
.column("recall_seen_count", &format!("recall_seen_count + {}", delta))
|
||
.column("last_recalled_at", &ts_lit);
|
||
// GA-P0-2 (t_c6d41ac6):召回命中写 recall_seen_count(出现即 ++),
|
||
// 不再写 recall_count(冻结)。recall_count 只保留历史值,不再增长。
|
||
// t_268b0353(治根):召回只更新 seen/last_recalled_at,
|
||
// 不再写 freshness='verified'。freshness=verified 语义是「经过校验/用户
|
||
// 反馈」;「被召回」≠「被验证」。此前每次召回都盖 verified → 越常被召回
|
||
// 的旧快照越「可信」,时效判断全被污染(过期快照被当现状的机制根因之一)。
|
||
// 这是主写路径(zhiyi-consolidate sidecar)——Go 侧 write_buffer 只是兜底。
|
||
match rt().block_on(op.execute()) {
|
||
Ok(n) => {
|
||
eprintln!(
|
||
"[lancedb] BATCH UPDATE recall DONE: delta={} ids={} rows={} (单事务)",
|
||
delta,
|
||
ids.len(),
|
||
n
|
||
);
|
||
total += n;
|
||
}
|
||
Err(e) => {
|
||
// 兜底:谓词/表达式被拒时逐条更新(版本数退化为 N,但数据不丢、计数正确)
|
||
eprintln!(
|
||
"[lancedb] BATCH UPDATE(delta={}) failed ({}), fallback per-item ({} ids)",
|
||
delta,
|
||
e,
|
||
ids.len()
|
||
);
|
||
for id in ids {
|
||
// GA-P0-2 (t_c6d41ac6):召回命中写 recall_seen_count;fallback 同主路径。
|
||
let fields = format!(
|
||
r#"[{{"column":"recall_seen_count","value":"recall_seen_count + {}"}},{{"column":"last_recalled_at","value":"{}"}}]"#,
|
||
delta, ts_lit
|
||
);
|
||
match self.update("memories", id, &fields) {
|
||
Ok(n) => total += n,
|
||
Err(e2) => eprintln!("[lancedb] fallback update {} failed: {}", id, e2),
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
Ok(total)
|
||
}
|
||
|
||
/// 累积软删除批量更新(2026-09-12 优化C)。
|
||
///
|
||
/// 背景:软删除操作(SoftDelete)原先对**每条记忆**同步调用 lancedb.Update
|
||
/// (is_deleted = true + updated_at)。每次 update 提交产生一个版本。
|
||
/// 实测生产:碎片快速道和 decay 触发产生大量 SoftDelete,导致 _versions 膨胀。
|
||
///
|
||
/// 方案:
|
||
///
|
||
/// Record() 只把增量写进内存缓冲;**同一批次在一个窗口内多次 SoftDelete 合并为 一次批量提交**
|
||
/// Flush() 后台按窗口(默认 300s)+ 阈值(默认 256 条不同记忆)批量提交一次;
|
||
/// Rust 侧 lancedb_soft_delete_batch 把整批**设置相同字段**(is_deleted=true + updated_at=ts)一次提交;
|
||
/// → 每批版本数 = 1(旧实现:每行 1 个版本)。
|
||
///
|
||
/// 语义取舍(明确记录,便于日后审计):
|
||
/// - updated_at 最多延迟一个窗口(分钟级)。遗忘/衰减判定以「天」为单位,无影响。
|
||
/// - 进程崩溃会丢最后一个窗口的增量(软删除标记,不是记忆数据本身),可接受。
|
||
///
|
||
/// items_json: [{"id":"mem_xxx"}],ts: RFC3339(updated_at)
|
||
pub fn soft_delete_batch(&self, items_json: &str, ts: &str) -> Result<u64, Box<dyn std::error::Error>> {
|
||
#[derive(serde::Deserialize)]
|
||
struct Item {
|
||
id: String,
|
||
}
|
||
let items: Vec<Item> = serde_json::from_str(items_json)?;
|
||
if items.is_empty() {
|
||
return Ok(0);
|
||
}
|
||
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
|
||
let tbl = rt().block_on(db.open_table("memories").execute())?;
|
||
|
||
let esc = |s: &str| s.replace("\\'", "''");
|
||
let ts_lit = format!("'{}'", esc(ts));
|
||
|
||
// 按 id 分组(其实每个 id 只出现一次,但为了统一接口仍保持分组逻辑)
|
||
let mut groups: std::collections::BTreeMap<String, Vec<String>> = std::collections::BTreeMap::new();
|
||
for it in &items {
|
||
groups.entry(it.id.clone()).or_default().push(it.id.clone());
|
||
}
|
||
|
||
let mut total: u64 = 0;
|
||
for (_key, ids) in &groups {
|
||
let predicate = format!(
|
||
"id IN ({})",
|
||
ids.iter().map(|id| format!("'{}'", esc(id))).collect::<Vec<_>>().join(",")
|
||
);
|
||
let op = tbl
|
||
.update()
|
||
.only_if(&predicate)
|
||
.column("is_deleted", "true")
|
||
// GA-P0-3 (t_f785d7f2):批量软删不携带逐条原因 ⇒ delete_reason=unknown(兜底)。
|
||
.column("delete_reason", "'unknown'")
|
||
.column("updated_at", &ts_lit);
|
||
match rt().block_on(op.execute()) {
|
||
Ok(n) => {
|
||
eprintln!(
|
||
"[lancedb] BATCH SOFT DELETE DONE: ids={} rows={} (单事务)",
|
||
ids.len(),
|
||
n
|
||
);
|
||
total += n;
|
||
}
|
||
Err(e) => {
|
||
// 兜底:谓词/表达式被拒时逐条更新(版本数退化为 N,但数据不丢)
|
||
eprintln!(
|
||
"[lancedb] BATCH SOFT DELETE failed ({}), fallback per-item ({} ids)",
|
||
e,
|
||
ids.len()
|
||
);
|
||
for id in ids {
|
||
let fields = format!(
|
||
r#"[{{"column":"is_deleted","value":"true"}},{{"column":"updated_at","value":"{}"}}]"#,
|
||
ts_lit
|
||
);
|
||
match self.update("memories", id, &fields) {
|
||
Ok(n) => total += n,
|
||
Err(e2) => eprintln!("[lancedb] fallback soft delete {} failed: {}", id, e2),
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
Ok(total)
|
||
}
|
||
|
||
/// 清理 id 为空的垃圾行(P1-1, 2026-10-08)。
|
||
///
|
||
/// 背景:DB 存在 id/content/source 全空的垃圾行 ⇒ 命中「<30 字碎片快速道」
|
||
/// ⇒ SoftDelete("") 被 Rust IPC 拒绝("id required")⇒ 每轮刷错误日志 + 遗忘计数虚高。
|
||
/// 这些行无 id 无法按 id 软删,只能按「id 为空」条件一次性软删(软删非硬删,is_deleted=true)。
|
||
///
|
||
/// 做法:一次 update 提交把 id 为空的存活行标记 is_deleted=true + updated_at=ts。
|
||
/// 只匹配精确空串(id = '');不匹配纯空白(LanceDB SQL 的 trim 支持未验证,
|
||
/// 宁缺勿滥 —— 防御/计数代码已能安全跳过任何残留空 id,不会刷日志)。
|
||
/// ts 由 Go 侧传入(与 soft_delete_batch 同构,避免 Rust 侧依赖时钟库)。
|
||
pub fn purge_invalid(&self, ts: &str) -> Result<u64, Box<dyn std::error::Error>> {
|
||
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
|
||
let tbl = rt().block_on(db.open_table("memories").execute())?;
|
||
|
||
let esc = |s: &str| s.replace("\\'", "''");
|
||
let ts_lit = format!("'{}'", esc(ts));
|
||
let op = tbl
|
||
.update()
|
||
.only_if("id = ''")
|
||
.column("is_deleted", "true")
|
||
.column("delete_reason", "'retention'") // GA-P0-3:id 为空垃圾行 = 保留策略清理
|
||
.column("updated_at", &ts_lit);
|
||
let updated = rt().block_on(op.execute())?;
|
||
eprintln!("[lancedb] PURGE INVALID DONE: rows={}", updated);
|
||
Ok(updated)
|
||
}
|
||
|
||
/// 全量扫描 memories 表(用于深整),上限 10000 条
|
||
// P2 2026-09-06 安全版遗忘候选全表扫描(替代 febc2c9 风暴版):
|
||
// 1. 强制 limit(调用方传, main.rs 钳制硬上限 5000)
|
||
// 2. 不读 vector 列(1024 维 × 3000 条 ≈ 12MB+,IPC JSON 会卡)
|
||
// 3. 真读 last_recalled_at 列(scan_all 旧实现 String::new() 恒空 → 遗忘判定失真)
|
||
/// 全表扫描存活记忆(过滤 is_deleted,跳过 vector)。
|
||
/// ⚠️ 名字来自历史用途(遗忘候选),语义实为「列出存活记忆」。
|
||
/// 2026-09-21:新增 namespace 下推过滤 —— 修前 Go 侧 /memories 与 /memories/export
|
||
/// 用「零向量 ANN」冒充列表,零向量无方向 ⇒ 返回任意前 N 行再按 ns 过滤,
|
||
/// 计数完全失真(实测 hermes-main 报 1 条,实际 139+)。
|
||
/// 用 only_if 下推过滤,limit 作用于**过滤后**的结果集,避免 5000 硬上限截断。
|
||
pub fn scan_for_forgetting(
|
||
&self,
|
||
limit: usize,
|
||
namespace: Option<&str>,
|
||
) -> Result<Vec<MemoryRecord>, Box<dyn std::error::Error>> {
|
||
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
|
||
let tbl = rt().block_on(db.open_table("memories").execute())?;
|
||
|
||
let mut filter = String::from("is_deleted = false");
|
||
if let Some(ns) = namespace {
|
||
if !ns.is_empty() {
|
||
let escaped = ns.replace('\\', "\\\\").replace('\'', "''");
|
||
filter.push_str(&format!(" AND namespace = '{}'", escaped));
|
||
}
|
||
}
|
||
|
||
let mut results = Box::pin(rt().block_on(
|
||
tbl.query()
|
||
.only_if(&filter)
|
||
.limit(limit)
|
||
.execute()
|
||
)?);
|
||
|
||
let mut records = Vec::new();
|
||
while let Some(Ok(batch)) = rt().block_on(results.next()) {
|
||
for i in 0..batch.num_rows() {
|
||
records.push(Self::record_from_batch(&batch, i));
|
||
}
|
||
}
|
||
eprintln!("[lancedb] scan_forgetting → {} records (limit={}, ns={:?})", records.len(), limit, namespace);
|
||
Ok(records)
|
||
}
|
||
|
||
/// 遗忘候选「旋转窗口」扫描(2026-10-11 t_a4b8fa52 方案 c′)。
|
||
///
|
||
/// 修复「固定 2000 窗口不轮转」:LanceDB 0.15 查询**无 order_by**,`.limit(N)`
|
||
/// 返回的是**物理存储序**前 N 条(compact/merge 后重排,非创建序)⇒ 新记忆落物理序
|
||
/// 后部 ⇒ 永不进遗忘候选池。方案 c′ 不依赖任何排序下推,只靠「列投影 + 集合成员」:
|
||
/// 1. 只读 id 列取**全部存活 id**:`select(["id"])` + `only_if("is_deleted = false")`
|
||
/// 2. Rust 内按字符串排序(id = mem_<unixnano>,定宽 ⇒ 字符串序 == 创建序严格单调)
|
||
/// 3. 取**连续窗口** `ids[start : start+limit]`,start 由游标 after_id 定位
|
||
/// 4. 区间过滤把窗口还原成**全字段记录**:`only_if("is_deleted = false AND id >= lo AND id <= hi")`
|
||
/// 5. 新游标 = hi(窗口末 id);窗口不足 limit(取到末尾)⇒ 回绕,下次从 0 起
|
||
/// 游标持久化在 Go 侧(/var/lib/memoryweave/forget-cursor.json,复用 conflict-seen 原子写),
|
||
/// Rust 无状态、只接收 after_id。
|
||
pub fn scan_forgetting_window(
|
||
&self,
|
||
limit: usize,
|
||
after_id: Option<&str>,
|
||
) -> Result<ForgetWindow, Box<dyn std::error::Error>> {
|
||
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
|
||
let tbl = rt().block_on(db.open_table("memories").execute())?;
|
||
|
||
// ── 1. 列投影:只读 id,取全部存活 id ──
|
||
// ⚠️ LanceDB 0.15 query() 默认 limit=10 —— 必须显式给大上限,否则只取 10 条
|
||
// (spike 实测:无 limit ⇒ live_total=10;加 limit(200_000) ⇒ 22398)
|
||
let mut ids: Vec<String> = Vec::new();
|
||
{
|
||
let mut it = Box::pin(rt().block_on(
|
||
tbl.query()
|
||
.only_if("is_deleted = false")
|
||
.select(Select::columns(&["id"]))
|
||
.limit(200_000)
|
||
.execute()
|
||
)?);
|
||
while let Some(Ok(batch)) = rt().block_on(it.next()) {
|
||
for i in 0..batch.num_rows() {
|
||
let id = col_str(&batch, i, "id");
|
||
// 只保留 mem_ 形态(排除 episode `ep_` 行 —— 本卡审定:episode 原始轮次不参与遗忘)
|
||
if id.starts_with("mem_") {
|
||
ids.push(id);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
let live_total = ids.len();
|
||
|
||
// ── 2. 排序 + 去重断言(AC-9:过滤后 id 必须唯一,防未来重复 id 静默跳行)──
|
||
ids.sort();
|
||
let before_dedup = ids.len();
|
||
ids.dedup();
|
||
if ids.len() != before_dedup {
|
||
eprintln!(
|
||
"[lancedb] WARN: forget window 过滤后出现 {} 条重复 id(已去重,不应发生)",
|
||
before_dedup - ids.len()
|
||
);
|
||
}
|
||
|
||
// ── 3. 游标定位窗口起点(纯逻辑,见 compute_forget_window)──
|
||
let (start, end, wrapped) = compute_forget_window(&ids, limit, after_id);
|
||
let window = &ids[start..end];
|
||
|
||
if window.is_empty() {
|
||
return Ok(ForgetWindow {
|
||
records: Vec::new(),
|
||
next_cursor: String::new(),
|
||
wrapped: true,
|
||
live_total,
|
||
});
|
||
}
|
||
let lo = &window[0];
|
||
let hi = &window[window.len() - 1];
|
||
let esc = |s: &str| s.replace('\'', "''");
|
||
let interval = format!(
|
||
"is_deleted = false AND id >= '{}' AND id <= '{}'",
|
||
esc(lo),
|
||
esc(hi)
|
||
);
|
||
|
||
// ── 4. 区间过滤还原全字段记录 ──
|
||
// ⚠️ 同样必须显式 limit(LanceDB 默认 10),否则窗口被截断
|
||
let mut records: Vec<MemoryRecord> = Vec::new();
|
||
{
|
||
let mut it = Box::pin(rt().block_on(
|
||
tbl.query()
|
||
.only_if(&interval)
|
||
.limit(200_000)
|
||
.execute()
|
||
)?);
|
||
while let Some(Ok(batch)) = rt().block_on(it.next()) {
|
||
for i in 0..batch.num_rows() {
|
||
records.push(Self::record_from_batch(&batch, i));
|
||
}
|
||
}
|
||
}
|
||
// 还原记录按 id 排序,与窗口切片一致(确定性输出)
|
||
records.sort_by(|a, b| a.id.cmp(&b.id));
|
||
|
||
let next_cursor = if wrapped { String::new() } else { hi.clone() };
|
||
eprintln!(
|
||
"[lancedb] scan_forgetting_window: limit={} after={:?} window=[{}-{}] n={} live_total={} wrapped={}",
|
||
limit, after_id, lo, hi, records.len(), live_total, wrapped
|
||
);
|
||
Ok(ForgetWindow {
|
||
records,
|
||
next_cursor,
|
||
wrapped,
|
||
live_total,
|
||
})
|
||
}
|
||
|
||
/// 从 RecordBatch 的一行构造 MemoryRecord(scan_forgetting / scan_forgetting_window 共用)。
|
||
fn record_from_batch(batch: &RecordBatch, i: usize) -> MemoryRecord {
|
||
MemoryRecord {
|
||
id: col_str(batch, i, "id"),
|
||
agent_id: col_str(batch, i, "agent_id"),
|
||
namespace: col_str(batch, i, "namespace"),
|
||
content: col_str(batch, i, "content"),
|
||
category: col_str(batch, i, "category"),
|
||
vector: Vec::new(), // 不读 vector — 遗忘候选不需要向量
|
||
tier: col_str(batch, i, "tier"),
|
||
importance: col_f64(batch, i, "importance"),
|
||
quality_score: col_f64(batch, i, "quality_score"),
|
||
recall_count: col_i64(batch, i, "recall_count"),
|
||
// GA-P0-2 (t_c6d41ac6):拆分字段读侧接线
|
||
recall_seen_count: col_i64(batch, i, "recall_seen_count"),
|
||
recall_verified_count: col_i64(batch, i, "recall_verified_count"),
|
||
useful_count: col_i64(batch, i, "useful_count"),
|
||
not_useful_count: col_i64(batch, i, "not_useful_count"),
|
||
human_not_useful_count: col_i64(batch, i, "human_not_useful_count"),
|
||
dedup_downweight_count: col_i64(batch, i, "dedup_downweight_count"),
|
||
freshness: col_str(batch, i, "freshness"),
|
||
version: col_i64(batch, i, "version"),
|
||
version_history: String::new(),
|
||
source: col_str(batch, i, "source"),
|
||
embedding_model: col_str(batch, i, "embedding_model"),
|
||
script_version: col_str(batch, i, "script_version"),
|
||
volatile_flag: col_bool(batch, i, "volatile_flag"),
|
||
is_deleted: col_bool(batch, i, "is_deleted"),
|
||
delete_reason: col_str(batch, i, "delete_reason"),
|
||
depends_on: String::new(),
|
||
derived_from: String::new(),
|
||
source_events: col_str(batch, i, "source_events"), // 2026-10-13 P1
|
||
supersedes: col_str(batch, i, "supersedes"), // 2026-10-13 P1
|
||
superseded_by: col_str(batch, i, "superseded_by"), // 2026-10-14 t_3e98b65a
|
||
valid_from: col_str(batch, i, "valid_from"), // 2026-10-14 t_3ada332b P3
|
||
valid_to: col_str(batch, i, "valid_to"), // 2026-10-14 t_3ada332b P3
|
||
provenance: col_str(batch, i, "provenance"), // 2026-10-14 t_3ada332b P3
|
||
last_recalled_at: col_str(batch, i, "last_recalled_at"),
|
||
created_at: col_str(batch, i, "created_at"),
|
||
updated_at: col_str(batch, i, "updated_at"),
|
||
confidence: col_f64_opt(batch, i, "confidence"),
|
||
confidence_source: col_str(batch, i, "confidence_source"),
|
||
}
|
||
}
|
||
|
||
/// 全表扫描(元数据;2026-09-21 治本改造)
|
||
/// ① 用 Select 投影跳过 vector 列(1024 维 = 4KB/条,Step 3/4 都不需要)
|
||
/// ② 上限从 10000 提到 200000(原值会静默丢记录:库已有 10065 条)
|
||
pub fn scan_all(&self) -> Result<Vec<MemoryRecord>, Box<dyn std::error::Error>> {
|
||
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
|
||
let tbl = rt().block_on(db.open_table("memories").execute())?;
|
||
|
||
const META_COLUMNS: &[&str] = &[
|
||
"id", "agent_id", "namespace", "content", "category",
|
||
"tier", "importance", "quality_score", "recall_count",
|
||
"useful_count", "not_useful_count", "freshness", "version",
|
||
"source", "volatile_flag", "is_deleted",
|
||
// GA-P0-1 (t_718d8707):scan_all 投影裁剪,漏加 ⇒ col_i64 取不到列 ⇒ 恒 0
|
||
"human_not_useful_count", "dedup_downweight_count",
|
||
// GA-P0-2 (t_c6d41ac6):拆分字段投影,漏加 ⇒ col_i64 取不到列 ⇒ 恒 0
|
||
"recall_seen_count", "recall_verified_count",
|
||
"last_recalled_at", "created_at", "updated_at",
|
||
// 2026-10-08 (Gap#2):scan_all 用投影裁剪,漏加 ⇒ col_f64_opt 取不到列 ⇒ 恒 None
|
||
"confidence", "confidence_source",
|
||
];
|
||
|
||
let mut results = Box::pin(rt().block_on(
|
||
tbl.query()
|
||
.only_if("is_deleted = false")
|
||
.select(Select::columns(META_COLUMNS))
|
||
.limit(200_000)
|
||
.execute()
|
||
)?);
|
||
|
||
let mut records = Vec::new();
|
||
while let Some(Ok(batch)) = rt().block_on(results.next()) {
|
||
for i in 0..batch.num_rows() {
|
||
records.push(MemoryRecord {
|
||
id: col_str(&batch, i, "id"),
|
||
agent_id: col_str(&batch, i, "agent_id"),
|
||
namespace: col_str(&batch, i, "namespace"),
|
||
content: col_str(&batch, i, "content"),
|
||
category: col_str(&batch, i, "category"),
|
||
vector: Vec::new(), // 投影扫描:不读 vector(Step 3/4 不需要,省 4KB/条)
|
||
tier: col_str(&batch, i, "tier"),
|
||
importance: col_f64(&batch, i, "importance"),
|
||
quality_score: col_f64(&batch, i, "quality_score"),
|
||
recall_count: col_i64(&batch, i, "recall_count"),
|
||
// GA-P0-2 (t_c6d41ac6):拆分字段读侧接线
|
||
recall_seen_count: col_i64(&batch, i, "recall_seen_count"),
|
||
recall_verified_count: col_i64(&batch, i, "recall_verified_count"),
|
||
useful_count: col_i64(&batch, i, "useful_count"),
|
||
not_useful_count: col_i64(&batch, i, "not_useful_count"),
|
||
// GA-P0-1 (t_718d8707):拆分字段读侧接线
|
||
human_not_useful_count: col_i64(&batch, i, "human_not_useful_count"),
|
||
dedup_downweight_count: col_i64(&batch, i, "dedup_downweight_count"),
|
||
freshness: col_str(&batch, i, "freshness"),
|
||
version: col_i64(&batch, i, "version"),
|
||
version_history: String::new(),
|
||
source: col_str(&batch, i, "source"),
|
||
embedding_model: col_str(&batch, i, "embedding_model"),
|
||
script_version: col_str(&batch, i, "script_version"),
|
||
volatile_flag: col_bool(&batch, i, "volatile_flag"),
|
||
is_deleted: col_bool(&batch, i, "is_deleted"),
|
||
delete_reason: col_str(&batch, i, "delete_reason"),
|
||
depends_on: String::new(),
|
||
derived_from: String::new(),
|
||
source_events: col_str(&batch, i, "source_events"), // 2026-10-13 P1
|
||
supersedes: col_str(&batch, i, "supersedes"), // 2026-10-13 P1
|
||
superseded_by: col_str(&batch, i, "superseded_by"), // 2026-10-14 t_3e98b65a
|
||
valid_from: col_str(&batch, i, "valid_from"), // 2026-10-14 t_3ada332b P3
|
||
valid_to: col_str(&batch, i, "valid_to"), // 2026-10-14 t_3ada332b P3
|
||
provenance: col_str(&batch, i, "provenance"), // 2026-10-14 t_3ada332b P3
|
||
last_recalled_at: String::new(),
|
||
created_at: col_str(&batch, i, "created_at"),
|
||
updated_at: col_str(&batch, i, "updated_at"),
|
||
// 2026-10-08 (Gap#2):读侧接线。col_f64_opt 保留 NULL 语义
|
||
confidence: col_f64_opt(&batch, i, "confidence"),
|
||
confidence_source: col_str(&batch, i, "confidence_source"),
|
||
});
|
||
}
|
||
}
|
||
eprintln!("[lancedb] scan_all → {} records", records.len());
|
||
Ok(records)
|
||
}
|
||
|
||
/// 全量扫描**已删除**行(is_deleted=true)(GA-P0-3 t_f785d7f2)。
|
||
///
|
||
/// 背景:所有 scan 路径(scan / scan_all / scan_forgetting / scan_forgetting_window)
|
||
/// 都过滤 is_deleted=false —— 存量 11228 条 is_deleted=true 的行**没有任何读路径能看**
|
||
/// (唯独 stats 能 count)。回填脚本需要枚举已删除行以回填 delete_reason 并出直方图。
|
||
///
|
||
/// 只读元数据(跳 vector),上限 200000。带 delete_reason 列(供回填时跳过已回填的)。
|
||
pub fn scan_deleted(&self) -> Result<Vec<MemoryRecord>, Box<dyn std::error::Error>> {
|
||
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
|
||
let tbl = rt().block_on(db.open_table("memories").execute())?;
|
||
|
||
const META_COLUMNS: &[&str] = &[
|
||
"id", "agent_id", "namespace", "content", "category",
|
||
"tier", "importance", "quality_score", "recall_count",
|
||
"useful_count", "not_useful_count", "freshness", "version",
|
||
"source", "volatile_flag", "is_deleted",
|
||
"human_not_useful_count", "dedup_downweight_count",
|
||
"recall_seen_count", "recall_verified_count",
|
||
"delete_reason", // GA-P0-3
|
||
"last_recalled_at", "created_at", "updated_at",
|
||
"confidence", "confidence_source",
|
||
];
|
||
|
||
let mut results = Box::pin(rt().block_on(
|
||
tbl.query()
|
||
.only_if("is_deleted = true")
|
||
.select(Select::columns(META_COLUMNS))
|
||
.limit(200_000)
|
||
.execute()
|
||
)?);
|
||
|
||
let mut records = Vec::new();
|
||
while let Some(Ok(batch)) = rt().block_on(results.next()) {
|
||
for i in 0..batch.num_rows() {
|
||
records.push(MemoryRecord {
|
||
id: col_str(&batch, i, "id"),
|
||
agent_id: col_str(&batch, i, "agent_id"),
|
||
namespace: col_str(&batch, i, "namespace"),
|
||
content: col_str(&batch, i, "content"),
|
||
category: col_str(&batch, i, "category"),
|
||
vector: Vec::new(),
|
||
tier: col_str(&batch, i, "tier"),
|
||
importance: col_f64(&batch, i, "importance"),
|
||
quality_score: col_f64(&batch, i, "quality_score"),
|
||
recall_count: col_i64(&batch, i, "recall_count"),
|
||
recall_seen_count: col_i64(&batch, i, "recall_seen_count"),
|
||
recall_verified_count: col_i64(&batch, i, "recall_verified_count"),
|
||
useful_count: col_i64(&batch, i, "useful_count"),
|
||
not_useful_count: col_i64(&batch, i, "not_useful_count"),
|
||
human_not_useful_count: col_i64(&batch, i, "human_not_useful_count"),
|
||
dedup_downweight_count: col_i64(&batch, i, "dedup_downweight_count"),
|
||
freshness: col_str(&batch, i, "freshness"),
|
||
version: col_i64(&batch, i, "version"),
|
||
version_history: String::new(),
|
||
source: col_str(&batch, i, "source"),
|
||
embedding_model: col_str(&batch, i, "embedding_model"),
|
||
script_version: col_str(&batch, i, "script_version"),
|
||
volatile_flag: col_bool(&batch, i, "volatile_flag"),
|
||
is_deleted: col_bool(&batch, i, "is_deleted"),
|
||
delete_reason: col_str(&batch, i, "delete_reason"),
|
||
depends_on: String::new(),
|
||
derived_from: String::new(),
|
||
source_events: col_str(&batch, i, "source_events"), // 2026-10-13 P1
|
||
supersedes: col_str(&batch, i, "supersedes"), // 2026-10-13 P1
|
||
superseded_by: col_str(&batch, i, "superseded_by"), // 2026-10-14 t_3e98b65a
|
||
valid_from: col_str(&batch, i, "valid_from"), // 2026-10-14 t_3ada332b P3
|
||
valid_to: col_str(&batch, i, "valid_to"), // 2026-10-14 t_3ada332b P3
|
||
provenance: col_str(&batch, i, "provenance"), // 2026-10-14 t_3ada332b P3
|
||
last_recalled_at: col_str(&batch, i, "last_recalled_at"),
|
||
created_at: col_str(&batch, i, "created_at"),
|
||
updated_at: col_str(&batch, i, "updated_at"),
|
||
confidence: col_f64_opt(&batch, i, "confidence"),
|
||
confidence_source: col_str(&batch, i, "confidence_source"),
|
||
});
|
||
}
|
||
}
|
||
eprintln!("[lancedb] scan_deleted → {} records", records.len());
|
||
Ok(records)
|
||
}
|
||
}
|
||
|
||
#[derive(Debug, Serialize)]
|
||
pub struct LanceDBStats {
|
||
pub total_memories: usize,
|
||
pub total_episodes: usize,
|
||
pub tombstone_count: usize,
|
||
pub data_dir: String,
|
||
}
|
||
|
||
impl LanceDBOps {
|
||
/// 追加一条真实 outcome 事件到 hit_events 表(GA-P0-4 t_bb10e8bc)。
|
||
/// append-only:每次真实 outcome 追加一行(带 created_at),**不覆盖式累加**。
|
||
pub fn append_hit_event(&self, memory_id: &str, outcome: &str, reason: &str, source: &str, ts: &str) -> Result<u64, Box<dyn std::error::Error>> {
|
||
let id = hit_event_id(memory_id);
|
||
let schema = Arc::new(hit_event_arrow_schema());
|
||
let columns: Vec<ArrayRef> = vec![
|
||
Arc::new(StringArray::from(vec![id])),
|
||
Arc::new(StringArray::from(vec![memory_id.to_string()])),
|
||
Arc::new(StringArray::from(vec![outcome.to_string()])),
|
||
Arc::new(StringArray::from(vec![reason.to_string()])),
|
||
Arc::new(StringArray::from(vec![source.to_string()])),
|
||
Arc::new(StringArray::from(vec![ts.to_string()])),
|
||
];
|
||
let batch = RecordBatch::try_new(schema.clone(), columns)?;
|
||
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
|
||
let tbl = rt().block_on(db.open_table("hit_events").execute())?;
|
||
let batches = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), Arc::clone(&schema));
|
||
rt().block_on(tbl.add(Box::new(batches)).execute())?;
|
||
eprintln!("[lancedb] hit_events: appended id={} outcome={} reason={}", memory_id, outcome, reason);
|
||
Ok(1)
|
||
}
|
||
|
||
/// 统计某 memory 在某时间窗起点(since, RFC3339)之后的 outcome 事件数(P2 自适应阈值按周窗口用)。
|
||
pub fn count_hit_events(&self, memory_id: &str, outcome: &str, since: &str) -> Result<u64, Box<dyn std::error::Error>> {
|
||
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
|
||
let tbl = rt().block_on(db.open_table("hit_events").execute())?;
|
||
// created_at 恒以 UTC(RFC3339 "Z") 存储(append 侧 chrono::Utc 生成)。
|
||
// since 可能是本地时区("+08:00")——字符串比较跨时区会错位,必须先归一到 UTC。
|
||
let since_utc = chrono::DateTime::parse_from_rfc3339(since)
|
||
.map(|dt| dt.with_timezone(&chrono::Utc).to_rfc3339_opts(chrono::SecondsFormat::Secs, true))
|
||
.unwrap_or_else(|_| since.to_string());
|
||
let filter = format!(
|
||
"memory_id = '{}' AND outcome = '{}' AND created_at >= '{}'",
|
||
sql_escape(memory_id), sql_escape(outcome), sql_escape(&since_utc)
|
||
);
|
||
let mut results = Box::pin(rt().block_on(tbl.query().only_if(&filter).execute())?);
|
||
let mut n: u64 = 0;
|
||
while let Some(Ok(batch)) = rt().block_on(results.next()) {
|
||
n += batch.num_rows() as u64;
|
||
}
|
||
Ok(n)
|
||
}
|
||
|
||
/// 读回某 memory 的事件明细(read-back 取证,非只看函数返回)。
|
||
pub fn get_hit_events(&self, memory_id: &str, limit: usize) -> Result<Vec<HitEventRecord>, Box<dyn std::error::Error>> {
|
||
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
|
||
let tbl = rt().block_on(db.open_table("hit_events").execute())?;
|
||
let filter = format!("memory_id = '{}'", sql_escape(memory_id));
|
||
let mut results = Box::pin(rt().block_on(
|
||
tbl.query().only_if(&filter).limit(limit).execute(),
|
||
)?);
|
||
let mut events = Vec::new();
|
||
while let Some(Ok(batch)) = rt().block_on(results.next()) {
|
||
for i in 0..batch.num_rows() {
|
||
events.push(HitEventRecord {
|
||
id: col_str(&batch, i, "id"),
|
||
memory_id: col_str(&batch, i, "memory_id"),
|
||
outcome: col_str(&batch, i, "outcome"),
|
||
reason: col_str(&batch, i, "reason"),
|
||
source: col_str(&batch, i, "source"),
|
||
created_at: col_str(&batch, i, "created_at"),
|
||
});
|
||
}
|
||
}
|
||
Ok(events)
|
||
}
|
||
|
||
pub fn stats(&self) -> Result<LanceDBStats, Box<dyn std::error::Error>> {
|
||
let db = rt().block_on(lancedb::connect(self.data_dir.to_str().unwrap()).execute())?;
|
||
let tables = rt().block_on(db.table_names().execute())?;
|
||
let mut m = 0usize; let mut e = 0usize; let mut t = 0usize;
|
||
for name in &tables {
|
||
if let Ok(tbl) = rt().block_on(db.open_table(name).execute()) {
|
||
if name == "memories" {
|
||
if let Ok(cnt) = rt().block_on(tbl.count_rows(None)) { m = cnt; }
|
||
if let Ok(tc) = rt().block_on(tbl.count_rows(Some("is_deleted = true".to_string()))) { t = tc; }
|
||
} else if let Ok(cnt) = rt().block_on(tbl.count_rows(None)) {
|
||
match name.as_str() {
|
||
"episodes" => e = cnt,
|
||
_ => {}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
Ok(LanceDBStats {
|
||
total_memories: m, total_episodes: e, tombstone_count: t,
|
||
data_dir: self.data_dir.to_string_lossy().to_string(),
|
||
})
|
||
}
|
||
}
|
||
|
||
// ── Arrow RecordBatch helpers ──
|
||
fn col_str(b: &RecordBatch, r: usize, c: &str) -> String {
|
||
b.column_by_name(c)
|
||
.and_then(|col| col.as_any().downcast_ref::<StringArray>())
|
||
.map(|a| a.value(r).to_string())
|
||
.unwrap_or_default()
|
||
}
|
||
fn col_f64(b: &RecordBatch, r: usize, c: &str) -> f64 {
|
||
b.column_by_name(c)
|
||
.and_then(|col| col.as_any().downcast_ref::<Float64Array>())
|
||
.map(|a| a.value(r))
|
||
.unwrap_or(0.0)
|
||
}
|
||
/// col_f64_opt 2026-10-08 (Gap#2):可空浮点读法。
|
||
/// 与 col_f64 的关键差别:**列不存在**或**该行是 NULL** 都返回 None,而不是 0.0。
|
||
/// 对 confidence 这是必须的 —— 0.0 会被读成「极不信任」,而 NULL 表示「未声明」。
|
||
fn col_f64_opt(b: &RecordBatch, r: usize, c: &str) -> Option<f64> {
|
||
b.column_by_name(c)
|
||
.and_then(|col| col.as_any().downcast_ref::<Float64Array>())
|
||
.and_then(|a| if a.is_null(r) { None } else { Some(a.value(r)) })
|
||
}
|
||
fn col_i64(b: &RecordBatch, r: usize, c: &str) -> i64 {
|
||
b.column_by_name(c)
|
||
.and_then(|col| col.as_any().downcast_ref::<Int64Array>())
|
||
.map(|a| a.value(r))
|
||
.unwrap_or(0)
|
||
}
|
||
fn col_bool(b: &RecordBatch, r: usize, c: &str) -> bool {
|
||
b.column_by_name(c)
|
||
.and_then(|col| col.as_any().downcast_ref::<BooleanArray>())
|
||
.map(|a| a.value(r))
|
||
.unwrap_or(false)
|
||
}
|
||
|
||
/// 读取 vector 列 (兼容 FixedSizeListArray 和 Float32Array)
|
||
fn col_vector(b: &RecordBatch, r: usize, c: &str) -> Vec<f32> {
|
||
let col = match b.column_by_name(c) {
|
||
Some(col) => col,
|
||
None => return vec![0.0_f32; 1024],
|
||
};
|
||
// 优先尝试 FixedSizeListArray (当前使用的格式)
|
||
if let Some(list_arr) = col.as_any().downcast_ref::<FixedSizeListArray>() {
|
||
let item_arr = list_arr.value(r);
|
||
if let Some(float_arr) = item_arr.as_any().downcast_ref::<Float32Array>() {
|
||
return float_arr.values().to_vec();
|
||
}
|
||
}
|
||
// 回退: Float32Array (展平格式)
|
||
if let Some(float_arr) = col.as_any().downcast_ref::<Float32Array>() {
|
||
let stride = 1024;
|
||
let start = r * stride;
|
||
if start + stride <= float_arr.len() {
|
||
return float_arr.values()[start..start + stride].to_vec();
|
||
}
|
||
}
|
||
vec![0.0_f32; 1024]
|
||
}
|
||
|
||
/// 纯逻辑:计算遗忘候选旋转窗口的下标区间。
|
||
/// - 输入:已排序去重的存活 id 集合(仅 mem_ 形态)、limit、after_id(游标)
|
||
/// - 返回:(start, end, wrapped)
|
||
/// - 窗口 = ids[start..end]
|
||
/// - wrapped=true 表示**游标回绕回 0**(本次传了游标,且游标 ≥ 全部 id 或已到存活集合末尾)
|
||
/// ⇒ 一轮完整遍历结束,下次从 0 起。无论 live 是否能被 limit 整除都能正确发出信号
|
||
/// (整除时靠「游标到末尾→下一轮回绕」触发,非整除时末窗口不足 limit 也回绕)。
|
||
/// 不依赖 LanceDB 排序下推,纯字符串集合成员计算(方案 c′ 核心,可单测)。
|
||
fn compute_forget_window(
|
||
ids: &[String],
|
||
limit: usize,
|
||
after_id: Option<&str>,
|
||
) -> (usize, usize, bool) {
|
||
let has_cursor = matches!(after_id, Some(a) if !a.is_empty());
|
||
if ids.is_empty() {
|
||
// 空库/全删:无可扫,直接标记一轮遍历结束(回绕语义,使 Go 侧游标复位不卡)
|
||
return (0, 0, true);
|
||
}
|
||
let mut start = if has_cursor {
|
||
// 第一个 > after_id 的下标;after_id 已删除/不在集合中 → 仍续扫之后(不丢、不崩)
|
||
ids.partition_point(|x| x.as_str() <= after_id.unwrap())
|
||
} else {
|
||
0
|
||
};
|
||
// 游标 ≥ 全部 id(已到末尾)→ 回绕到 0,并标记「一轮遍历结束」
|
||
let mut wrapped = false;
|
||
if start >= ids.len() {
|
||
start = 0;
|
||
wrapped = has_cursor;
|
||
}
|
||
let end = (start + limit).min(ids.len());
|
||
(start, end, wrapped)
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
/// 构造 N 个连续 mem_<n> 形态 id(定宽前缀保证字符串序 == 数字序)。
|
||
fn mk_ids(n: u64) -> Vec<String> {
|
||
(0..n).map(|i| format!("mem_{:019}", i)).collect()
|
||
}
|
||
|
||
/// 断言:连续 N 轮窗口的并集 == 全部存活 id(全覆盖判据,AC-1)。
|
||
/// 轮数上界 = ceil(live/limit),模拟真实调度(每 6h 一轮)。
|
||
fn union_covers_all(live: usize, limit: usize) -> bool {
|
||
let ids = mk_ids(live as u64);
|
||
let mut cursor = String::new();
|
||
let mut union = std::collections::HashSet::new();
|
||
let mut rounds = 0usize;
|
||
let max_rounds = live.div_ceil(limit) + 2; // 上界 + 余量
|
||
loop {
|
||
let after = if cursor.is_empty() { None } else { Some(cursor.as_str()) };
|
||
let (s, e, wrapped) = compute_forget_window(&ids, limit, after);
|
||
for x in &ids[s..e] {
|
||
union.insert(x.clone());
|
||
}
|
||
rounds += 1;
|
||
if wrapped {
|
||
break;
|
||
}
|
||
cursor = ids[e - 1].clone(); // 窗口末 id = 新游标
|
||
if rounds > max_rounds {
|
||
break; // 防死循环(理论不会触发)
|
||
}
|
||
}
|
||
union.len() == live && rounds <= max_rounds
|
||
}
|
||
|
||
/// 断言:窗口内全部「不满足遗忘条件」时,下一轮窗口仍与上一轮不同(不卡死,AC-2)。
|
||
fn not_stuck(live: usize, limit: usize) -> bool {
|
||
let ids = mk_ids(live as u64);
|
||
let after = Some(ids[0].as_str()); // 假设第 0 条已在窗口
|
||
let (s0, e0, _) = compute_forget_window(&ids, limit, None);
|
||
let w0: Vec<String> = ids[s0..e0].to_vec();
|
||
let (s1, e1, _) = compute_forget_window(&ids, limit, after);
|
||
let w1: Vec<String> = ids[s1..e1].to_vec();
|
||
// 窗口前进 → 集合不同(不卡死)
|
||
w0 != w1 && !w1.is_empty()
|
||
}
|
||
|
||
#[test]
|
||
fn test_window_full_coverage() {
|
||
// 小表(能整除):9 存活 / 3 窗口 → 3 轮全覆盖
|
||
assert!(union_covers_all(9, 3), "9/3 应 3 轮全覆盖");
|
||
// 非整除:10 存活 / 3 窗口 → 最后一轮不足 → 仍全覆盖
|
||
assert!(union_covers_all(10, 3), "10/3 应 ceil=4 轮全覆盖");
|
||
// 大表:22447 存活 / 2000 窗口 → ceil=12 轮全覆盖(贴近生产量级)
|
||
assert!(union_covers_all(22447, 2000), "22447/2000 应 12 轮全覆盖");
|
||
// 单条 / 空边界不崩
|
||
assert!(union_covers_all(1, 2000), "1 条应 1 轮全覆盖");
|
||
}
|
||
|
||
#[test]
|
||
fn test_window_not_stuck() {
|
||
// 窗口全不满足遗忘条件(模拟前 2000 条被高 recall 保护)→ 下轮仍换一批
|
||
assert!(not_stuck(10, 3), "前 3 条保护 → 下轮应换一批");
|
||
assert!(not_stuck(10000, 2000), "大表不卡死");
|
||
}
|
||
|
||
#[test]
|
||
fn test_window_wrap_boundary() {
|
||
// 空集合:window 为空,wrapped=true,不 panic
|
||
let empty: Vec<String> = vec![];
|
||
let (s, e, wrapped) = compute_forget_window(&empty, 2000, None);
|
||
assert_eq!(s, 0);
|
||
assert_eq!(e, 0);
|
||
assert!(wrapped);
|
||
// 单条:首轮返回该条(未 wrapped,游标=该 id);次轮游标回绕才 wrapped
|
||
let ids = mk_ids(1);
|
||
let (s, e, wrapped) = compute_forget_window(&ids, 2000, None);
|
||
assert_eq!(s, 0);
|
||
assert_eq!(e, 1);
|
||
assert!(!wrapped, "单条首轮覆盖完成,但游标尚未回绕 → 不标记");
|
||
let only = ids[0].clone();
|
||
let (s2, _e2, wrapped2) = compute_forget_window(&ids, 2000, Some(&only));
|
||
assert_eq!(s2, 0);
|
||
assert!(wrapped2, "单条次轮游标回绕 → 标记一轮遍历结束");
|
||
// 游标已到末尾(= 最后一个 id)→ 下轮回绕从 0 起,并标记一轮遍历结束
|
||
let ids = mk_ids(10);
|
||
let last = ids.last().unwrap().clone();
|
||
let (s, e, wrapped) = compute_forget_window(&ids, 3, Some(&last));
|
||
assert_eq!(s, 0, "游标在末尾 → 应回绕到 0");
|
||
assert_eq!(e, 3);
|
||
assert!(wrapped, "回绕到 0 → 标记一轮遍历结束");
|
||
}
|
||
|
||
#[test]
|
||
fn test_window_cursor_deleted_resilience() {
|
||
// 游标 id 已被删除(不在集合中)→ 降级续扫其后,不 panic(AC-8)
|
||
let mut ids = mk_ids(10);
|
||
// 真删掉 id=5(模拟 soft-delete 后该 id 不在存活集合)
|
||
ids.retain(|x| x != "mem_0000000000000000005");
|
||
let gone = format!("mem_{:019}", 5); // 游标指向已删的 5
|
||
let (s, e, _) = compute_forget_window(&ids, 3, Some(&gone));
|
||
assert_eq!(s, 5, "游标=已删的5 → 应从第一个>5的下标续扫(不丢不崩)");
|
||
assert_eq!(e, 8);
|
||
// 游标大于全部 id → 回绕从头
|
||
let big = format!("mem_{:019}", 999);
|
||
let (s, e, wrapped) = compute_forget_window(&ids, 3, Some(&big));
|
||
assert_eq!(s, 0, "游标大于全部 → 回绕从头");
|
||
assert_eq!(e, 3);
|
||
assert!(wrapped, "游标越界 → 标记一轮遍历结束");
|
||
}
|
||
|
||
#[test]
|
||
fn test_window_dedup_assertion() {
|
||
// 排序后去重:重复 id 应被去重且不崩(AC-9)
|
||
let mut ids = mk_ids(5);
|
||
ids.push("mem_0000000000000000002".to_string()); // 重复
|
||
ids.sort();
|
||
let before = ids.len();
|
||
ids.dedup();
|
||
assert_eq!(ids.len(), before - 1, "重复 id 应去重");
|
||
// 去重后仍能正确轮转
|
||
let (s, e, _) = compute_forget_window(&ids, 2000, None);
|
||
assert_eq!(s, 0);
|
||
assert_eq!(e, ids.len());
|
||
}
|
||
|
||
/// 真实库只读覆盖率测试(AC-1/2,需 LIVE_DB_WINDOW=1 才跑,避免污染常规 cargo test)。
|
||
/// 用生产代码路径 scan_forgetting_window 循环取窗直到 wrapped,验证:
|
||
/// ① 连续窗口两两不同(不卡死)
|
||
/// ② 并集最终覆盖全部存活 mem_ 记忆(覆盖率→100%)
|
||
/// ③ 池内 created_at>10-07 条数从旧实现~2 显著上升
|
||
#[test]
|
||
fn test_live_db_full_coverage() {
|
||
if std::env::var("LIVE_DB_WINDOW").as_deref() != Ok("1") {
|
||
eprintln!("[skip] LIVE_DB_WINDOW!=1");
|
||
return;
|
||
}
|
||
let ldb = LanceDBOps::open("/var/lib/memoryweave").expect("open live db");
|
||
let mut union = std::collections::HashSet::new();
|
||
let mut cursor = String::new();
|
||
let mut round = 0usize;
|
||
let mut live_total = 0usize;
|
||
let mut max_recent = 0usize;
|
||
let mut rounds_with_recent = 0usize;
|
||
loop {
|
||
let after = if cursor.is_empty() { None } else { Some(cursor.as_str()) };
|
||
let w = ldb.scan_forgetting_window(2000, after).expect("window");
|
||
if round == 0 { live_total = w.live_total; }
|
||
let recent = w.records.iter().filter(|r| r.created_at > "2026-10-07T00:00:00Z".to_string()).count();
|
||
max_recent = max_recent.max(recent);
|
||
if recent > 0 { rounds_with_recent += 1; }
|
||
for r in &w.records { union.insert(r.id.clone()); }
|
||
round += 1;
|
||
println!("round{} n={} live_total={} wrapped={} recent>10-07={} union={}/{}",
|
||
round, w.records.len(), w.live_total, w.wrapped, recent, union.len(), live_total);
|
||
if w.wrapped { break; }
|
||
cursor = w.next_cursor;
|
||
if round > live_total.div_ceil(2000) + 5 { println!("guard break"); break; }
|
||
}
|
||
println!("RESULT: rounds={} union={} live_total={} coverage={:.4} max_recent_in_pool={} rounds_with_recent={}",
|
||
round, union.len(), live_total, union.len() as f64 / live_total as f64, max_recent, rounds_with_recent);
|
||
assert_eq!(union.len(), live_total, "全库存活 mem_ 记忆应被全覆盖");
|
||
}
|
||
} |