Files
memoryweave/rust/src/main.rs
T
小唯 941b7eec32 fix(storage): t_20bfbd9a 边车线程风暴根治 — Rust 连接并发上限+RSS护栏降级串行;命中事件重试跨commit指数退避+积压上限
AC1 Rust accept 循环对每连接 thread::spawn 无上限,Go 侧每 rpc 新 dial ⇒
   IPC 洪泛线程线性涨到 736。加 AtomicUsize 连接门(默认64, env ZHIYI_CONN_MAX),
   超限回 heavy_conn_busy;RSS 超上限(默认3000MB, env ZHIYI_RSS_LIMIT_MB)降级串行。
AC3 usage_recall 加 retry_count/next_retry_at:持久失败行指数退避(1s..600s)不再
   每 commit 热重扫;积压超 ZHIYI_USAGE_BACKLOG_MAX(默认500)本轮跳过扫描。
AC4 新增 usageBackoff/backlog 单测。Go 全仓测试通过。
2026-10-18 02:19:03 +08:00

1085 lines
46 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// 织忆 MemoryWeave — zhiyi-consolidate (Rust 整合引擎)
// Unix Socket + Protobuf 协议,Go ↔ Rust IPC
// 深度整合 5 步:DBSCAN → 修剪 → 衰减校准 → 质量回溯 → 报告
mod lancedb_ops;
mod embed;
mod rerank;
mod cluster;
mod decay_calibrate;
mod graph_prune;
mod quality_backtrace;
mod report;
use clap::Parser;
use serde::{Deserialize, Serialize};
use std::time::Duration;
use crate::lancedb_ops::MemoryRecord;
use std::collections::HashMap;
use std::io::{Read, Write};
use std::net::{Shutdown, TcpListener, TcpStream};
use std::os::unix::net::{UnixListener, UnixStream};
use std::thread;
use std::time::Instant;
use cluster::Clusterer;
use decay_calibrate::{DecayCalibrator, DecaySample};
use embed::BGEEncoder;
use graph_prune::GraphPruner;
use lancedb_ops::LanceDBOps;
use quality_backtrace::{QualityBacktracer, QualitySample};
use report::{ConsolidationReport, DashboardSnapshot, ReportGenerator};
use std::path::Path;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::Arc;
// ─── 风暴守卫 + 健康探针(t_89d22fc8, 2026-10-16 22:42 事故)────────────────
// 事故链:触发器 30s 循环在「触发器状态重置后」同秒发 cluster_only+full+full ⇒
// 每个 consolidate 连接各起一个线程并发跑 DBSCAN(10K 向量)⇒ 内存/swap 打满
// (RSS 2.0G peak + swap 1.1G, zram 满, PSI 0.31)⇒ 35min 全量整合跑不完 ⇒
// Go 侧 10min 超时放弃后下一轮再发 full ⇒ 线程堆积 ⇒ 主实例 OOM 被 kill ⇒
// 停/重启循环 ⇒ /tmp/zhiyi-ipc.sock 消失数秒 ⇒ Go 全量写 refused(3800 次)。
// 治本:同时只允许一个整合任务(重活隔离:轻 IPC 永远在线程里各自跑,不受影响);
// 整合在途时新的 consolidate 请求立即拒绝(consolidate_busy),不排队、不堆积。
static CONSOLIDATING: AtomicBool = AtomicBool::new(false);
/// 整合任务 RAII 守卫:获取成功 = 独占整合槽;Drop(正常返回或 panic 展开)自动释放。
struct ConsolidatingGuard;
impl ConsolidatingGuard {
fn try_acquire() -> Option<Self> {
CONSOLIDATING
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
.ok()
.map(|_| ConsolidatingGuard)
}
}
impl Drop for ConsolidatingGuard {
fn drop(&mut self) {
CONSOLIDATING.store(false, Ordering::SeqCst);
}
}
// ─── 重读准入信号量(方案①,t_690cc333)────────────────────────────────
// 事故链(t_55af3236 实证,t_89d22fc8 事故):并发重读(search top_k=10000 /
// forget_window / scan_all / scan_deleted)是真正的内存放大器 —— 3×search10000
// ⇒ swap 吞 1.5GB ⇒ OOM ⇒ /tmp/zhiyi-ipc.sock 消失 ⇒ 3800 次写失败。
// 风暴守卫只拦 consolidate,不拦这些非整合重读 ⇒ 设计缺口。
// 治本:重读准入计数信号量(默认 N=2,env ZHIYI_HEAVY_READ_CONCURRENCY 可调)。
// 轻操作(ping/insert/update/append_hit_event/stats…)永不排队、各自线程执行
// (写仍单写者/串行,写路径零改动 ⇒ 不破坏单写者语义)。
// 重读无许可 ⇒ 有界等待(100ms)后拒绝 heavy_read_busy(Go 侧降级/重试,
// 不排队、不占内存)。
// 实现:AtomicUsize 计数门(与 ConsolidatingGuard 同族,无外部依赖;本机
// rustup stable 工具链无 std::sync::Semaphore)。
static HEAVY_READ_DEFAULT: usize = 2;
static HEAVY_READ_WAIT_MS: u64 = 100;
// ─── 单次重读行数上限(方案②,t_576714de,2026-10-17 残留缺口)────────
// N=2 准入只拦「并发叠加」,不拦「单次请求自身的内存量」:一个 top_k=1000000 /
// scan_all 请求可独自吞下全表(实测 43547 行 × 1024 维向量 ≈ 260MB Rust + JSON ~650MB),
// 照样 thrash(t_eedb2d39 根因复盘定性的 ② 单次量无上限)。
// 治本:给单次重读的**返回行数**设硬上限(默认 20000,env ZHIYI_HEAVY_READ_MAX_ROWS 可调)。
// 请求量超上限 ⇒ 截断到上限 + 标记 truncated:true(降级返回部分结果,而非硬吃内存)。
// 热路径 recall 的 top_k ≤ 1000 永不触发;只挡 admin/穷举型单次大请求。
static HEAVY_READ_MAX_ROWS_DEFAULT: usize = 20000;
fn heavy_read_max_rows() -> usize {
match std::env::var("ZHIYI_HEAVY_READ_MAX_ROWS") {
Ok(v) => v.trim().parse::<usize>().unwrap_or(HEAVY_READ_MAX_ROWS_DEFAULT).max(1),
Err(_) => HEAVY_READ_MAX_ROWS_DEFAULT,
}
}
fn heavy_read_permits() -> usize {
match std::env::var("ZHIYI_HEAVY_READ_CONCURRENCY") {
Ok(v) => v.trim().parse::<usize>().unwrap_or(HEAVY_READ_DEFAULT),
Err(_) => HEAVY_READ_DEFAULT,
}
}
/// 重读许可 RAII 守卫:Drop 自动归还计数(正常返回或 panic 展开都不泄漏)。
struct HeavyReadPermit {
counter: Arc<AtomicUsize>,
}
impl Drop for HeavyReadPermit {
fn drop(&mut self) {
self.counter.fetch_sub(1, Ordering::SeqCst);
}
}
/// 重读准入门:permits = 最大并发重读数(N);in_flight = 在途重读数。
struct HeavyReadGate {
permits: usize,
in_flight: Arc<AtomicUsize>,
}
impl HeavyReadGate {
fn new(n: usize) -> Self {
Self {
permits: n.max(1),
in_flight: Arc::new(AtomicUsize::new(0)),
}
}
/// 有界等待后获取;无槽释放 ⇒ 拒绝(heavy_read_busy),不排队、不占内存。
fn try_acquire(&self) -> Result<HeavyReadPermit, String> {
let deadline = Instant::now() + Duration::from_millis(HEAVY_READ_WAIT_MS);
loop {
let cur = self.in_flight.load(Ordering::SeqCst);
if cur < self.permits
&& self
.in_flight
.compare_exchange(cur, cur + 1, Ordering::SeqCst, Ordering::SeqCst)
.is_ok()
{
return Ok(HeavyReadPermit {
counter: Arc::clone(&self.in_flight),
});
}
if Instant::now() >= deadline {
return Err(
"heavy read admission exhausted (N heavy reads in flight); \
request dropped, retry later"
.into(),
);
}
thread::sleep(Duration::from_millis(10));
}
}
/// 当前在途重读数(供 zhiyi_ping 观测)。
fn in_flight(&self) -> usize {
self.in_flight.load(Ordering::SeqCst)
}
}
// ─── 连接并发上限 + RSS 护栏(t_20bfbd9a 治本)────────────────────────
// 事故链(t_20bfbd9a 收口):accept 循环对每个 TCP 连接 thread::spawn,无并发上限;
// Go 侧每 rpc 新 dial(lancedb_ipc.go rpc() → net.DialTimeout),IPC 洪泛 → 线程
// 线性涨到 736(实测),每线程持独立 stream + LanceDB 批次内存 ⇒ 5.1G 单进程。
// 治本①连接并发上限:AtomicUsize in_flight 计数,spawn 前 CAS +1,超上限 ⇒ 拒绝
// (回 heavy_conn_busy);线程结束 RAII Drop 归还。轻热路径(ping/insert/update)
// 在正常流量下永不超过上限,不改变单写者语义。
// 治本②RSS 护栏:每 accept 采样 VmRSS,超上限 ⇒ 降级到串行(并发 1,inline 处理
// + 拒重读),不再叠加线程;低于上限自动恢复。上限 env ZHIYI_RSS_LIMIT_MB 可调。
static CONN_MAX_DEFAULT: usize = 64;
static RSS_LIMIT_MB_DEFAULT: u64 = 3000;
fn conn_permits() -> usize {
match std::env::var("ZHIYI_CONN_MAX") {
Ok(v) => v.trim().parse::<usize>().unwrap_or(CONN_MAX_DEFAULT).max(1),
Err(_) => CONN_MAX_DEFAULT,
}
}
fn rss_limit_kb() -> u64 {
match std::env::var("ZHIYI_RSS_LIMIT_MB") {
Ok(v) => v.trim().parse::<u64>().unwrap_or(RSS_LIMIT_MB_DEFAULT).max(64) * 1024,
Err(_) => RSS_LIMIT_MB_DEFAULT * 1024,
}
}
/// 连接许可 RAII 守卫:Drop 自动归还 in_flight(线程正常返回或 panic 都不泄漏)。
struct ConnPermit {
counter: Arc<AtomicUsize>,
}
impl Drop for ConnPermit {
fn drop(&mut self) {
self.counter.fetch_sub(1, Ordering::SeqCst);
}
}
/// 尝试登记一个并发连接;有槽 ⇒ Some(许可),满 ⇒ None(调用方拒绝/降级)。
fn conn_try_acquire(counter: &Arc<AtomicUsize>, limit: usize) -> Option<ConnPermit> {
let mut cur = counter.load(Ordering::SeqCst);
loop {
if cur >= limit {
return None;
}
match counter.compare_exchange_weak(
cur,
cur + 1,
Ordering::SeqCst,
Ordering::SeqCst,
) {
Ok(_) => {
return Some(ConnPermit {
counter: Arc::clone(counter),
})
}
Err(actual) => cur = actual,
}
}
}
// ─── 内存回收 & 观测(2026-09-21 治本)────────────────────────────────
// 根因:长驻 sidecar 里每次操作的大块分配(Arrow/LanceDB 批次、Vec<MemoryRecord>)
// 释放后 glibc 不还给 OS(arena 保留)→ RSS 只涨不落,5 小时累积到 1.09G
// (单次 full consolidate 峰值仅 250M)。对策:每个重活阶段后 malloc_trim(0)。
extern "C" {
fn malloc_trim(pad: usize) -> i32;
}
/// 读自身 RSS / 峰值 RSS(KB)
fn rss_kb() -> (u64, u64) {
let (mut rss, mut hwm) = (0u64, 0u64);
if let Ok(s) = std::fs::read_to_string("/proc/self/status") {
for line in s.lines() {
if let Some(v) = line.strip_prefix("VmRSS:") {
rss = v.split_whitespace().next().and_then(|x| x.parse().ok()).unwrap_or(0);
} else if let Some(v) = line.strip_prefix("VmHWM:") {
hwm = v.split_whitespace().next().and_then(|x| x.parse().ok()).unwrap_or(0);
}
}
}
(rss, hwm)
}
/// 把已释放但未归还的堆内存还给 OS,并打印一行内存观测
fn reclaim(phase: &str) {
unsafe { malloc_trim(0) };
let (rss, hwm) = rss_kb();
eprintln!("[mem] {} rss={}MB peak={}MB", phase, rss / 1024, hwm / 1024);
}
/// 静默 trim(高频 IPC 用,不刷日志)
fn trim_only() {
unsafe { malloc_trim(0) };
}
/// CLI 参数
#[derive(Parser, Debug, Clone)]
#[command(name = "zhiyi-consolidate")]
#[command(about = "MemoryWeave 整合引擎 — DBSCAN + 衰减 + 修剪 + 质量回溯")]
struct Args {
/// 工作模式: full / cluster_only / prune_only / encode
#[arg(short, long, default_value = "full")]
mode: String,
/// 数据目录
#[arg(long, default_value = "./data")]
data_dir: String,
/// SQLite 图谱路径
#[arg(long, default_value = "./data/graph.db")]
sqlite_path: String,
/// LLM 蒸馏 API 端点
#[arg(long, default_value = "")]
llm_endpoint: String,
/// LLM 模型名
#[arg(long, default_value = "deepseek/deepseek-v4-pro")]
llm_model: String,
/// LLM API Key (Authorization header)
#[arg(long, default_value = "")]
llm_api_key: String,
/// LLM 预算(本次可用次数)
#[arg(long, default_value = "20")]
llm_budget: usize,
/// Unix Socket 路径
#[arg(long)]
socket: Option<String>,
/// DBSCAN epsilon
#[arg(long, default_value = "0.5")]
epsilon: f64,
/// DBSCAN min_points
#[arg(long, default_value = "3")]
min_points: usize,
/// BGE 模型目录 (用于 encode 模式)
#[arg(long, default_value = "/home/muc/models/bge-m3/onnx")]
model_dir: String,
/// encode 模式输入 (单个文本)
#[arg(long)]
text: Option<String>,
/// encode 模式批量输入 (逗号分隔)
#[arg(long)]
texts: Option<String>,
}
/// Protobuf-like 请求消息(简化版)
#[derive(Debug, Serialize, Deserialize)]
struct ConsolidateRequest {
task: String,
lancedb_path: String,
sqlite_path: String,
#[serde(rename = "llm_endpoint")] llm_endpoint: String,
#[serde(rename = "llm_model")] llm_model: String,
#[serde(rename = "llm_api_key")] llm_api_key: String,
llm_budget: usize,
epsilon: f64,
min_points: usize,
#[serde(rename = "model_dir", default)] model_dir: Option<String>,
}
#[derive(Debug, Serialize, Deserialize)]
struct ConsolidateResponse {
status: String,
report_json: String,
failure_step: String,
error_detail: String,
// 方案②(t_576714de):单次重读被行数上限截断时为 true(降级返回部分结果)。
// 旧 Go 客户端无此字段 ⇒ serde 缺省 false,向后兼容。
#[serde(default)]
truncated: bool,
}
fn main() {
let args = Args::parse();
if let Some(ref socket_path) = args.socket {
run_socket_server(socket_path, &args);
} else if args.mode == "encode" {
// encode 模式: BGE 编码
run_encode(&args);
} else {
// CLI 模式: 执行一次并输出 JSON
let ldb = LanceDBOps::open(&args.data_dir).expect("lancedb open");
let report = run_consolidation(&args, &ldb);
println!("{}", serde_json::to_string_pretty(&report).unwrap_or_default());
}
}
fn run_encode(args: &Args) {
let encoder = match embed::BGEEncoder::new(&args.model_dir) {
Ok(e) => e,
Err(e) => {
eprintln!("[encode] failed to load BGE model: {}", e);
return;
}
};
let texts: Vec<String> = if let Some(t) = &args.texts {
t.split(',').map(|s| s.trim().to_string()).collect()
} else if let Some(t) = &args.text {
vec![t.clone()]
} else {
eprintln!("[encode] use --text or --texts");
return;
};
let vectors = match encoder.encode_batch(&texts) {
Ok(v) => v,
Err(e) => {
eprintln!("[encode] failed: {}", e);
return;
}
};
println!("{}", serde_json::to_string(&vectors).unwrap_or_default());
}
/// Socket 服务共享状态
struct ServerState {
encoder: Option<Arc<BGEEncoder>>,
/// 重读准入信号量(方案① t_690cc333):限制并发重读(search>1000/forget_window/
/// scan_all/scan_deleted)的内存无界叠加。N 默认 2,env ZHIYI_HEAVY_READ_CONCURRENCY 可调。
heavy_read: HeavyReadGate,
}
/// Unix Socket 服务端
fn run_socket_server(socket_path: &str, args: &Args) {
if Path::new(socket_path).exists() {
std::fs::remove_file(socket_path).ok();
}
let listener = UnixListener::bind(socket_path).expect("bind");
let lancedb = LanceDBOps::open(&args.data_dir).expect("lancedb open");
lancedb.init_tables().ok();
// 初始化 BGE 编码器
let encoder: Option<Arc<BGEEncoder>> = if !args.model_dir.is_empty() {
match BGEEncoder::new(&args.model_dir) {
Ok(e) => {
eprintln!("[zhiyi] BGE encoder loaded: {} (dim={})", args.model_dir, e.dim);
Some(Arc::new(e))
}
Err(e) => {
eprintln!("[zhiyi] WARN: BGE encoder init failed: {} — encode 请求将失败", e);
None
}
}
} else {
eprintln!("[zhiyi] BGE encoder not configured (--model-dir not set)");
None
};
let n = heavy_read_permits();
let state = Arc::new(ServerState {
encoder,
heavy_read: HeavyReadGate::new(n),
});
let conn_limit = conn_permits();
let conn_in_flight = Arc::new(AtomicUsize::new(0));
eprintln!(
"[zhiyi] listening on {} (lancedb: {}), heavy-read admission N={} (env ZHIYI_HEAVY_READ_CONCURRENCY), \
conn cap={} (env ZHIYI_CONN_MAX), rss limit={}MB (env ZHIYI_RSS_LIMIT_MB)",
socket_path, args.data_dir, n, conn_limit, rss_limit_kb() / 1024
);
for stream in listener.incoming() {
match stream {
Ok(stream) => {
let a = args.clone();
let ldb = lancedb.clone();
let st = Arc::clone(&state);
// RSS 护栏(治本②):超上限 ⇒ 降级到串行(并发 1,inline 处理 + 拒重读),
// 不再叠加线程;低于上限自动恢复。这是对「无内存护栏」事故的直接兜底。
let (rss, _) = rss_kb();
if rss > rss_limit_kb() {
eprintln!(
"[zhiyi] RSS {}MB > limit {}MB: degrade to serial (concurrency 1)",
rss / 1024,
rss_limit_kb() / 1024
);
handle_client(stream, &a, ldb, &st);
continue;
}
// 连接并发上限(治本①):CAS +1,满 ⇒ 回 heavy_conn_busy 并拒绝。
let permit = match conn_try_acquire(&conn_in_flight, conn_limit) {
Some(p) => p,
None => {
eprintln!(
"[zhiyi] connection cap {} reached: reject (heavy_conn_busy)",
conn_limit
);
// 直接回一个 JSON 错误给客户端(对齐 Go rpc() 的读 deadline)
let mut stream = stream;
let _ = stream.write_all(b"\x00\x00\x00\x46");
let _ = stream.write_all(
br#"{"status":"error","error":"heavy_conn_busy","detail":"connection cap exhausted"}"#,
);
continue;
}
};
thread::spawn(move || {
handle_client(stream, &a, ldb, &st);
drop(permit); // RAII 归还 in_flight(也兜 panic 展开)
});
}
Err(e) => eprintln!("[zhiyi] accept error: {}", e),
}
}
}
fn handle_client(mut stream: UnixStream, args: &Args, lancedb: LanceDBOps, state: &ServerState) {
let mut len_buf = [0u8; 4];
if stream.read_exact(&mut len_buf).is_err() {
return;
}
let msg_len = u32::from_be_bytes(len_buf) as usize;
let mut body = vec![0u8; msg_len];
if stream.read_exact(&mut body).is_err() {
return;
}
let msg: serde_json::Value = match serde_json::from_slice(&body) {
Ok(m) => m,
Err(e) => {
send_error(&mut stream, "parse", &e.to_string());
return;
}
};
let msg_type = msg["type"].as_str().unwrap_or("consolidate");
eprintln!(
"[zhiyi] msg type = {:?}, raw keys = {:?}",
msg_type,
msg.as_object().map(|o| o.keys().collect::<Vec<_>>())
);
match msg_type {
"encode" => {
let texts: Vec<String> = msg["texts"]
.as_array()
.map(|a| {
a.iter()
.filter_map(|v| v.as_str())
.map(|s| s.to_string())
.collect()
})
.unwrap_or_default();
if texts.is_empty() {
send_error(&mut stream, "encode", "empty texts");
return;
}
let encoder = match &state.encoder {
Some(e) => e,
None => {
send_error(&mut stream, "encode", "encoder not initialized");
return;
}
};
match encoder.encode_batch(&texts) {
Ok(vectors) => {
let json = serde_json::to_string(&vectors).unwrap_or_default();
send_ok(&mut stream, &json);
}
Err(e) => send_error(&mut stream, "encode", &e.to_string()),
}
}
"lancedb_search" => {
let v: Vec<f32> = msg["vector"].as_array()
.map(|a| a.iter().filter_map(|v| v.as_f64()).map(|v| v as f32).collect())
.unwrap_or_default();
let requested = msg["top_k"].as_u64().unwrap_or(10) as usize;
// 空 namespace = 不过滤(匹配所有)
let ns = msg["namespace"].as_str().filter(|s| !s.is_empty());
// 方案②(t_576714de):单次重读行数硬上限 —— 截断到上限,超限标 truncated:true,
// 降级返回部分结果而非让单个请求独自吞下全表向量(43547 行 ≈ 260MB Rust + JSON)。
let cap = heavy_read_max_rows();
let tk = requested.min(cap);
let truncated = requested > tk;
// 重读准入(t_690cc333 方案①):top_k>1000 的重搜索是内存放大器,
// 计入信号量 N;无许可 ⇒ 有界等待后拒绝 heavy_read_busy。
let _perm = if tk > 1000 {
match state.heavy_read.try_acquire() {
Ok(p) => Some(p),
Err(e) => {
eprintln!("[zhiyi-ipc] search top_k={} REJECTED (heavy_read_busy)", tk);
send_error(&mut stream, "heavy_read_busy", &e);
trim_only();
return;
}
}
} else {
None
};
match lancedb.search(&v, tk, ns) {
Ok(r) => {
let json = serde_json::to_string(&r).unwrap_or_default();
if truncated {
eprintln!("[zhiyi-ipc] search top_k={} CLAMPED to {} (single-read cap {}), truncated=true", requested, r.len(), cap);
send_ok_truncated(&mut stream, &json);
} else {
send_ok(&mut stream, &json);
}
}
Err(e) => send_error(&mut stream, "lancedb_search", &e.to_string()),
}
}
"lancedb_insert" => {
let table = msg["table"].as_str().unwrap_or("memories");
let records = msg["records"].as_str().unwrap_or("[]");
match lancedb.insert_batch(table, records) {
Ok(n) => send_ok(&mut stream, &format!(r#"{{"inserted":{}}}"#, n)),
Err(e) => send_error(&mut stream, "lancedb_insert", &e.to_string()),
}
}
"lancedb_stats" => {
match lancedb.stats() {
Ok(s) => send_ok(&mut stream, &serde_json::to_string(&s).unwrap_or_default()),
Err(e) => send_error(&mut stream, "lancedb_stats", &e.to_string()),
}
}
"lancedb_update" => {
let table = msg["table"].as_str().unwrap_or("memories");
let id = msg["id"].as_str().unwrap_or("");
let fields = msg["fields"].as_str().unwrap_or("[]");
if id.is_empty() {
send_error(&mut stream, "lancedb_update", "id required");
} else {
match lancedb.update(table, id, fields) {
Ok(n) => send_ok(&mut stream, &format!(r#"{{"updated":{}}}"#, n)),
Err(e) => send_error(&mut stream, "lancedb_update", &e.to_string()),
}
}
}
"lancedb_update_batch" => {
// 累积召回批量更新(2026-09-12 优化B):一次 update 提交 = 1 个 LanceDB 版本
let _table = msg["table"].as_str().unwrap_or("memories");
let items = msg["items"].as_str().unwrap_or("[]");
let ts = msg["ts"].as_str().unwrap_or("");
match lancedb.update_recall_batch(items, ts) {
Ok(n) => {
let cnt: usize = serde_json::from_str::<Vec<serde_json::Value>>(items)
.map(|v| v.len())
.unwrap_or(0);
send_ok(&mut stream, &format!(r#"{{"updated":{},"items":{}}}"#, n, cnt));
}
Err(e) => send_error(&mut stream, "lancedb_update_batch", &e.to_string()),
}
}
// GA-P0-4 (t_bb10e8bc):hits+/hits− outcome 事件(append-only 事件表)
"lancedb_append_hit_event" => {
let id = msg["id"].as_str().unwrap_or("");
let outcome = msg["outcome"].as_str().unwrap_or("");
let reason = msg["reason"].as_str().unwrap_or("");
let source = msg["source"].as_str().unwrap_or("");
let ts = msg["ts"].as_str().unwrap_or("");
if id.is_empty() {
send_error(&mut stream, "lancedb_append_hit_event", "id required");
} else {
let now = if ts.is_empty() {
chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true)
} else {
ts.to_string()
};
match lancedb.append_hit_event(id, outcome, reason, source, &now) {
Ok(n) => send_ok(&mut stream, &format!(r#"{{"appended":{}}}"#, n)),
Err(e) => send_error(&mut stream, "lancedb_append_hit_event", &e.to_string()),
}
}
}
"lancedb_count_hit_events" => {
let id = msg["id"].as_str().unwrap_or("");
let outcome = msg["outcome"].as_str().unwrap_or("");
let since = msg["since"].as_str().unwrap_or("");
match lancedb.count_hit_events(id, outcome, since) {
Ok(n) => send_ok(&mut stream, &format!(r#"{{"count":{}}}"#, n)),
Err(e) => send_error(&mut stream, "lancedb_count_hit_events", &e.to_string()),
}
}
"lancedb_get_hit_events" => {
let id = msg["id"].as_str().unwrap_or("");
let limit = msg["limit"].as_u64().unwrap_or(100) as usize;
match lancedb.get_hit_events(id, limit) {
Ok(evs) => send_ok(&mut stream, &serde_json::to_string(&evs).unwrap_or_default()),
Err(e) => send_error(&mut stream, "lancedb_get_hit_events", &e.to_string()),
}
}
"lancedb_query" => {
let min_recall = msg["min_recall"].as_i64().unwrap_or(5) as i64;
let limit = msg["limit"].as_u64().unwrap_or(20) as usize;
eprintln!("[lancedb] query: min_recall={} limit={}", min_recall, limit);
match lancedb.query_memories(min_recall, limit) {
Ok(r) => {
eprintln!("[lancedb] query returned {} records", r.len());
send_ok(&mut stream, &serde_json::to_string(&r).unwrap_or_default())
}
Err(e) => send_error(&mut stream, "lancedb_query", &e.to_string()),
}
}
"lancedb_scan" => {
// P2 2026-09-06: 遗忘候选安全全表扫描 — 限长(硬上限 5000) + 跳 vector
let mut limit = msg["limit"].as_u64().unwrap_or(2000) as usize;
if limit > 5000 {
limit = 5000;
}
// 2026-09-21: 可选 namespace 下推过滤(列表/导出端点用)
let namespace = msg["namespace"].as_str().filter(|s| !s.is_empty());
eprintln!("[lancedb] scan_forgetting: limit={} namespace={:?}", limit, namespace);
match lancedb.scan_for_forgetting(limit, namespace) {
Ok(r) => send_ok(&mut stream, &serde_json::to_string(&r).unwrap_or_default()),
Err(e) => send_error(&mut stream, "lancedb_scan", &e.to_string()),
}
}
"lancedb_scan_all" => {
// GA-P0-1 (t_718d8707):全表扫描(scan_all 上限 200000,仅存活行),
// 供 not_useful_count 拆分迁移脚本读取全部行做统计。scan 硬上限 5000
// 覆盖不了 35010 行。只读,不落库。
// 重读准入(t_690cc333 方案①):全表扫描是内存放大器,计入信号量 N。
let _perm = match state.heavy_read.try_acquire() {
Ok(p) => Some(p),
Err(e) => {
eprintln!("[zhiyi-ipc] scan_all REJECTED (heavy_read_busy)");
send_error(&mut stream, "heavy_read_busy", &e);
trim_only();
return;
}
};
// 方案②(t_576714de):单次扫描行数上限 —— 超上限截断 + truncated:true(降级部分结果)。
let cap = heavy_read_max_rows();
match lancedb.scan_all(cap) {
Ok(r) => {
let json = serde_json::to_string(&r).unwrap_or_default();
let trunc = r.len() >= cap;
if trunc {
eprintln!("[zhiyi-ipc] scan_all hit cap {} → truncated=true", cap);
send_ok_truncated(&mut stream, &json);
} else {
send_ok(&mut stream, &json);
}
}
Err(e) => send_error(&mut stream, "lancedb_scan_all", &e.to_string()),
}
}
"lancedb_scan_deleted" => {
// GA-P0-3 (t_f785d7f2):全量扫描**已删除**行(is_deleted=true),供 delete_reason
// 回填脚本枚举 + 出直方图。只读,不落库。
// 重读准入(t_690cc333 方案①):全表扫描是内存放大器,计入信号量 N。
let _perm = match state.heavy_read.try_acquire() {
Ok(p) => Some(p),
Err(e) => {
eprintln!("[zhiyi-ipc] scan_deleted REJECTED (heavy_read_busy)");
send_error(&mut stream, "heavy_read_busy", &e);
trim_only();
return;
}
};
// 方案②(t_576714de):单次扫描行数上限 —— 超上限截断 + truncated:true。
let cap = heavy_read_max_rows();
match lancedb.scan_deleted(cap) {
Ok(r) => {
let json = serde_json::to_string(&r).unwrap_or_default();
let trunc = r.len() >= cap;
if trunc {
eprintln!("[zhiyi-ipc] scan_deleted hit cap {} → truncated=true", cap);
send_ok_truncated(&mut stream, &json);
} else {
send_ok(&mut stream, &json);
}
}
Err(e) => send_error(&mut stream, "lancedb_scan_deleted", &e.to_string()),
}
}
"lancedb_forget_window" => {
// P1 2026-10-11 (t_a4b8fa52 方案 c′): 遗忘候选「旋转窗口」扫描
// 修复固定 2000 窗口不轮转。接收 after_id(游标),返回 ForgetWindow(records+next_cursor+wrapped+live_total)。
// 重读准入(t_690cc333 方案①):整窗扫描是内存放大器,计入信号量 N。
let _perm = match state.heavy_read.try_acquire() {
Ok(p) => Some(p),
Err(e) => {
eprintln!("[zhiyi-ipc] forget_window REJECTED (heavy_read_busy)");
send_error(&mut stream, "heavy_read_busy", &e);
trim_only();
return;
}
};
let limit = msg["limit"].as_u64().unwrap_or(2000) as usize;
let after_id = msg["after_id"].as_str().filter(|s| !s.is_empty());
eprintln!("[lancedb] forget_window: limit={} after_id={:?}", limit, after_id);
match lancedb.scan_forgetting_window(limit, after_id) {
Ok(w) => send_ok(&mut stream, &serde_json::to_string(&w).unwrap_or_default()),
Err(e) => send_error(&mut stream, "lancedb_forget_window", &e.to_string()),
}
}
"lancedb_purge_invalid" => {
// P1-1 2026-10-08: 清理 id 为空的垃圾行(无法按 id 软删,按 id='' 条件一次性软删)。
// ts 由 Go 侧传入(RFC3339),与 soft_delete_batch 同构。
let ts = msg["ts"].as_str().unwrap_or("");
match lancedb.purge_invalid(ts) {
Ok(n) => send_ok(&mut stream, &format!(r#"{{"purged":{}}}"#, n)),
Err(e) => send_error(&mut stream, "lancedb_purge_invalid", &e.to_string()),
}
}
// 健康探针(t_89d22fc8 AC3):轻量 echo + 整合在途标志。
// 探针判据:connect ≤3s 且 zhiyi_ping 往返 ≤5s = 健康;
// consolidating=true 只表示「重活在跑」(轻 IPC 仍正常响应,不算故障)。
"zhiyi_ping" => {
let resp = serde_json::json!({
"ok": true,
"consolidating": CONSOLIDATING.load(Ordering::SeqCst),
"heavy_read_in_flight": state.heavy_read.in_flight(),
"heavy_read_permits": heavy_read_permits(),
"heavy_read_max_rows": heavy_read_max_rows(),
"ts": chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true),
});
send_ok(&mut stream, &resp.to_string());
}
_ => {
let req: ConsolidateRequest = match serde_json::from_value(msg) {
Ok(r) => r, Err(e) => {
send_error(&mut stream, "parse", &e.to_string()); return;
}
};
eprintln!("[zhiyi-ipc] full IPC msg keys: task={:?}, model_dir={:?}, epsilon={}",
req.task, req.model_dir, req.epsilon);
let run_args = Args {
mode: req.task.clone(),
data_dir: req.lancedb_path.clone(),
sqlite_path: req.sqlite_path.clone(),
llm_endpoint: req.llm_endpoint.clone(),
llm_model: req.llm_model.clone(),
llm_api_key: req.llm_api_key.clone(),
llm_budget: req.llm_budget,
socket: None,
model_dir: req.model_dir.clone().unwrap_or_default(),
text: None,
texts: None,
epsilon: req.epsilon,
min_points: req.min_points,
};
// 风暴守卫(t_89d22fc8 AC2①):整合独占槽。在途时立即拒绝,不排队、
// 不并起第二个 DBSCAN 线程(22:07 三连发 + 35min 内存压力 ⇒ OOM 的根因)。
// Go 侧收到 consolidate_busy 会即时降级 Go 启发式(既有行为),不会等 10min 超时。
let _guard = match ConsolidatingGuard::try_acquire() {
Some(g) => Some(g),
None => {
eprintln!("[zhiyi-ipc] consolidate REJECTED (busy): task={:?} — another consolidation in progress (storm guard)", req.task);
send_error(&mut stream, "consolidate_busy",
"another consolidation is in progress; request dropped, will retry next trigger cycle");
trim_only();
return;
}
};
let report = run_consolidation(&run_args, &lancedb);
drop(_guard);
send_ok(&mut stream, &serde_json::to_string(&report).unwrap_or_default());
}
}
// 每个 IPC 请求处理完把已释放的堆还给 OS(治本:长驻进程不累积)
trim_only();
}
fn send_ok(stream: &mut UnixStream, json: &str) {
let resp = ConsolidateResponse { status: "ok".into(), report_json: json.to_string(), failure_step: String::new(), error_detail: String::new(), truncated: false };
send_response(stream, &resp);
}
/// 方案②(t_576714de):被行数上限截断的正常返回(report_json=部分结果,truncated=true)。
fn send_ok_truncated(stream: &mut UnixStream, json: &str) {
let resp = ConsolidateResponse { status: "ok".into(), report_json: json.to_string(), failure_step: String::new(), error_detail: String::new(), truncated: true };
send_response(stream, &resp);
}
fn send_error(stream: &mut UnixStream, step: &str, detail: &str) {
let resp = ConsolidateResponse { status: "error".into(), report_json: String::new(), failure_step: step.to_string(), error_detail: detail.to_string(), truncated: false };
send_response(stream, &resp);
}
fn send_response(stream: &mut UnixStream, resp: &ConsolidateResponse) {
let json = serde_json::to_vec(resp).unwrap_or_default();
let len = (json.len() as u32).to_be_bytes();
stream.write_all(&len).ok();
stream.write_all(&json).ok();
}
/// 执行完整整合流程
fn run_consolidation(args: &Args, lancedb: &LanceDBOps) -> ConsolidationReport {
let start = Instant::now();
let mut clusters_found = 0usize;
let mut noise_points = 0usize;
let mut duplicate_pairs = 0usize;
let mut pruned_stats: Option<report::PruneStats> = None;
let mut new_decay_rates: HashMap<String, f64> = HashMap::new();
let mut r_squared_values: HashMap<String, f64> = HashMap::new();
let mut quality_score = 0.0f64;
let mut low_info = 0usize;
let mut hallucinations = 0usize;
// Step 1: DBSCAN 聚类 — 从 LanceDB 加载向量
if args.mode == "full" || args.mode == "cluster_only" {
let clusterer = Clusterer::new(args.epsilon, args.min_points);
// 从 LanceDB 加载所有记忆向量(用零向量搜索,高 limit)
let zero_vec = vec![0.0f32; 1024];
let mut ids: Vec<String>;
let mut vectors: Vec<Vec<f32>>;
let mut all_records: Vec<MemoryRecord> = Vec::new();
match lancedb.search(&zero_vec, 10000, None) {
Ok(records) => {
all_records = records;
}
Err(e) => {
eprintln!("[consolidate] Step 1: failed to load vectors from LanceDB: {}", e);
}
}
ids = all_records.iter().map(|r| r.id.clone()).collect();
vectors = all_records.iter().map(|r| r.vector.clone()).collect();
// 检查向量是否为全零(Python migration 写入格式错误导致 LanceDB 读回 null)
// 如果 BGE HTTP 可用则重新编码,否则跳过
let zero_count = vectors.iter().filter(|v| v.iter().all(|x| x.abs() < 1e-6)).count();
if zero_count > 0 {
eprintln!("[consolidate] WARNING: {} / {} vectors are zero (LanceDB format issue)", zero_count, vectors.len());
}
eprintln!("[consolidate] DBSCAN: eps={}, min_pts={}, vectors={}", args.epsilon, args.min_points, vectors.len());
// Debug: check vector norms and sample distances
if !vectors.is_empty() {
let mut norm_sum = 0.0f64;
let mut norm_min = f64::MAX;
let mut norm_max = 0.0f64;
for v in &vectors {
let s = (v.iter().map(|x| (*x as f64) * (*x as f64)).sum::<f64>()).sqrt();
norm_sum += s;
if s < norm_min { norm_min = s; }
if s > norm_max { norm_max = s; }
}
let avg_norm = norm_sum / vectors.len() as f64;
eprintln!("[consolidate] vector norms: min={:.4} max={:.4} avg={:.4}", norm_min, norm_max, avg_norm);
// Sample pairwise distances among first 20 vectors
let n = vectors.len().min(20);
let mut all_dists: Vec<f64> = Vec::new();
for i in 0..n {
for j in (i+1)..n {
let d = (vectors[i].iter().zip(&vectors[j]).map(|(a,b)| (a-b).powi(2)).sum::<f32>() as f64).sqrt();
all_dists.push(d);
}
}
all_dists.sort_by(|a,b| a.partial_cmp(b).unwrap());
if !all_dists.is_empty() {
let len = all_dists.len();
eprintln!("[consolidate] sample distances ({} vecs, {} pairs): p5={:.3} p50={:.3} p95={:.3} min={:.3} max={:.3}",
n, len, all_dists[(0.05 * len as f64) as usize].min(all_dists[len-1]),
all_dists[(len/2)],
all_dists[((len-1) as f64 * 0.95) as usize].min(all_dists[len-1]),
all_dists[0], all_dists[len-1]);
}
}
match clusterer.cluster(&vectors, &ids) {
Ok(result) => {
clusters_found = result.num_clusters;
noise_points = result.num_noise;
if !vectors.is_empty() {
let centroids = Clusterer::extract_centroids(&vectors, &result.labels, result.num_clusters);
let sizes = result.cluster_sizes.clone();
duplicate_pairs = Clusterer::find_duplicates(&centroids, &sizes).len();
}
}
Err(e) => {
eprintln!("[consolidate] Step 1 (cluster) failed: {}", e);
}
}
}
reclaim("Step 1 后(聚类)");
// Step 2: 图谱修剪
if args.mode == "full" || args.mode == "prune_only" {
let pruner = GraphPruner::new(&args.sqlite_path);
match pruner.prune() {
Ok(stats) => {
pruned_stats = Some(report::PruneStats {
isolated_nodes_removed: stats.isolated_nodes_removed,
low_weight_edges_removed: stats.low_weight_edges_removed,
redundant_edges_merged: stats.redundant_edges_merged,
nodes_before: stats.nodes_before,
nodes_after: stats.nodes_after,
edges_before: stats.edges_before,
edges_after: stats.edges_after,
});
}
Err(e) => {
eprintln!("[consolidate] Step 2 (prune) failed: {}", e);
}
}
}
reclaim("Step 2 后(图谱修剪)");
// Step 3: 衰减校准
if args.mode == "full" {
let calibrator = DecayCalibrator::new();
let old_rates = decay_calibrate::default_decay_rates();
// 整合内部全量扫描(非 IPC 边界,需全量;行数上限只作用于 IPC 单次重读,t_576714de)。
let all_records = lancedb.scan_all(200_000).unwrap_or_default();
let now_secs = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_secs() as f64;
let samples: Vec<DecaySample> = all_records
.iter()
.filter_map(|r| {
let created = chrono::DateTime::parse_from_rfc3339(&r.created_at).ok()
.map(|dt| dt.timestamp() as f64)
.unwrap_or(0.0);
let days = if created > 0.0 { (now_secs - created) / 86400.0 } else { 0.0 };
if days < 0.0 { return None; }
Some(DecaySample {
category: if r.category.is_empty() { "general".into() } else { r.category.clone() },
days_old: days,
current_score: r.quality_score as f64,
original_score: r.importance as f64,
})
})
.collect();
eprintln!("[consolidate] Step 3: {} decay samples from LanceDB", samples.len());
let result = calibrator.calibrate(&samples, &old_rates);
new_decay_rates = result.rates;
r_squared_values = result.r_squared;
}
reclaim("Step 3 后(衰减校准)");
// Step 4: 质量回溯 (需要 BGE HTTP 服务在 localhost:8000)
if args.mode == "full" && !args.llm_endpoint.is_empty() {
// 先检查 BGE HTTP 是否可达 (2秒超时)
let bge_ok = std::net::TcpStream::connect_timeout(
&std::net::SocketAddr::from(([127, 0, 0, 1], 8000)),
std::time::Duration::from_secs(2),
).is_ok();
if !bge_ok {
eprintln!("[consolidate] Step 4: BGE HTTP (port 8000) unreachable — skipping quality backtrace");
} else {
let backtracer = QualityBacktracer::new(&args.llm_endpoint, &args.llm_model, &args.llm_api_key);
// 从 LanceDB 加载(content = distilled, source metadata = original)。整合内部全量,非 IPC 边界。
let all_records = lancedb.scan_all(200_000).unwrap_or_default();
let samples: Vec<QualitySample> = all_records
.iter()
.filter(|r| !r.content.is_empty())
.map(|r| QualitySample {
memory_id: r.id.clone(),
distilled_content: r.content.clone(),
original_episode: if !r.source.is_empty() { r.source.clone() } else { r.content.clone() },
category: if r.category.is_empty() { "general".into() } else { r.category.clone() },
tier: if r.freshness.is_empty() { "fresh".into() } else { r.freshness.clone() },
})
.collect();
eprintln!(
"[consolidate] Step 4: {} quality samples from LanceDB",
samples.len()
);
let sample_20 = quality_backtrace::stratified_sample(&samples, 5, 5, 5, 5);
// 用本地 BGE HTTP 服务编码
let encoder = match BGEEncoder::new(&args.model_dir) {
Ok(e) => Some(e),
Err(e) => {
eprintln!("[consolidate] Step 4: BGE encoder init failed: {} — skipping quality backtrace", e);
None
}
};
if let Some(ref enc) = encoder {
let embed_fn = &|text: &str| -> Result<Vec<f32>, Box<dyn std::error::Error>> {
enc.encode(text)
};
match backtracer.backtrace(&sample_20, embed_fn) {
Ok(result) => {
quality_score = result.score;
low_info = result.low_info_loss;
hallucinations = result.hallucinations;
}
Err(e) => {
eprintln!("[consolidate] Step 4 (quality) failed: {}", e);
}
}
}
}
}
// Step 5: 生成报告
reclaim("Step 4 后(质量回溯)");
let dashboard = DashboardSnapshot {
recall_useful_rate: 0.85,
recall_hit_rate: 0.90,
gap_closure_rate: 0.75,
cascade_propagation_rate: 0.60,
deprecation_rate: 3.0,
distill_loss_rate: quality_score.max(0.0),
auto_resolve_rate: 0.70,
};
let gen = ReportGenerator::new();
let mut report = gen.generate(
clusters_found,
noise_points,
duplicate_pairs,
pruned_stats,
new_decay_rates,
r_squared_values,
quality_score,
low_info,
hallucinations,
&dashboard,
);
report.duration_secs = start.elapsed().as_secs_f64();
eprintln!(
"[consolidate] completed in {:.1}s: mode={}, clusters={}, quality={:.3}",
report.duration_secs, args.mode, clusters_found, quality_score
);
report
}