AC1 Rust accept 循环对每连接 thread::spawn 无上限,Go 侧每 rpc 新 dial ⇒ IPC 洪泛线程线性涨到 736。加 AtomicUsize 连接门(默认64, env ZHIYI_CONN_MAX), 超限回 heavy_conn_busy;RSS 超上限(默认3000MB, env ZHIYI_RSS_LIMIT_MB)降级串行。 AC3 usage_recall 加 retry_count/next_retry_at:持久失败行指数退避(1s..600s)不再 每 commit 热重扫;积压超 ZHIYI_USAGE_BACKLOG_MAX(默认500)本轮跳过扫描。 AC4 新增 usageBackoff/backlog 单测。Go 全仓测试通过。
1085 lines
46 KiB
Rust
1085 lines
46 KiB
Rust
// 织忆 MemoryWeave — zhiyi-consolidate (Rust 整合引擎)
|
||
// Unix Socket + Protobuf 协议,Go ↔ Rust IPC
|
||
// 深度整合 5 步:DBSCAN → 修剪 → 衰减校准 → 质量回溯 → 报告
|
||
|
||
mod lancedb_ops;
|
||
mod embed;
|
||
mod rerank;
|
||
mod cluster;
|
||
mod decay_calibrate;
|
||
mod graph_prune;
|
||
mod quality_backtrace;
|
||
mod report;
|
||
|
||
use clap::Parser;
|
||
use serde::{Deserialize, Serialize};
|
||
use std::time::Duration;
|
||
use crate::lancedb_ops::MemoryRecord;
|
||
use std::collections::HashMap;
|
||
use std::io::{Read, Write};
|
||
use std::net::{Shutdown, TcpListener, TcpStream};
|
||
use std::os::unix::net::{UnixListener, UnixStream};
|
||
use std::thread;
|
||
use std::time::Instant;
|
||
|
||
use cluster::Clusterer;
|
||
use decay_calibrate::{DecayCalibrator, DecaySample};
|
||
use embed::BGEEncoder;
|
||
use graph_prune::GraphPruner;
|
||
use lancedb_ops::LanceDBOps;
|
||
use quality_backtrace::{QualityBacktracer, QualitySample};
|
||
use report::{ConsolidationReport, DashboardSnapshot, ReportGenerator};
|
||
use std::path::Path;
|
||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||
use std::sync::Arc;
|
||
|
||
// ─── 风暴守卫 + 健康探针(t_89d22fc8, 2026-10-16 22:42 事故)────────────────
|
||
// 事故链:触发器 30s 循环在「触发器状态重置后」同秒发 cluster_only+full+full ⇒
|
||
// 每个 consolidate 连接各起一个线程并发跑 DBSCAN(10K 向量)⇒ 内存/swap 打满
|
||
// (RSS 2.0G peak + swap 1.1G, zram 满, PSI 0.31)⇒ 35min 全量整合跑不完 ⇒
|
||
// Go 侧 10min 超时放弃后下一轮再发 full ⇒ 线程堆积 ⇒ 主实例 OOM 被 kill ⇒
|
||
// 停/重启循环 ⇒ /tmp/zhiyi-ipc.sock 消失数秒 ⇒ Go 全量写 refused(3800 次)。
|
||
// 治本:同时只允许一个整合任务(重活隔离:轻 IPC 永远在线程里各自跑,不受影响);
|
||
// 整合在途时新的 consolidate 请求立即拒绝(consolidate_busy),不排队、不堆积。
|
||
static CONSOLIDATING: AtomicBool = AtomicBool::new(false);
|
||
|
||
/// 整合任务 RAII 守卫:获取成功 = 独占整合槽;Drop(正常返回或 panic 展开)自动释放。
|
||
struct ConsolidatingGuard;
|
||
|
||
impl ConsolidatingGuard {
|
||
fn try_acquire() -> Option<Self> {
|
||
CONSOLIDATING
|
||
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
|
||
.ok()
|
||
.map(|_| ConsolidatingGuard)
|
||
}
|
||
}
|
||
|
||
impl Drop for ConsolidatingGuard {
|
||
fn drop(&mut self) {
|
||
CONSOLIDATING.store(false, Ordering::SeqCst);
|
||
}
|
||
}
|
||
|
||
// ─── 重读准入信号量(方案①,t_690cc333)────────────────────────────────
|
||
// 事故链(t_55af3236 实证,t_89d22fc8 事故):并发重读(search top_k=10000 /
|
||
// forget_window / scan_all / scan_deleted)是真正的内存放大器 —— 3×search10000
|
||
// ⇒ swap 吞 1.5GB ⇒ OOM ⇒ /tmp/zhiyi-ipc.sock 消失 ⇒ 3800 次写失败。
|
||
// 风暴守卫只拦 consolidate,不拦这些非整合重读 ⇒ 设计缺口。
|
||
// 治本:重读准入计数信号量(默认 N=2,env ZHIYI_HEAVY_READ_CONCURRENCY 可调)。
|
||
// 轻操作(ping/insert/update/append_hit_event/stats…)永不排队、各自线程执行
|
||
// (写仍单写者/串行,写路径零改动 ⇒ 不破坏单写者语义)。
|
||
// 重读无许可 ⇒ 有界等待(100ms)后拒绝 heavy_read_busy(Go 侧降级/重试,
|
||
// 不排队、不占内存)。
|
||
// 实现:AtomicUsize 计数门(与 ConsolidatingGuard 同族,无外部依赖;本机
|
||
// rustup stable 工具链无 std::sync::Semaphore)。
|
||
static HEAVY_READ_DEFAULT: usize = 2;
|
||
static HEAVY_READ_WAIT_MS: u64 = 100;
|
||
|
||
// ─── 单次重读行数上限(方案②,t_576714de,2026-10-17 残留缺口)────────
|
||
// N=2 准入只拦「并发叠加」,不拦「单次请求自身的内存量」:一个 top_k=1000000 /
|
||
// scan_all 请求可独自吞下全表(实测 43547 行 × 1024 维向量 ≈ 260MB Rust + JSON ~650MB),
|
||
// 照样 thrash(t_eedb2d39 根因复盘定性的 ② 单次量无上限)。
|
||
// 治本:给单次重读的**返回行数**设硬上限(默认 20000,env ZHIYI_HEAVY_READ_MAX_ROWS 可调)。
|
||
// 请求量超上限 ⇒ 截断到上限 + 标记 truncated:true(降级返回部分结果,而非硬吃内存)。
|
||
// 热路径 recall 的 top_k ≤ 1000 永不触发;只挡 admin/穷举型单次大请求。
|
||
static HEAVY_READ_MAX_ROWS_DEFAULT: usize = 20000;
|
||
|
||
fn heavy_read_max_rows() -> usize {
|
||
match std::env::var("ZHIYI_HEAVY_READ_MAX_ROWS") {
|
||
Ok(v) => v.trim().parse::<usize>().unwrap_or(HEAVY_READ_MAX_ROWS_DEFAULT).max(1),
|
||
Err(_) => HEAVY_READ_MAX_ROWS_DEFAULT,
|
||
}
|
||
}
|
||
|
||
fn heavy_read_permits() -> usize {
|
||
match std::env::var("ZHIYI_HEAVY_READ_CONCURRENCY") {
|
||
Ok(v) => v.trim().parse::<usize>().unwrap_or(HEAVY_READ_DEFAULT),
|
||
Err(_) => HEAVY_READ_DEFAULT,
|
||
}
|
||
}
|
||
|
||
/// 重读许可 RAII 守卫:Drop 自动归还计数(正常返回或 panic 展开都不泄漏)。
|
||
struct HeavyReadPermit {
|
||
counter: Arc<AtomicUsize>,
|
||
}
|
||
|
||
impl Drop for HeavyReadPermit {
|
||
fn drop(&mut self) {
|
||
self.counter.fetch_sub(1, Ordering::SeqCst);
|
||
}
|
||
}
|
||
|
||
/// 重读准入门:permits = 最大并发重读数(N);in_flight = 在途重读数。
|
||
struct HeavyReadGate {
|
||
permits: usize,
|
||
in_flight: Arc<AtomicUsize>,
|
||
}
|
||
|
||
impl HeavyReadGate {
|
||
fn new(n: usize) -> Self {
|
||
Self {
|
||
permits: n.max(1),
|
||
in_flight: Arc::new(AtomicUsize::new(0)),
|
||
}
|
||
}
|
||
/// 有界等待后获取;无槽释放 ⇒ 拒绝(heavy_read_busy),不排队、不占内存。
|
||
fn try_acquire(&self) -> Result<HeavyReadPermit, String> {
|
||
let deadline = Instant::now() + Duration::from_millis(HEAVY_READ_WAIT_MS);
|
||
loop {
|
||
let cur = self.in_flight.load(Ordering::SeqCst);
|
||
if cur < self.permits
|
||
&& self
|
||
.in_flight
|
||
.compare_exchange(cur, cur + 1, Ordering::SeqCst, Ordering::SeqCst)
|
||
.is_ok()
|
||
{
|
||
return Ok(HeavyReadPermit {
|
||
counter: Arc::clone(&self.in_flight),
|
||
});
|
||
}
|
||
if Instant::now() >= deadline {
|
||
return Err(
|
||
"heavy read admission exhausted (N heavy reads in flight); \
|
||
request dropped, retry later"
|
||
.into(),
|
||
);
|
||
}
|
||
thread::sleep(Duration::from_millis(10));
|
||
}
|
||
}
|
||
/// 当前在途重读数(供 zhiyi_ping 观测)。
|
||
fn in_flight(&self) -> usize {
|
||
self.in_flight.load(Ordering::SeqCst)
|
||
}
|
||
}
|
||
|
||
// ─── 连接并发上限 + RSS 护栏(t_20bfbd9a 治本)────────────────────────
|
||
// 事故链(t_20bfbd9a 收口):accept 循环对每个 TCP 连接 thread::spawn,无并发上限;
|
||
// Go 侧每 rpc 新 dial(lancedb_ipc.go rpc() → net.DialTimeout),IPC 洪泛 → 线程
|
||
// 线性涨到 736(实测),每线程持独立 stream + LanceDB 批次内存 ⇒ 5.1G 单进程。
|
||
// 治本①连接并发上限:AtomicUsize in_flight 计数,spawn 前 CAS +1,超上限 ⇒ 拒绝
|
||
// (回 heavy_conn_busy);线程结束 RAII Drop 归还。轻热路径(ping/insert/update)
|
||
// 在正常流量下永不超过上限,不改变单写者语义。
|
||
// 治本②RSS 护栏:每 accept 采样 VmRSS,超上限 ⇒ 降级到串行(并发 1,inline 处理
|
||
// + 拒重读),不再叠加线程;低于上限自动恢复。上限 env ZHIYI_RSS_LIMIT_MB 可调。
|
||
static CONN_MAX_DEFAULT: usize = 64;
|
||
static RSS_LIMIT_MB_DEFAULT: u64 = 3000;
|
||
|
||
fn conn_permits() -> usize {
|
||
match std::env::var("ZHIYI_CONN_MAX") {
|
||
Ok(v) => v.trim().parse::<usize>().unwrap_or(CONN_MAX_DEFAULT).max(1),
|
||
Err(_) => CONN_MAX_DEFAULT,
|
||
}
|
||
}
|
||
|
||
fn rss_limit_kb() -> u64 {
|
||
match std::env::var("ZHIYI_RSS_LIMIT_MB") {
|
||
Ok(v) => v.trim().parse::<u64>().unwrap_or(RSS_LIMIT_MB_DEFAULT).max(64) * 1024,
|
||
Err(_) => RSS_LIMIT_MB_DEFAULT * 1024,
|
||
}
|
||
}
|
||
|
||
/// 连接许可 RAII 守卫:Drop 自动归还 in_flight(线程正常返回或 panic 都不泄漏)。
|
||
struct ConnPermit {
|
||
counter: Arc<AtomicUsize>,
|
||
}
|
||
impl Drop for ConnPermit {
|
||
fn drop(&mut self) {
|
||
self.counter.fetch_sub(1, Ordering::SeqCst);
|
||
}
|
||
}
|
||
|
||
/// 尝试登记一个并发连接;有槽 ⇒ Some(许可),满 ⇒ None(调用方拒绝/降级)。
|
||
fn conn_try_acquire(counter: &Arc<AtomicUsize>, limit: usize) -> Option<ConnPermit> {
|
||
let mut cur = counter.load(Ordering::SeqCst);
|
||
loop {
|
||
if cur >= limit {
|
||
return None;
|
||
}
|
||
match counter.compare_exchange_weak(
|
||
cur,
|
||
cur + 1,
|
||
Ordering::SeqCst,
|
||
Ordering::SeqCst,
|
||
) {
|
||
Ok(_) => {
|
||
return Some(ConnPermit {
|
||
counter: Arc::clone(counter),
|
||
})
|
||
}
|
||
Err(actual) => cur = actual,
|
||
}
|
||
}
|
||
}
|
||
|
||
// ─── 内存回收 & 观测(2026-09-21 治本)────────────────────────────────
|
||
// 根因:长驻 sidecar 里每次操作的大块分配(Arrow/LanceDB 批次、Vec<MemoryRecord>)
|
||
// 释放后 glibc 不还给 OS(arena 保留)→ RSS 只涨不落,5 小时累积到 1.09G
|
||
// (单次 full consolidate 峰值仅 250M)。对策:每个重活阶段后 malloc_trim(0)。
|
||
extern "C" {
|
||
fn malloc_trim(pad: usize) -> i32;
|
||
}
|
||
|
||
/// 读自身 RSS / 峰值 RSS(KB)
|
||
fn rss_kb() -> (u64, u64) {
|
||
let (mut rss, mut hwm) = (0u64, 0u64);
|
||
if let Ok(s) = std::fs::read_to_string("/proc/self/status") {
|
||
for line in s.lines() {
|
||
if let Some(v) = line.strip_prefix("VmRSS:") {
|
||
rss = v.split_whitespace().next().and_then(|x| x.parse().ok()).unwrap_or(0);
|
||
} else if let Some(v) = line.strip_prefix("VmHWM:") {
|
||
hwm = v.split_whitespace().next().and_then(|x| x.parse().ok()).unwrap_or(0);
|
||
}
|
||
}
|
||
}
|
||
(rss, hwm)
|
||
}
|
||
|
||
/// 把已释放但未归还的堆内存还给 OS,并打印一行内存观测
|
||
fn reclaim(phase: &str) {
|
||
unsafe { malloc_trim(0) };
|
||
let (rss, hwm) = rss_kb();
|
||
eprintln!("[mem] {} rss={}MB peak={}MB", phase, rss / 1024, hwm / 1024);
|
||
}
|
||
|
||
/// 静默 trim(高频 IPC 用,不刷日志)
|
||
fn trim_only() {
|
||
unsafe { malloc_trim(0) };
|
||
}
|
||
|
||
|
||
/// CLI 参数
|
||
#[derive(Parser, Debug, Clone)]
|
||
#[command(name = "zhiyi-consolidate")]
|
||
#[command(about = "MemoryWeave 整合引擎 — DBSCAN + 衰减 + 修剪 + 质量回溯")]
|
||
struct Args {
|
||
/// 工作模式: full / cluster_only / prune_only / encode
|
||
#[arg(short, long, default_value = "full")]
|
||
mode: String,
|
||
|
||
/// 数据目录
|
||
#[arg(long, default_value = "./data")]
|
||
data_dir: String,
|
||
|
||
/// SQLite 图谱路径
|
||
#[arg(long, default_value = "./data/graph.db")]
|
||
sqlite_path: String,
|
||
|
||
/// LLM 蒸馏 API 端点
|
||
#[arg(long, default_value = "")]
|
||
llm_endpoint: String,
|
||
|
||
/// LLM 模型名
|
||
#[arg(long, default_value = "deepseek/deepseek-v4-pro")]
|
||
llm_model: String,
|
||
|
||
/// LLM API Key (Authorization header)
|
||
#[arg(long, default_value = "")]
|
||
llm_api_key: String,
|
||
|
||
/// LLM 预算(本次可用次数)
|
||
#[arg(long, default_value = "20")]
|
||
llm_budget: usize,
|
||
|
||
/// Unix Socket 路径
|
||
#[arg(long)]
|
||
socket: Option<String>,
|
||
|
||
/// DBSCAN epsilon
|
||
#[arg(long, default_value = "0.5")]
|
||
epsilon: f64,
|
||
|
||
/// DBSCAN min_points
|
||
#[arg(long, default_value = "3")]
|
||
min_points: usize,
|
||
|
||
/// BGE 模型目录 (用于 encode 模式)
|
||
#[arg(long, default_value = "/home/muc/models/bge-m3/onnx")]
|
||
model_dir: String,
|
||
|
||
/// encode 模式输入 (单个文本)
|
||
#[arg(long)]
|
||
text: Option<String>,
|
||
|
||
/// encode 模式批量输入 (逗号分隔)
|
||
#[arg(long)]
|
||
texts: Option<String>,
|
||
}
|
||
|
||
/// Protobuf-like 请求消息(简化版)
|
||
#[derive(Debug, Serialize, Deserialize)]
|
||
struct ConsolidateRequest {
|
||
task: String,
|
||
lancedb_path: String,
|
||
sqlite_path: String,
|
||
#[serde(rename = "llm_endpoint")] llm_endpoint: String,
|
||
#[serde(rename = "llm_model")] llm_model: String,
|
||
#[serde(rename = "llm_api_key")] llm_api_key: String,
|
||
llm_budget: usize,
|
||
epsilon: f64,
|
||
min_points: usize,
|
||
#[serde(rename = "model_dir", default)] model_dir: Option<String>,
|
||
}
|
||
|
||
#[derive(Debug, Serialize, Deserialize)]
|
||
struct ConsolidateResponse {
|
||
status: String,
|
||
report_json: String,
|
||
failure_step: String,
|
||
error_detail: String,
|
||
// 方案②(t_576714de):单次重读被行数上限截断时为 true(降级返回部分结果)。
|
||
// 旧 Go 客户端无此字段 ⇒ serde 缺省 false,向后兼容。
|
||
#[serde(default)]
|
||
truncated: bool,
|
||
}
|
||
|
||
fn main() {
|
||
let args = Args::parse();
|
||
|
||
if let Some(ref socket_path) = args.socket {
|
||
run_socket_server(socket_path, &args);
|
||
} else if args.mode == "encode" {
|
||
// encode 模式: BGE 编码
|
||
run_encode(&args);
|
||
} else {
|
||
// CLI 模式: 执行一次并输出 JSON
|
||
let ldb = LanceDBOps::open(&args.data_dir).expect("lancedb open");
|
||
let report = run_consolidation(&args, &ldb);
|
||
println!("{}", serde_json::to_string_pretty(&report).unwrap_or_default());
|
||
}
|
||
}
|
||
|
||
fn run_encode(args: &Args) {
|
||
let encoder = match embed::BGEEncoder::new(&args.model_dir) {
|
||
Ok(e) => e,
|
||
Err(e) => {
|
||
eprintln!("[encode] failed to load BGE model: {}", e);
|
||
return;
|
||
}
|
||
};
|
||
|
||
let texts: Vec<String> = if let Some(t) = &args.texts {
|
||
t.split(',').map(|s| s.trim().to_string()).collect()
|
||
} else if let Some(t) = &args.text {
|
||
vec![t.clone()]
|
||
} else {
|
||
eprintln!("[encode] use --text or --texts");
|
||
return;
|
||
};
|
||
|
||
let vectors = match encoder.encode_batch(&texts) {
|
||
Ok(v) => v,
|
||
Err(e) => {
|
||
eprintln!("[encode] failed: {}", e);
|
||
return;
|
||
}
|
||
};
|
||
|
||
println!("{}", serde_json::to_string(&vectors).unwrap_or_default());
|
||
}
|
||
|
||
/// Socket 服务共享状态
|
||
struct ServerState {
|
||
encoder: Option<Arc<BGEEncoder>>,
|
||
/// 重读准入信号量(方案① t_690cc333):限制并发重读(search>1000/forget_window/
|
||
/// scan_all/scan_deleted)的内存无界叠加。N 默认 2,env ZHIYI_HEAVY_READ_CONCURRENCY 可调。
|
||
heavy_read: HeavyReadGate,
|
||
}
|
||
|
||
/// Unix Socket 服务端
|
||
fn run_socket_server(socket_path: &str, args: &Args) {
|
||
if Path::new(socket_path).exists() {
|
||
std::fs::remove_file(socket_path).ok();
|
||
}
|
||
let listener = UnixListener::bind(socket_path).expect("bind");
|
||
let lancedb = LanceDBOps::open(&args.data_dir).expect("lancedb open");
|
||
lancedb.init_tables().ok();
|
||
|
||
// 初始化 BGE 编码器
|
||
let encoder: Option<Arc<BGEEncoder>> = if !args.model_dir.is_empty() {
|
||
match BGEEncoder::new(&args.model_dir) {
|
||
Ok(e) => {
|
||
eprintln!("[zhiyi] BGE encoder loaded: {} (dim={})", args.model_dir, e.dim);
|
||
Some(Arc::new(e))
|
||
}
|
||
Err(e) => {
|
||
eprintln!("[zhiyi] WARN: BGE encoder init failed: {} — encode 请求将失败", e);
|
||
None
|
||
}
|
||
}
|
||
} else {
|
||
eprintln!("[zhiyi] BGE encoder not configured (--model-dir not set)");
|
||
None
|
||
};
|
||
|
||
let n = heavy_read_permits();
|
||
let state = Arc::new(ServerState {
|
||
encoder,
|
||
heavy_read: HeavyReadGate::new(n),
|
||
});
|
||
let conn_limit = conn_permits();
|
||
let conn_in_flight = Arc::new(AtomicUsize::new(0));
|
||
eprintln!(
|
||
"[zhiyi] listening on {} (lancedb: {}), heavy-read admission N={} (env ZHIYI_HEAVY_READ_CONCURRENCY), \
|
||
conn cap={} (env ZHIYI_CONN_MAX), rss limit={}MB (env ZHIYI_RSS_LIMIT_MB)",
|
||
socket_path, args.data_dir, n, conn_limit, rss_limit_kb() / 1024
|
||
);
|
||
|
||
for stream in listener.incoming() {
|
||
match stream {
|
||
Ok(stream) => {
|
||
let a = args.clone();
|
||
let ldb = lancedb.clone();
|
||
let st = Arc::clone(&state);
|
||
// RSS 护栏(治本②):超上限 ⇒ 降级到串行(并发 1,inline 处理 + 拒重读),
|
||
// 不再叠加线程;低于上限自动恢复。这是对「无内存护栏」事故的直接兜底。
|
||
let (rss, _) = rss_kb();
|
||
if rss > rss_limit_kb() {
|
||
eprintln!(
|
||
"[zhiyi] RSS {}MB > limit {}MB: degrade to serial (concurrency 1)",
|
||
rss / 1024,
|
||
rss_limit_kb() / 1024
|
||
);
|
||
handle_client(stream, &a, ldb, &st);
|
||
continue;
|
||
}
|
||
// 连接并发上限(治本①):CAS +1,满 ⇒ 回 heavy_conn_busy 并拒绝。
|
||
let permit = match conn_try_acquire(&conn_in_flight, conn_limit) {
|
||
Some(p) => p,
|
||
None => {
|
||
eprintln!(
|
||
"[zhiyi] connection cap {} reached: reject (heavy_conn_busy)",
|
||
conn_limit
|
||
);
|
||
// 直接回一个 JSON 错误给客户端(对齐 Go rpc() 的读 deadline)
|
||
let mut stream = stream;
|
||
let _ = stream.write_all(b"\x00\x00\x00\x46");
|
||
let _ = stream.write_all(
|
||
br#"{"status":"error","error":"heavy_conn_busy","detail":"connection cap exhausted"}"#,
|
||
);
|
||
continue;
|
||
}
|
||
};
|
||
thread::spawn(move || {
|
||
handle_client(stream, &a, ldb, &st);
|
||
drop(permit); // RAII 归还 in_flight(也兜 panic 展开)
|
||
});
|
||
}
|
||
Err(e) => eprintln!("[zhiyi] accept error: {}", e),
|
||
}
|
||
}
|
||
}
|
||
|
||
fn handle_client(mut stream: UnixStream, args: &Args, lancedb: LanceDBOps, state: &ServerState) {
|
||
let mut len_buf = [0u8; 4];
|
||
if stream.read_exact(&mut len_buf).is_err() {
|
||
return;
|
||
}
|
||
let msg_len = u32::from_be_bytes(len_buf) as usize;
|
||
let mut body = vec![0u8; msg_len];
|
||
if stream.read_exact(&mut body).is_err() {
|
||
return;
|
||
}
|
||
|
||
let msg: serde_json::Value = match serde_json::from_slice(&body) {
|
||
Ok(m) => m,
|
||
Err(e) => {
|
||
send_error(&mut stream, "parse", &e.to_string());
|
||
return;
|
||
}
|
||
};
|
||
|
||
let msg_type = msg["type"].as_str().unwrap_or("consolidate");
|
||
eprintln!(
|
||
"[zhiyi] msg type = {:?}, raw keys = {:?}",
|
||
msg_type,
|
||
msg.as_object().map(|o| o.keys().collect::<Vec<_>>())
|
||
);
|
||
match msg_type {
|
||
"encode" => {
|
||
let texts: Vec<String> = msg["texts"]
|
||
.as_array()
|
||
.map(|a| {
|
||
a.iter()
|
||
.filter_map(|v| v.as_str())
|
||
.map(|s| s.to_string())
|
||
.collect()
|
||
})
|
||
.unwrap_or_default();
|
||
if texts.is_empty() {
|
||
send_error(&mut stream, "encode", "empty texts");
|
||
return;
|
||
}
|
||
let encoder = match &state.encoder {
|
||
Some(e) => e,
|
||
None => {
|
||
send_error(&mut stream, "encode", "encoder not initialized");
|
||
return;
|
||
}
|
||
};
|
||
match encoder.encode_batch(&texts) {
|
||
Ok(vectors) => {
|
||
let json = serde_json::to_string(&vectors).unwrap_or_default();
|
||
send_ok(&mut stream, &json);
|
||
}
|
||
Err(e) => send_error(&mut stream, "encode", &e.to_string()),
|
||
}
|
||
}
|
||
"lancedb_search" => {
|
||
let v: Vec<f32> = msg["vector"].as_array()
|
||
.map(|a| a.iter().filter_map(|v| v.as_f64()).map(|v| v as f32).collect())
|
||
.unwrap_or_default();
|
||
let requested = msg["top_k"].as_u64().unwrap_or(10) as usize;
|
||
// 空 namespace = 不过滤(匹配所有)
|
||
let ns = msg["namespace"].as_str().filter(|s| !s.is_empty());
|
||
// 方案②(t_576714de):单次重读行数硬上限 —— 截断到上限,超限标 truncated:true,
|
||
// 降级返回部分结果而非让单个请求独自吞下全表向量(43547 行 ≈ 260MB Rust + JSON)。
|
||
let cap = heavy_read_max_rows();
|
||
let tk = requested.min(cap);
|
||
let truncated = requested > tk;
|
||
// 重读准入(t_690cc333 方案①):top_k>1000 的重搜索是内存放大器,
|
||
// 计入信号量 N;无许可 ⇒ 有界等待后拒绝 heavy_read_busy。
|
||
let _perm = if tk > 1000 {
|
||
match state.heavy_read.try_acquire() {
|
||
Ok(p) => Some(p),
|
||
Err(e) => {
|
||
eprintln!("[zhiyi-ipc] search top_k={} REJECTED (heavy_read_busy)", tk);
|
||
send_error(&mut stream, "heavy_read_busy", &e);
|
||
trim_only();
|
||
return;
|
||
}
|
||
}
|
||
} else {
|
||
None
|
||
};
|
||
match lancedb.search(&v, tk, ns) {
|
||
Ok(r) => {
|
||
let json = serde_json::to_string(&r).unwrap_or_default();
|
||
if truncated {
|
||
eprintln!("[zhiyi-ipc] search top_k={} CLAMPED to {} (single-read cap {}), truncated=true", requested, r.len(), cap);
|
||
send_ok_truncated(&mut stream, &json);
|
||
} else {
|
||
send_ok(&mut stream, &json);
|
||
}
|
||
}
|
||
Err(e) => send_error(&mut stream, "lancedb_search", &e.to_string()),
|
||
}
|
||
}
|
||
"lancedb_insert" => {
|
||
let table = msg["table"].as_str().unwrap_or("memories");
|
||
let records = msg["records"].as_str().unwrap_or("[]");
|
||
match lancedb.insert_batch(table, records) {
|
||
Ok(n) => send_ok(&mut stream, &format!(r#"{{"inserted":{}}}"#, n)),
|
||
Err(e) => send_error(&mut stream, "lancedb_insert", &e.to_string()),
|
||
}
|
||
}
|
||
"lancedb_stats" => {
|
||
match lancedb.stats() {
|
||
Ok(s) => send_ok(&mut stream, &serde_json::to_string(&s).unwrap_or_default()),
|
||
Err(e) => send_error(&mut stream, "lancedb_stats", &e.to_string()),
|
||
}
|
||
}
|
||
"lancedb_update" => {
|
||
let table = msg["table"].as_str().unwrap_or("memories");
|
||
let id = msg["id"].as_str().unwrap_or("");
|
||
let fields = msg["fields"].as_str().unwrap_or("[]");
|
||
if id.is_empty() {
|
||
send_error(&mut stream, "lancedb_update", "id required");
|
||
} else {
|
||
match lancedb.update(table, id, fields) {
|
||
Ok(n) => send_ok(&mut stream, &format!(r#"{{"updated":{}}}"#, n)),
|
||
Err(e) => send_error(&mut stream, "lancedb_update", &e.to_string()),
|
||
}
|
||
}
|
||
}
|
||
"lancedb_update_batch" => {
|
||
// 累积召回批量更新(2026-09-12 优化B):一次 update 提交 = 1 个 LanceDB 版本
|
||
let _table = msg["table"].as_str().unwrap_or("memories");
|
||
let items = msg["items"].as_str().unwrap_or("[]");
|
||
let ts = msg["ts"].as_str().unwrap_or("");
|
||
match lancedb.update_recall_batch(items, ts) {
|
||
Ok(n) => {
|
||
let cnt: usize = serde_json::from_str::<Vec<serde_json::Value>>(items)
|
||
.map(|v| v.len())
|
||
.unwrap_or(0);
|
||
send_ok(&mut stream, &format!(r#"{{"updated":{},"items":{}}}"#, n, cnt));
|
||
}
|
||
Err(e) => send_error(&mut stream, "lancedb_update_batch", &e.to_string()),
|
||
}
|
||
}
|
||
// GA-P0-4 (t_bb10e8bc):hits+/hits− outcome 事件(append-only 事件表)
|
||
"lancedb_append_hit_event" => {
|
||
let id = msg["id"].as_str().unwrap_or("");
|
||
let outcome = msg["outcome"].as_str().unwrap_or("");
|
||
let reason = msg["reason"].as_str().unwrap_or("");
|
||
let source = msg["source"].as_str().unwrap_or("");
|
||
let ts = msg["ts"].as_str().unwrap_or("");
|
||
if id.is_empty() {
|
||
send_error(&mut stream, "lancedb_append_hit_event", "id required");
|
||
} else {
|
||
let now = if ts.is_empty() {
|
||
chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true)
|
||
} else {
|
||
ts.to_string()
|
||
};
|
||
match lancedb.append_hit_event(id, outcome, reason, source, &now) {
|
||
Ok(n) => send_ok(&mut stream, &format!(r#"{{"appended":{}}}"#, n)),
|
||
Err(e) => send_error(&mut stream, "lancedb_append_hit_event", &e.to_string()),
|
||
}
|
||
}
|
||
}
|
||
"lancedb_count_hit_events" => {
|
||
let id = msg["id"].as_str().unwrap_or("");
|
||
let outcome = msg["outcome"].as_str().unwrap_or("");
|
||
let since = msg["since"].as_str().unwrap_or("");
|
||
match lancedb.count_hit_events(id, outcome, since) {
|
||
Ok(n) => send_ok(&mut stream, &format!(r#"{{"count":{}}}"#, n)),
|
||
Err(e) => send_error(&mut stream, "lancedb_count_hit_events", &e.to_string()),
|
||
}
|
||
}
|
||
"lancedb_get_hit_events" => {
|
||
let id = msg["id"].as_str().unwrap_or("");
|
||
let limit = msg["limit"].as_u64().unwrap_or(100) as usize;
|
||
match lancedb.get_hit_events(id, limit) {
|
||
Ok(evs) => send_ok(&mut stream, &serde_json::to_string(&evs).unwrap_or_default()),
|
||
Err(e) => send_error(&mut stream, "lancedb_get_hit_events", &e.to_string()),
|
||
}
|
||
}
|
||
"lancedb_query" => {
|
||
let min_recall = msg["min_recall"].as_i64().unwrap_or(5) as i64;
|
||
let limit = msg["limit"].as_u64().unwrap_or(20) as usize;
|
||
eprintln!("[lancedb] query: min_recall={} limit={}", min_recall, limit);
|
||
match lancedb.query_memories(min_recall, limit) {
|
||
Ok(r) => {
|
||
eprintln!("[lancedb] query returned {} records", r.len());
|
||
send_ok(&mut stream, &serde_json::to_string(&r).unwrap_or_default())
|
||
}
|
||
Err(e) => send_error(&mut stream, "lancedb_query", &e.to_string()),
|
||
}
|
||
}
|
||
"lancedb_scan" => {
|
||
// P2 2026-09-06: 遗忘候选安全全表扫描 — 限长(硬上限 5000) + 跳 vector
|
||
let mut limit = msg["limit"].as_u64().unwrap_or(2000) as usize;
|
||
if limit > 5000 {
|
||
limit = 5000;
|
||
}
|
||
// 2026-09-21: 可选 namespace 下推过滤(列表/导出端点用)
|
||
let namespace = msg["namespace"].as_str().filter(|s| !s.is_empty());
|
||
eprintln!("[lancedb] scan_forgetting: limit={} namespace={:?}", limit, namespace);
|
||
match lancedb.scan_for_forgetting(limit, namespace) {
|
||
Ok(r) => send_ok(&mut stream, &serde_json::to_string(&r).unwrap_or_default()),
|
||
Err(e) => send_error(&mut stream, "lancedb_scan", &e.to_string()),
|
||
}
|
||
}
|
||
"lancedb_scan_all" => {
|
||
// GA-P0-1 (t_718d8707):全表扫描(scan_all 上限 200000,仅存活行),
|
||
// 供 not_useful_count 拆分迁移脚本读取全部行做统计。scan 硬上限 5000
|
||
// 覆盖不了 35010 行。只读,不落库。
|
||
// 重读准入(t_690cc333 方案①):全表扫描是内存放大器,计入信号量 N。
|
||
let _perm = match state.heavy_read.try_acquire() {
|
||
Ok(p) => Some(p),
|
||
Err(e) => {
|
||
eprintln!("[zhiyi-ipc] scan_all REJECTED (heavy_read_busy)");
|
||
send_error(&mut stream, "heavy_read_busy", &e);
|
||
trim_only();
|
||
return;
|
||
}
|
||
};
|
||
// 方案②(t_576714de):单次扫描行数上限 —— 超上限截断 + truncated:true(降级部分结果)。
|
||
let cap = heavy_read_max_rows();
|
||
match lancedb.scan_all(cap) {
|
||
Ok(r) => {
|
||
let json = serde_json::to_string(&r).unwrap_or_default();
|
||
let trunc = r.len() >= cap;
|
||
if trunc {
|
||
eprintln!("[zhiyi-ipc] scan_all hit cap {} → truncated=true", cap);
|
||
send_ok_truncated(&mut stream, &json);
|
||
} else {
|
||
send_ok(&mut stream, &json);
|
||
}
|
||
}
|
||
Err(e) => send_error(&mut stream, "lancedb_scan_all", &e.to_string()),
|
||
}
|
||
}
|
||
"lancedb_scan_deleted" => {
|
||
// GA-P0-3 (t_f785d7f2):全量扫描**已删除**行(is_deleted=true),供 delete_reason
|
||
// 回填脚本枚举 + 出直方图。只读,不落库。
|
||
// 重读准入(t_690cc333 方案①):全表扫描是内存放大器,计入信号量 N。
|
||
let _perm = match state.heavy_read.try_acquire() {
|
||
Ok(p) => Some(p),
|
||
Err(e) => {
|
||
eprintln!("[zhiyi-ipc] scan_deleted REJECTED (heavy_read_busy)");
|
||
send_error(&mut stream, "heavy_read_busy", &e);
|
||
trim_only();
|
||
return;
|
||
}
|
||
};
|
||
// 方案②(t_576714de):单次扫描行数上限 —— 超上限截断 + truncated:true。
|
||
let cap = heavy_read_max_rows();
|
||
match lancedb.scan_deleted(cap) {
|
||
Ok(r) => {
|
||
let json = serde_json::to_string(&r).unwrap_or_default();
|
||
let trunc = r.len() >= cap;
|
||
if trunc {
|
||
eprintln!("[zhiyi-ipc] scan_deleted hit cap {} → truncated=true", cap);
|
||
send_ok_truncated(&mut stream, &json);
|
||
} else {
|
||
send_ok(&mut stream, &json);
|
||
}
|
||
}
|
||
Err(e) => send_error(&mut stream, "lancedb_scan_deleted", &e.to_string()),
|
||
}
|
||
}
|
||
"lancedb_forget_window" => {
|
||
// P1 2026-10-11 (t_a4b8fa52 方案 c′): 遗忘候选「旋转窗口」扫描
|
||
// 修复固定 2000 窗口不轮转。接收 after_id(游标),返回 ForgetWindow(records+next_cursor+wrapped+live_total)。
|
||
// 重读准入(t_690cc333 方案①):整窗扫描是内存放大器,计入信号量 N。
|
||
let _perm = match state.heavy_read.try_acquire() {
|
||
Ok(p) => Some(p),
|
||
Err(e) => {
|
||
eprintln!("[zhiyi-ipc] forget_window REJECTED (heavy_read_busy)");
|
||
send_error(&mut stream, "heavy_read_busy", &e);
|
||
trim_only();
|
||
return;
|
||
}
|
||
};
|
||
let limit = msg["limit"].as_u64().unwrap_or(2000) as usize;
|
||
let after_id = msg["after_id"].as_str().filter(|s| !s.is_empty());
|
||
eprintln!("[lancedb] forget_window: limit={} after_id={:?}", limit, after_id);
|
||
match lancedb.scan_forgetting_window(limit, after_id) {
|
||
Ok(w) => send_ok(&mut stream, &serde_json::to_string(&w).unwrap_or_default()),
|
||
Err(e) => send_error(&mut stream, "lancedb_forget_window", &e.to_string()),
|
||
}
|
||
}
|
||
"lancedb_purge_invalid" => {
|
||
// P1-1 2026-10-08: 清理 id 为空的垃圾行(无法按 id 软删,按 id='' 条件一次性软删)。
|
||
// ts 由 Go 侧传入(RFC3339),与 soft_delete_batch 同构。
|
||
let ts = msg["ts"].as_str().unwrap_or("");
|
||
match lancedb.purge_invalid(ts) {
|
||
Ok(n) => send_ok(&mut stream, &format!(r#"{{"purged":{}}}"#, n)),
|
||
Err(e) => send_error(&mut stream, "lancedb_purge_invalid", &e.to_string()),
|
||
}
|
||
}
|
||
// 健康探针(t_89d22fc8 AC3):轻量 echo + 整合在途标志。
|
||
// 探针判据:connect ≤3s 且 zhiyi_ping 往返 ≤5s = 健康;
|
||
// consolidating=true 只表示「重活在跑」(轻 IPC 仍正常响应,不算故障)。
|
||
"zhiyi_ping" => {
|
||
let resp = serde_json::json!({
|
||
"ok": true,
|
||
"consolidating": CONSOLIDATING.load(Ordering::SeqCst),
|
||
"heavy_read_in_flight": state.heavy_read.in_flight(),
|
||
"heavy_read_permits": heavy_read_permits(),
|
||
"heavy_read_max_rows": heavy_read_max_rows(),
|
||
"ts": chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true),
|
||
});
|
||
send_ok(&mut stream, &resp.to_string());
|
||
}
|
||
_ => {
|
||
let req: ConsolidateRequest = match serde_json::from_value(msg) {
|
||
Ok(r) => r, Err(e) => {
|
||
send_error(&mut stream, "parse", &e.to_string()); return;
|
||
}
|
||
};
|
||
eprintln!("[zhiyi-ipc] full IPC msg keys: task={:?}, model_dir={:?}, epsilon={}",
|
||
req.task, req.model_dir, req.epsilon);
|
||
let run_args = Args {
|
||
mode: req.task.clone(),
|
||
data_dir: req.lancedb_path.clone(),
|
||
sqlite_path: req.sqlite_path.clone(),
|
||
llm_endpoint: req.llm_endpoint.clone(),
|
||
llm_model: req.llm_model.clone(),
|
||
llm_api_key: req.llm_api_key.clone(),
|
||
llm_budget: req.llm_budget,
|
||
socket: None,
|
||
model_dir: req.model_dir.clone().unwrap_or_default(),
|
||
text: None,
|
||
texts: None,
|
||
epsilon: req.epsilon,
|
||
min_points: req.min_points,
|
||
};
|
||
// 风暴守卫(t_89d22fc8 AC2①):整合独占槽。在途时立即拒绝,不排队、
|
||
// 不并起第二个 DBSCAN 线程(22:07 三连发 + 35min 内存压力 ⇒ OOM 的根因)。
|
||
// Go 侧收到 consolidate_busy 会即时降级 Go 启发式(既有行为),不会等 10min 超时。
|
||
let _guard = match ConsolidatingGuard::try_acquire() {
|
||
Some(g) => Some(g),
|
||
None => {
|
||
eprintln!("[zhiyi-ipc] consolidate REJECTED (busy): task={:?} — another consolidation in progress (storm guard)", req.task);
|
||
send_error(&mut stream, "consolidate_busy",
|
||
"another consolidation is in progress; request dropped, will retry next trigger cycle");
|
||
trim_only();
|
||
return;
|
||
}
|
||
};
|
||
let report = run_consolidation(&run_args, &lancedb);
|
||
drop(_guard);
|
||
send_ok(&mut stream, &serde_json::to_string(&report).unwrap_or_default());
|
||
}
|
||
}
|
||
// 每个 IPC 请求处理完把已释放的堆还给 OS(治本:长驻进程不累积)
|
||
trim_only();
|
||
}
|
||
|
||
fn send_ok(stream: &mut UnixStream, json: &str) {
|
||
let resp = ConsolidateResponse { status: "ok".into(), report_json: json.to_string(), failure_step: String::new(), error_detail: String::new(), truncated: false };
|
||
send_response(stream, &resp);
|
||
}
|
||
/// 方案②(t_576714de):被行数上限截断的正常返回(report_json=部分结果,truncated=true)。
|
||
fn send_ok_truncated(stream: &mut UnixStream, json: &str) {
|
||
let resp = ConsolidateResponse { status: "ok".into(), report_json: json.to_string(), failure_step: String::new(), error_detail: String::new(), truncated: true };
|
||
send_response(stream, &resp);
|
||
}
|
||
fn send_error(stream: &mut UnixStream, step: &str, detail: &str) {
|
||
let resp = ConsolidateResponse { status: "error".into(), report_json: String::new(), failure_step: step.to_string(), error_detail: detail.to_string(), truncated: false };
|
||
send_response(stream, &resp);
|
||
}
|
||
|
||
fn send_response(stream: &mut UnixStream, resp: &ConsolidateResponse) {
|
||
let json = serde_json::to_vec(resp).unwrap_or_default();
|
||
let len = (json.len() as u32).to_be_bytes();
|
||
stream.write_all(&len).ok();
|
||
stream.write_all(&json).ok();
|
||
}
|
||
|
||
/// 执行完整整合流程
|
||
fn run_consolidation(args: &Args, lancedb: &LanceDBOps) -> ConsolidationReport {
|
||
let start = Instant::now();
|
||
|
||
let mut clusters_found = 0usize;
|
||
let mut noise_points = 0usize;
|
||
let mut duplicate_pairs = 0usize;
|
||
let mut pruned_stats: Option<report::PruneStats> = None;
|
||
let mut new_decay_rates: HashMap<String, f64> = HashMap::new();
|
||
let mut r_squared_values: HashMap<String, f64> = HashMap::new();
|
||
let mut quality_score = 0.0f64;
|
||
let mut low_info = 0usize;
|
||
let mut hallucinations = 0usize;
|
||
|
||
// Step 1: DBSCAN 聚类 — 从 LanceDB 加载向量
|
||
if args.mode == "full" || args.mode == "cluster_only" {
|
||
let clusterer = Clusterer::new(args.epsilon, args.min_points);
|
||
|
||
// 从 LanceDB 加载所有记忆向量(用零向量搜索,高 limit)
|
||
let zero_vec = vec![0.0f32; 1024];
|
||
let mut ids: Vec<String>;
|
||
let mut vectors: Vec<Vec<f32>>;
|
||
let mut all_records: Vec<MemoryRecord> = Vec::new();
|
||
|
||
match lancedb.search(&zero_vec, 10000, None) {
|
||
Ok(records) => {
|
||
all_records = records;
|
||
}
|
||
Err(e) => {
|
||
eprintln!("[consolidate] Step 1: failed to load vectors from LanceDB: {}", e);
|
||
}
|
||
}
|
||
|
||
ids = all_records.iter().map(|r| r.id.clone()).collect();
|
||
vectors = all_records.iter().map(|r| r.vector.clone()).collect();
|
||
|
||
// 检查向量是否为全零(Python migration 写入格式错误导致 LanceDB 读回 null)
|
||
// 如果 BGE HTTP 可用则重新编码,否则跳过
|
||
let zero_count = vectors.iter().filter(|v| v.iter().all(|x| x.abs() < 1e-6)).count();
|
||
if zero_count > 0 {
|
||
eprintln!("[consolidate] WARNING: {} / {} vectors are zero (LanceDB format issue)", zero_count, vectors.len());
|
||
}
|
||
|
||
eprintln!("[consolidate] DBSCAN: eps={}, min_pts={}, vectors={}", args.epsilon, args.min_points, vectors.len());
|
||
// Debug: check vector norms and sample distances
|
||
if !vectors.is_empty() {
|
||
let mut norm_sum = 0.0f64;
|
||
let mut norm_min = f64::MAX;
|
||
let mut norm_max = 0.0f64;
|
||
for v in &vectors {
|
||
let s = (v.iter().map(|x| (*x as f64) * (*x as f64)).sum::<f64>()).sqrt();
|
||
norm_sum += s;
|
||
if s < norm_min { norm_min = s; }
|
||
if s > norm_max { norm_max = s; }
|
||
}
|
||
let avg_norm = norm_sum / vectors.len() as f64;
|
||
eprintln!("[consolidate] vector norms: min={:.4} max={:.4} avg={:.4}", norm_min, norm_max, avg_norm);
|
||
|
||
// Sample pairwise distances among first 20 vectors
|
||
let n = vectors.len().min(20);
|
||
let mut all_dists: Vec<f64> = Vec::new();
|
||
for i in 0..n {
|
||
for j in (i+1)..n {
|
||
let d = (vectors[i].iter().zip(&vectors[j]).map(|(a,b)| (a-b).powi(2)).sum::<f32>() as f64).sqrt();
|
||
all_dists.push(d);
|
||
}
|
||
}
|
||
all_dists.sort_by(|a,b| a.partial_cmp(b).unwrap());
|
||
if !all_dists.is_empty() {
|
||
let len = all_dists.len();
|
||
eprintln!("[consolidate] sample distances ({} vecs, {} pairs): p5={:.3} p50={:.3} p95={:.3} min={:.3} max={:.3}",
|
||
n, len, all_dists[(0.05 * len as f64) as usize].min(all_dists[len-1]),
|
||
all_dists[(len/2)],
|
||
all_dists[((len-1) as f64 * 0.95) as usize].min(all_dists[len-1]),
|
||
all_dists[0], all_dists[len-1]);
|
||
}
|
||
}
|
||
|
||
match clusterer.cluster(&vectors, &ids) {
|
||
Ok(result) => {
|
||
clusters_found = result.num_clusters;
|
||
noise_points = result.num_noise;
|
||
if !vectors.is_empty() {
|
||
let centroids = Clusterer::extract_centroids(&vectors, &result.labels, result.num_clusters);
|
||
let sizes = result.cluster_sizes.clone();
|
||
duplicate_pairs = Clusterer::find_duplicates(¢roids, &sizes).len();
|
||
}
|
||
}
|
||
Err(e) => {
|
||
eprintln!("[consolidate] Step 1 (cluster) failed: {}", e);
|
||
}
|
||
}
|
||
}
|
||
reclaim("Step 1 后(聚类)");
|
||
// Step 2: 图谱修剪
|
||
if args.mode == "full" || args.mode == "prune_only" {
|
||
let pruner = GraphPruner::new(&args.sqlite_path);
|
||
match pruner.prune() {
|
||
Ok(stats) => {
|
||
pruned_stats = Some(report::PruneStats {
|
||
isolated_nodes_removed: stats.isolated_nodes_removed,
|
||
low_weight_edges_removed: stats.low_weight_edges_removed,
|
||
redundant_edges_merged: stats.redundant_edges_merged,
|
||
nodes_before: stats.nodes_before,
|
||
nodes_after: stats.nodes_after,
|
||
edges_before: stats.edges_before,
|
||
edges_after: stats.edges_after,
|
||
});
|
||
}
|
||
Err(e) => {
|
||
eprintln!("[consolidate] Step 2 (prune) failed: {}", e);
|
||
}
|
||
}
|
||
}
|
||
reclaim("Step 2 后(图谱修剪)");
|
||
|
||
// Step 3: 衰减校准
|
||
if args.mode == "full" {
|
||
let calibrator = DecayCalibrator::new();
|
||
let old_rates = decay_calibrate::default_decay_rates();
|
||
// 整合内部全量扫描(非 IPC 边界,需全量;行数上限只作用于 IPC 单次重读,t_576714de)。
|
||
let all_records = lancedb.scan_all(200_000).unwrap_or_default();
|
||
let now_secs = std::time::SystemTime::now()
|
||
.duration_since(std::time::UNIX_EPOCH)
|
||
.unwrap_or_default()
|
||
.as_secs() as f64;
|
||
let samples: Vec<DecaySample> = all_records
|
||
.iter()
|
||
.filter_map(|r| {
|
||
let created = chrono::DateTime::parse_from_rfc3339(&r.created_at).ok()
|
||
.map(|dt| dt.timestamp() as f64)
|
||
.unwrap_or(0.0);
|
||
let days = if created > 0.0 { (now_secs - created) / 86400.0 } else { 0.0 };
|
||
if days < 0.0 { return None; }
|
||
Some(DecaySample {
|
||
category: if r.category.is_empty() { "general".into() } else { r.category.clone() },
|
||
days_old: days,
|
||
current_score: r.quality_score as f64,
|
||
original_score: r.importance as f64,
|
||
})
|
||
})
|
||
.collect();
|
||
eprintln!("[consolidate] Step 3: {} decay samples from LanceDB", samples.len());
|
||
let result = calibrator.calibrate(&samples, &old_rates);
|
||
new_decay_rates = result.rates;
|
||
r_squared_values = result.r_squared;
|
||
}
|
||
reclaim("Step 3 后(衰减校准)");
|
||
|
||
// Step 4: 质量回溯 (需要 BGE HTTP 服务在 localhost:8000)
|
||
if args.mode == "full" && !args.llm_endpoint.is_empty() {
|
||
// 先检查 BGE HTTP 是否可达 (2秒超时)
|
||
let bge_ok = std::net::TcpStream::connect_timeout(
|
||
&std::net::SocketAddr::from(([127, 0, 0, 1], 8000)),
|
||
std::time::Duration::from_secs(2),
|
||
).is_ok();
|
||
if !bge_ok {
|
||
eprintln!("[consolidate] Step 4: BGE HTTP (port 8000) unreachable — skipping quality backtrace");
|
||
} else {
|
||
let backtracer = QualityBacktracer::new(&args.llm_endpoint, &args.llm_model, &args.llm_api_key);
|
||
// 从 LanceDB 加载(content = distilled, source metadata = original)。整合内部全量,非 IPC 边界。
|
||
let all_records = lancedb.scan_all(200_000).unwrap_or_default();
|
||
let samples: Vec<QualitySample> = all_records
|
||
.iter()
|
||
.filter(|r| !r.content.is_empty())
|
||
.map(|r| QualitySample {
|
||
memory_id: r.id.clone(),
|
||
distilled_content: r.content.clone(),
|
||
original_episode: if !r.source.is_empty() { r.source.clone() } else { r.content.clone() },
|
||
category: if r.category.is_empty() { "general".into() } else { r.category.clone() },
|
||
tier: if r.freshness.is_empty() { "fresh".into() } else { r.freshness.clone() },
|
||
})
|
||
.collect();
|
||
eprintln!(
|
||
"[consolidate] Step 4: {} quality samples from LanceDB",
|
||
samples.len()
|
||
);
|
||
let sample_20 = quality_backtrace::stratified_sample(&samples, 5, 5, 5, 5);
|
||
|
||
// 用本地 BGE HTTP 服务编码
|
||
let encoder = match BGEEncoder::new(&args.model_dir) {
|
||
Ok(e) => Some(e),
|
||
Err(e) => {
|
||
eprintln!("[consolidate] Step 4: BGE encoder init failed: {} — skipping quality backtrace", e);
|
||
None
|
||
}
|
||
};
|
||
|
||
if let Some(ref enc) = encoder {
|
||
let embed_fn = &|text: &str| -> Result<Vec<f32>, Box<dyn std::error::Error>> {
|
||
enc.encode(text)
|
||
};
|
||
|
||
match backtracer.backtrace(&sample_20, embed_fn) {
|
||
Ok(result) => {
|
||
quality_score = result.score;
|
||
low_info = result.low_info_loss;
|
||
hallucinations = result.hallucinations;
|
||
}
|
||
Err(e) => {
|
||
eprintln!("[consolidate] Step 4 (quality) failed: {}", e);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// Step 5: 生成报告
|
||
reclaim("Step 4 后(质量回溯)");
|
||
let dashboard = DashboardSnapshot {
|
||
recall_useful_rate: 0.85,
|
||
recall_hit_rate: 0.90,
|
||
gap_closure_rate: 0.75,
|
||
cascade_propagation_rate: 0.60,
|
||
deprecation_rate: 3.0,
|
||
distill_loss_rate: quality_score.max(0.0),
|
||
auto_resolve_rate: 0.70,
|
||
};
|
||
|
||
let gen = ReportGenerator::new();
|
||
let mut report = gen.generate(
|
||
clusters_found,
|
||
noise_points,
|
||
duplicate_pairs,
|
||
pruned_stats,
|
||
new_decay_rates,
|
||
r_squared_values,
|
||
quality_score,
|
||
low_info,
|
||
hallucinations,
|
||
&dashboard,
|
||
);
|
||
report.duration_secs = start.elapsed().as_secs_f64();
|
||
|
||
eprintln!(
|
||
"[consolidate] completed in {:.1}s: mode={}, clusters={}, quality={:.3}",
|
||
report.duration_secs, args.mode, clusters_found, quality_score
|
||
);
|
||
|
||
report
|
||
}
|