527 lines
18 KiB
Rust
527 lines
18 KiB
Rust
// 织忆 MemoryWeave — zhiyi-consolidate (Rust 整合引擎)
|
||
// Unix Socket + Protobuf 协议,Go ↔ Rust IPC
|
||
// 深度整合 5 步:DBSCAN → 修剪 → 衰减校准 → 质量回溯 → 报告
|
||
|
||
mod lancedb_ops;
|
||
mod embed;
|
||
mod rerank;
|
||
mod cluster;
|
||
mod decay_calibrate;
|
||
mod graph_prune;
|
||
mod quality_backtrace;
|
||
mod report;
|
||
|
||
use clap::Parser;
|
||
use serde::{Deserialize, Serialize};
|
||
use std::collections::HashMap;
|
||
use std::io::{Read, Write};
|
||
use std::net::{Shutdown, TcpListener, TcpStream};
|
||
use std::os::unix::net::{UnixListener, UnixStream};
|
||
use std::thread;
|
||
use std::time::Instant;
|
||
|
||
use cluster::Clusterer;
|
||
use decay_calibrate::{DecayCalibrator, DecaySample};
|
||
use embed::BGEEncoder;
|
||
use graph_prune::GraphPruner;
|
||
use lancedb_ops::LanceDBOps;
|
||
use quality_backtrace::{QualityBacktracer, QualitySample};
|
||
use report::{ConsolidationReport, DashboardSnapshot, ReportGenerator};
|
||
use std::path::Path;
|
||
use std::sync::Arc;
|
||
|
||
/// CLI 参数
|
||
#[derive(Parser, Debug, Clone)]
|
||
#[command(name = "zhiyi-consolidate")]
|
||
#[command(about = "MemoryWeave 整合引擎 — DBSCAN + 衰减 + 修剪 + 质量回溯")]
|
||
struct Args {
|
||
/// 工作模式: full / cluster_only / prune_only / encode
|
||
#[arg(short, long, default_value = "full")]
|
||
mode: String,
|
||
|
||
/// 数据目录
|
||
#[arg(long, default_value = "./data")]
|
||
data_dir: String,
|
||
|
||
/// SQLite 图谱路径
|
||
#[arg(long, default_value = "./data/graph.db")]
|
||
sqlite_path: String,
|
||
|
||
/// LLM 蒸馏 API 端点
|
||
#[arg(long, default_value = "")]
|
||
llm_endpoint: String,
|
||
|
||
/// LLM 模型名
|
||
#[arg(long, default_value = "deepseek/deepseek-v4-pro")]
|
||
llm_model: String,
|
||
|
||
/// LLM API Key (Authorization header)
|
||
#[arg(long, default_value = "")]
|
||
llm_api_key: String,
|
||
|
||
/// LLM 预算(本次可用次数)
|
||
#[arg(long, default_value = "20")]
|
||
llm_budget: usize,
|
||
|
||
/// Unix Socket 路径
|
||
#[arg(long)]
|
||
socket: Option<String>,
|
||
|
||
/// DBSCAN epsilon
|
||
#[arg(long, default_value = "0.5")]
|
||
epsilon: f64,
|
||
|
||
/// DBSCAN min_points
|
||
#[arg(long, default_value = "3")]
|
||
min_points: usize,
|
||
|
||
/// BGE 模型目录 (用于 encode 模式)
|
||
#[arg(long, default_value = "/home/muc/models/bge-m3/onnx")]
|
||
model_dir: String,
|
||
|
||
/// encode 模式输入 (单个文本)
|
||
#[arg(long)]
|
||
text: Option<String>,
|
||
|
||
/// encode 模式批量输入 (逗号分隔)
|
||
#[arg(long)]
|
||
texts: Option<String>,
|
||
}
|
||
|
||
/// Protobuf-like 请求消息(简化版)
|
||
#[derive(Debug, Serialize, Deserialize)]
|
||
struct ConsolidateRequest {
|
||
task: String,
|
||
lancedb_path: String,
|
||
sqlite_path: String,
|
||
#[serde(rename = "llm_endpoint")] llm_endpoint: String,
|
||
#[serde(rename = "llm_model")] llm_model: String,
|
||
#[serde(rename = "llm_api_key")] llm_api_key: String,
|
||
llm_budget: usize,
|
||
epsilon: f64,
|
||
min_points: usize,
|
||
}
|
||
|
||
#[derive(Debug, Serialize, Deserialize)]
|
||
struct ConsolidateResponse {
|
||
status: String,
|
||
report_json: String,
|
||
failure_step: String,
|
||
error_detail: String,
|
||
}
|
||
|
||
fn main() {
|
||
let args = Args::parse();
|
||
|
||
if let Some(ref socket_path) = args.socket {
|
||
run_socket_server(socket_path, &args);
|
||
} else if args.mode == "encode" {
|
||
// encode 模式: BGE 编码
|
||
run_encode(&args);
|
||
} else {
|
||
// CLI 模式: 执行一次并输出 JSON
|
||
let ldb = LanceDBOps::open(&args.data_dir).expect("lancedb open");
|
||
let report = run_consolidation(&args, &ldb);
|
||
println!("{}", serde_json::to_string_pretty(&report).unwrap_or_default());
|
||
}
|
||
}
|
||
|
||
fn run_encode(args: &Args) {
|
||
let encoder = match embed::BGEEncoder::new(&args.model_dir) {
|
||
Ok(e) => e,
|
||
Err(e) => {
|
||
eprintln!("[encode] failed to load BGE model: {}", e);
|
||
return;
|
||
}
|
||
};
|
||
|
||
let texts: Vec<String> = if let Some(t) = &args.texts {
|
||
t.split(',').map(|s| s.trim().to_string()).collect()
|
||
} else if let Some(t) = &args.text {
|
||
vec![t.clone()]
|
||
} else {
|
||
eprintln!("[encode] use --text or --texts");
|
||
return;
|
||
};
|
||
|
||
let vectors = match encoder.encode_batch(&texts) {
|
||
Ok(v) => v,
|
||
Err(e) => {
|
||
eprintln!("[encode] failed: {}", e);
|
||
return;
|
||
}
|
||
};
|
||
|
||
println!("{}", serde_json::to_string(&vectors).unwrap_or_default());
|
||
}
|
||
|
||
/// Socket 服务共享状态
|
||
struct ServerState {
|
||
encoder: Option<Arc<BGEEncoder>>,
|
||
}
|
||
|
||
/// Unix Socket 服务端
|
||
fn run_socket_server(socket_path: &str, args: &Args) {
|
||
if Path::new(socket_path).exists() {
|
||
std::fs::remove_file(socket_path).ok();
|
||
}
|
||
let listener = UnixListener::bind(socket_path).expect("bind");
|
||
let lancedb = LanceDBOps::open(&args.data_dir).expect("lancedb open");
|
||
lancedb.init_tables().ok();
|
||
|
||
// 初始化 BGE 编码器
|
||
let encoder: Option<Arc<BGEEncoder>> = if !args.model_dir.is_empty() {
|
||
match BGEEncoder::new(&args.model_dir) {
|
||
Ok(e) => {
|
||
eprintln!("[zhiyi] BGE encoder loaded: {} (dim={})", args.model_dir, e.dim);
|
||
Some(Arc::new(e))
|
||
}
|
||
Err(e) => {
|
||
eprintln!("[zhiyi] WARN: BGE encoder init failed: {} — encode 请求将失败", e);
|
||
None
|
||
}
|
||
}
|
||
} else {
|
||
eprintln!("[zhiyi] BGE encoder not configured (--model-dir not set)");
|
||
None
|
||
};
|
||
|
||
let state = Arc::new(ServerState { encoder });
|
||
|
||
eprintln!("[zhiyi] listening on {} (lancedb: {})", socket_path, args.data_dir);
|
||
|
||
for stream in listener.incoming() {
|
||
match stream {
|
||
Ok(stream) => {
|
||
let a = args.clone();
|
||
let ldb = lancedb.clone();
|
||
let st = Arc::clone(&state);
|
||
thread::spawn(move || handle_client(stream, &a, ldb, &st));
|
||
}
|
||
Err(e) => eprintln!("[zhiyi] accept error: {}", e),
|
||
}
|
||
}
|
||
}
|
||
|
||
fn handle_client(mut stream: UnixStream, args: &Args, lancedb: LanceDBOps, state: &ServerState) {
|
||
let mut len_buf = [0u8; 4];
|
||
if stream.read_exact(&mut len_buf).is_err() {
|
||
return;
|
||
}
|
||
let msg_len = u32::from_be_bytes(len_buf) as usize;
|
||
let mut body = vec![0u8; msg_len];
|
||
if stream.read_exact(&mut body).is_err() {
|
||
return;
|
||
}
|
||
|
||
let msg: serde_json::Value = match serde_json::from_slice(&body) {
|
||
Ok(m) => m,
|
||
Err(e) => {
|
||
send_error(&mut stream, "parse", &e.to_string());
|
||
return;
|
||
}
|
||
};
|
||
|
||
let msg_type = msg["type"].as_str().unwrap_or("consolidate");
|
||
eprintln!(
|
||
"[zhiyi] msg type = {:?}, raw keys = {:?}",
|
||
msg_type,
|
||
msg.as_object().map(|o| o.keys().collect::<Vec<_>>())
|
||
);
|
||
match msg_type {
|
||
"encode" => {
|
||
let texts: Vec<String> = msg["texts"]
|
||
.as_array()
|
||
.map(|a| {
|
||
a.iter()
|
||
.filter_map(|v| v.as_str())
|
||
.map(|s| s.to_string())
|
||
.collect()
|
||
})
|
||
.unwrap_or_default();
|
||
if texts.is_empty() {
|
||
send_error(&mut stream, "encode", "empty texts");
|
||
return;
|
||
}
|
||
let encoder = match &state.encoder {
|
||
Some(e) => e,
|
||
None => {
|
||
send_error(&mut stream, "encode", "encoder not initialized");
|
||
return;
|
||
}
|
||
};
|
||
match encoder.encode_batch(&texts) {
|
||
Ok(vectors) => {
|
||
let json = serde_json::to_string(&vectors).unwrap_or_default();
|
||
send_ok(&mut stream, &json);
|
||
}
|
||
Err(e) => send_error(&mut stream, "encode", &e.to_string()),
|
||
}
|
||
}
|
||
"lancedb_search" => {
|
||
let v: Vec<f32> = msg["vector"].as_array()
|
||
.map(|a| a.iter().filter_map(|v| v.as_f64()).map(|v| v as f32).collect())
|
||
.unwrap_or_default();
|
||
let tk = msg["top_k"].as_u64().unwrap_or(10) as usize;
|
||
// 空 namespace = 不过滤(匹配所有)
|
||
let ns = msg["namespace"].as_str().filter(|s| !s.is_empty());
|
||
match lancedb.search(&v, tk, ns) {
|
||
Ok(r) => send_ok(&mut stream, &serde_json::to_string(&r).unwrap_or_default()),
|
||
Err(e) => send_error(&mut stream, "lancedb_search", &e.to_string()),
|
||
}
|
||
}
|
||
"lancedb_insert" => {
|
||
let table = msg["table"].as_str().unwrap_or("memories");
|
||
let records = msg["records"].as_str().unwrap_or("[]");
|
||
match lancedb.insert_batch(table, records) {
|
||
Ok(n) => send_ok(&mut stream, &format!(r#"{{"inserted":{}}}"#, n)),
|
||
Err(e) => send_error(&mut stream, "lancedb_insert", &e.to_string()),
|
||
}
|
||
}
|
||
"lancedb_stats" => {
|
||
match lancedb.stats() {
|
||
Ok(s) => send_ok(&mut stream, &serde_json::to_string(&s).unwrap_or_default()),
|
||
Err(e) => send_error(&mut stream, "lancedb_stats", &e.to_string()),
|
||
}
|
||
}
|
||
"lancedb_update" => {
|
||
let table = msg["table"].as_str().unwrap_or("memories");
|
||
let id = msg["id"].as_str().unwrap_or("");
|
||
let fields = msg["fields"].as_str().unwrap_or("[]");
|
||
if id.is_empty() {
|
||
send_error(&mut stream, "lancedb_update", "id required");
|
||
} else {
|
||
match lancedb.update(table, id, fields) {
|
||
Ok(n) => send_ok(&mut stream, &format!(r#"{{"updated":{}}}"#, n)),
|
||
Err(e) => send_error(&mut stream, "lancedb_update", &e.to_string()),
|
||
}
|
||
}
|
||
}
|
||
_ => {
|
||
let req: ConsolidateRequest = match serde_json::from_value(msg) {
|
||
Ok(r) => r, Err(e) => {
|
||
send_error(&mut stream, "parse", &e.to_string()); return;
|
||
}
|
||
};
|
||
let run_args = Args {
|
||
mode: req.task.clone(),
|
||
data_dir: req.lancedb_path.clone(),
|
||
sqlite_path: req.sqlite_path.clone(),
|
||
llm_endpoint: req.llm_endpoint.clone(),
|
||
llm_model: req.llm_model.clone(),
|
||
llm_api_key: req.llm_api_key.clone(),
|
||
llm_budget: req.llm_budget,
|
||
socket: None,
|
||
model_dir: String::new(),
|
||
text: None,
|
||
texts: None,
|
||
epsilon: req.epsilon,
|
||
min_points: req.min_points,
|
||
};
|
||
let report = run_consolidation(&run_args, &lancedb);
|
||
send_ok(&mut stream, &serde_json::to_string(&report).unwrap_or_default());
|
||
}
|
||
}
|
||
}
|
||
|
||
fn send_ok(stream: &mut UnixStream, json: &str) {
|
||
let resp = ConsolidateResponse { status: "ok".into(), report_json: json.to_string(), failure_step: "".into(), error_detail: "".into() };
|
||
send_response(stream, &resp);
|
||
}
|
||
fn send_error(stream: &mut UnixStream, step: &str, detail: &str) {
|
||
let resp = ConsolidateResponse { status: "error".into(), report_json: "".into(), failure_step: step.to_string(), error_detail: detail.to_string() };
|
||
send_response(stream, &resp);
|
||
}
|
||
|
||
fn send_response(stream: &mut UnixStream, resp: &ConsolidateResponse) {
|
||
let json = serde_json::to_vec(resp).unwrap_or_default();
|
||
let len = (json.len() as u32).to_be_bytes();
|
||
stream.write_all(&len).ok();
|
||
stream.write_all(&json).ok();
|
||
}
|
||
|
||
/// 执行完整整合流程
|
||
fn run_consolidation(args: &Args, lancedb: &LanceDBOps) -> ConsolidationReport {
|
||
let start = Instant::now();
|
||
|
||
let mut clusters_found = 0usize;
|
||
let mut noise_points = 0usize;
|
||
let mut duplicate_pairs = 0usize;
|
||
let mut pruned_stats: Option<report::PruneStats> = None;
|
||
let mut new_decay_rates: HashMap<String, f64> = HashMap::new();
|
||
let mut r_squared_values: HashMap<String, f64> = HashMap::new();
|
||
let mut quality_score = 0.0f64;
|
||
let mut low_info = 0usize;
|
||
let mut hallucinations = 0usize;
|
||
|
||
// Step 1: DBSCAN 聚类 — 从 LanceDB 加载向量
|
||
if args.mode == "full" || args.mode == "cluster_only" {
|
||
let clusterer = Clusterer::new(args.epsilon, args.min_points);
|
||
|
||
// 从 LanceDB 加载所有记忆向量(用零向量搜索,高 limit)
|
||
let zero_vec = vec![0.0f32; 1024];
|
||
let vectors: Vec<Vec<f32>>;
|
||
let ids: Vec<String>;
|
||
match lancedb.search(&zero_vec, 10000, None) {
|
||
Ok(records) => {
|
||
ids = records.iter().map(|r| r.id.clone()).collect();
|
||
vectors = records.iter().map(|r| r.vector.clone()).collect();
|
||
}
|
||
Err(e) => {
|
||
eprintln!("[consolidate] Step 1: failed to load vectors from LanceDB: {}", e);
|
||
vectors = Vec::new(); ids = Vec::new();
|
||
}
|
||
}
|
||
|
||
match clusterer.cluster(&vectors, &ids) {
|
||
Ok(result) => {
|
||
clusters_found = result.num_clusters;
|
||
noise_points = result.num_noise;
|
||
if !vectors.is_empty() {
|
||
let centroids = Clusterer::extract_centroids(&vectors, &result.labels, result.num_clusters);
|
||
let sizes: Vec<usize> = result.cluster_sizes.clone();
|
||
duplicate_pairs = Clusterer::find_duplicates(¢roids, &sizes).len();
|
||
}
|
||
}
|
||
Err(e) => {
|
||
eprintln!("[consolidate] Step 1 (cluster) failed: {}", e);
|
||
}
|
||
}
|
||
}
|
||
|
||
// Step 2: 图谱修剪
|
||
if args.mode == "full" || args.mode == "prune_only" {
|
||
let pruner = GraphPruner::new(&args.sqlite_path);
|
||
match pruner.prune() {
|
||
Ok(stats) => {
|
||
pruned_stats = Some(report::PruneStats {
|
||
isolated_nodes_removed: stats.isolated_nodes_removed,
|
||
low_weight_edges_removed: stats.low_weight_edges_removed,
|
||
redundant_edges_merged: stats.redundant_edges_merged,
|
||
nodes_before: stats.nodes_before,
|
||
nodes_after: stats.nodes_after,
|
||
edges_before: stats.edges_before,
|
||
edges_after: stats.edges_after,
|
||
});
|
||
}
|
||
Err(e) => {
|
||
eprintln!("[consolidate] Step 2 (prune) failed: {}", e);
|
||
}
|
||
}
|
||
}
|
||
|
||
// Step 3: 衰减校准
|
||
if args.mode == "full" {
|
||
let calibrator = DecayCalibrator::new();
|
||
let old_rates = decay_calibrate::default_decay_rates();
|
||
// 从 LanceDB 加载真实样本
|
||
let all_records = lancedb.scan_all().unwrap_or_default();
|
||
let now_secs = std::time::SystemTime::now()
|
||
.duration_since(std::time::UNIX_EPOCH)
|
||
.unwrap_or_default()
|
||
.as_secs() as f64;
|
||
let samples: Vec<DecaySample> = all_records
|
||
.iter()
|
||
.filter_map(|r| {
|
||
let created = chrono::DateTime::parse_from_rfc3339(&r.created_at).ok()
|
||
.map(|dt| dt.timestamp() as f64)
|
||
.unwrap_or(0.0);
|
||
let days = if created > 0.0 { (now_secs - created) / 86400.0 } else { 0.0 };
|
||
if days < 0.0 { return None; }
|
||
Some(DecaySample {
|
||
category: if r.category.is_empty() { "general".into() } else { r.category.clone() },
|
||
days_old: days,
|
||
current_score: r.quality_score as f64,
|
||
original_score: r.importance as f64,
|
||
})
|
||
})
|
||
.collect();
|
||
eprintln!("[consolidate] Step 3: {} decay samples from LanceDB", samples.len());
|
||
let result = calibrator.calibrate(&samples, &old_rates);
|
||
new_decay_rates = result.rates;
|
||
r_squared_values = result.r_squared;
|
||
}
|
||
|
||
// Step 4: 质量回溯
|
||
if args.mode == "full" && !args.llm_endpoint.is_empty() {
|
||
let backtracer = QualityBacktracer::new(&args.llm_endpoint, &args.llm_model, &args.llm_api_key);
|
||
// 从 LanceDB 加载(content = distilled, source metadata = original)
|
||
let all_records = lancedb.scan_all().unwrap_or_default();
|
||
let samples: Vec<QualitySample> = all_records
|
||
.iter()
|
||
.filter(|r| !r.content.is_empty())
|
||
.map(|r| QualitySample {
|
||
memory_id: r.id.clone(),
|
||
distilled_content: r.content.clone(),
|
||
original_episode: if !r.source.is_empty() { r.source.clone() } else { r.content.clone() },
|
||
category: if r.category.is_empty() { "general".into() } else { r.category.clone() },
|
||
tier: if r.tier.is_empty() { "normal".into() } else { r.tier.clone() },
|
||
})
|
||
.collect();
|
||
eprintln!(
|
||
"[consolidate] Step 4: {} quality samples from LanceDB",
|
||
samples.len()
|
||
);
|
||
let sample_20 = quality_backtrace::stratified_sample(&samples, 5, 5, 5, 5);
|
||
|
||
// 用本地 BGE HTTP 服务编码
|
||
let encoder = match BGEEncoder::new(&args.model_dir) {
|
||
Ok(e) => Some(e),
|
||
Err(e) => {
|
||
eprintln!("[consolidate] Step 4: BGE encoder init failed: {} — skipping quality backtrace", e);
|
||
None
|
||
}
|
||
};
|
||
|
||
if let Some(ref enc) = encoder {
|
||
let embed_fn = &|text: &str| -> Result<Vec<f32>, Box<dyn std::error::Error>> {
|
||
enc.encode(text)
|
||
};
|
||
|
||
match backtracer.backtrace(&sample_20, embed_fn) {
|
||
Ok(result) => {
|
||
quality_score = result.score;
|
||
low_info = result.low_info_loss;
|
||
hallucinations = result.hallucinations;
|
||
}
|
||
Err(e) => {
|
||
eprintln!("[consolidate] Step 4 (quality) failed: {}", e);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// Step 5: 生成报告
|
||
let dashboard = DashboardSnapshot {
|
||
recall_useful_rate: 0.85,
|
||
recall_hit_rate: 0.90,
|
||
gap_closure_rate: 0.75,
|
||
cascade_propagation_rate: 0.60,
|
||
deprecation_rate: 3.0,
|
||
distill_loss_rate: quality_score.max(0.0),
|
||
auto_resolve_rate: 0.70,
|
||
};
|
||
|
||
let gen = ReportGenerator::new();
|
||
let mut report = gen.generate(
|
||
clusters_found,
|
||
noise_points,
|
||
duplicate_pairs,
|
||
pruned_stats,
|
||
new_decay_rates,
|
||
r_squared_values,
|
||
quality_score,
|
||
low_info,
|
||
hallucinations,
|
||
&dashboard,
|
||
);
|
||
report.duration_secs = start.elapsed().as_secs_f64();
|
||
|
||
eprintln!(
|
||
"[consolidate] completed in {:.1}s: mode={}, clusters={}, quality={:.3}",
|
||
report.duration_secs, args.mode, clusters_found, quality_score
|
||
);
|
||
|
||
report
|
||
}
|