Files
memoryweave/rust/src/main.rs
T

527 lines
18 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// 织忆 MemoryWeave — zhiyi-consolidate (Rust 整合引擎)
// Unix Socket + Protobuf 协议,Go ↔ Rust IPC
// 深度整合 5 步:DBSCAN → 修剪 → 衰减校准 → 质量回溯 → 报告
mod lancedb_ops;
mod embed;
mod rerank;
mod cluster;
mod decay_calibrate;
mod graph_prune;
mod quality_backtrace;
mod report;
use clap::Parser;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::io::{Read, Write};
use std::net::{Shutdown, TcpListener, TcpStream};
use std::os::unix::net::{UnixListener, UnixStream};
use std::thread;
use std::time::Instant;
use cluster::Clusterer;
use decay_calibrate::{DecayCalibrator, DecaySample};
use embed::BGEEncoder;
use graph_prune::GraphPruner;
use lancedb_ops::LanceDBOps;
use quality_backtrace::{QualityBacktracer, QualitySample};
use report::{ConsolidationReport, DashboardSnapshot, ReportGenerator};
use std::path::Path;
use std::sync::Arc;
/// CLI 参数
#[derive(Parser, Debug, Clone)]
#[command(name = "zhiyi-consolidate")]
#[command(about = "MemoryWeave 整合引擎 — DBSCAN + 衰减 + 修剪 + 质量回溯")]
struct Args {
/// 工作模式: full / cluster_only / prune_only / encode
#[arg(short, long, default_value = "full")]
mode: String,
/// 数据目录
#[arg(long, default_value = "./data")]
data_dir: String,
/// SQLite 图谱路径
#[arg(long, default_value = "./data/graph.db")]
sqlite_path: String,
/// LLM 蒸馏 API 端点
#[arg(long, default_value = "")]
llm_endpoint: String,
/// LLM 模型名
#[arg(long, default_value = "deepseek/deepseek-v4-pro")]
llm_model: String,
/// LLM API Key (Authorization header)
#[arg(long, default_value = "")]
llm_api_key: String,
/// LLM 预算(本次可用次数)
#[arg(long, default_value = "20")]
llm_budget: usize,
/// Unix Socket 路径
#[arg(long)]
socket: Option<String>,
/// DBSCAN epsilon
#[arg(long, default_value = "0.5")]
epsilon: f64,
/// DBSCAN min_points
#[arg(long, default_value = "3")]
min_points: usize,
/// BGE 模型目录 (用于 encode 模式)
#[arg(long, default_value = "/home/muc/models/bge-m3/onnx")]
model_dir: String,
/// encode 模式输入 (单个文本)
#[arg(long)]
text: Option<String>,
/// encode 模式批量输入 (逗号分隔)
#[arg(long)]
texts: Option<String>,
}
/// Protobuf-like 请求消息(简化版)
#[derive(Debug, Serialize, Deserialize)]
struct ConsolidateRequest {
task: String,
lancedb_path: String,
sqlite_path: String,
#[serde(rename = "llm_endpoint")] llm_endpoint: String,
#[serde(rename = "llm_model")] llm_model: String,
#[serde(rename = "llm_api_key")] llm_api_key: String,
llm_budget: usize,
epsilon: f64,
min_points: usize,
}
#[derive(Debug, Serialize, Deserialize)]
struct ConsolidateResponse {
status: String,
report_json: String,
failure_step: String,
error_detail: String,
}
fn main() {
let args = Args::parse();
if let Some(ref socket_path) = args.socket {
run_socket_server(socket_path, &args);
} else if args.mode == "encode" {
// encode 模式: BGE 编码
run_encode(&args);
} else {
// CLI 模式: 执行一次并输出 JSON
let ldb = LanceDBOps::open(&args.data_dir).expect("lancedb open");
let report = run_consolidation(&args, &ldb);
println!("{}", serde_json::to_string_pretty(&report).unwrap_or_default());
}
}
fn run_encode(args: &Args) {
let encoder = match embed::BGEEncoder::new(&args.model_dir) {
Ok(e) => e,
Err(e) => {
eprintln!("[encode] failed to load BGE model: {}", e);
return;
}
};
let texts: Vec<String> = if let Some(t) = &args.texts {
t.split(',').map(|s| s.trim().to_string()).collect()
} else if let Some(t) = &args.text {
vec![t.clone()]
} else {
eprintln!("[encode] use --text or --texts");
return;
};
let vectors = match encoder.encode_batch(&texts) {
Ok(v) => v,
Err(e) => {
eprintln!("[encode] failed: {}", e);
return;
}
};
println!("{}", serde_json::to_string(&vectors).unwrap_or_default());
}
/// Socket 服务共享状态
struct ServerState {
encoder: Option<Arc<BGEEncoder>>,
}
/// Unix Socket 服务端
fn run_socket_server(socket_path: &str, args: &Args) {
if Path::new(socket_path).exists() {
std::fs::remove_file(socket_path).ok();
}
let listener = UnixListener::bind(socket_path).expect("bind");
let lancedb = LanceDBOps::open(&args.data_dir).expect("lancedb open");
lancedb.init_tables().ok();
// 初始化 BGE 编码器
let encoder: Option<Arc<BGEEncoder>> = if !args.model_dir.is_empty() {
match BGEEncoder::new(&args.model_dir) {
Ok(e) => {
eprintln!("[zhiyi] BGE encoder loaded: {} (dim={})", args.model_dir, e.dim);
Some(Arc::new(e))
}
Err(e) => {
eprintln!("[zhiyi] WARN: BGE encoder init failed: {} — encode 请求将失败", e);
None
}
}
} else {
eprintln!("[zhiyi] BGE encoder not configured (--model-dir not set)");
None
};
let state = Arc::new(ServerState { encoder });
eprintln!("[zhiyi] listening on {} (lancedb: {})", socket_path, args.data_dir);
for stream in listener.incoming() {
match stream {
Ok(stream) => {
let a = args.clone();
let ldb = lancedb.clone();
let st = Arc::clone(&state);
thread::spawn(move || handle_client(stream, &a, ldb, &st));
}
Err(e) => eprintln!("[zhiyi] accept error: {}", e),
}
}
}
fn handle_client(mut stream: UnixStream, args: &Args, lancedb: LanceDBOps, state: &ServerState) {
let mut len_buf = [0u8; 4];
if stream.read_exact(&mut len_buf).is_err() {
return;
}
let msg_len = u32::from_be_bytes(len_buf) as usize;
let mut body = vec![0u8; msg_len];
if stream.read_exact(&mut body).is_err() {
return;
}
let msg: serde_json::Value = match serde_json::from_slice(&body) {
Ok(m) => m,
Err(e) => {
send_error(&mut stream, "parse", &e.to_string());
return;
}
};
let msg_type = msg["type"].as_str().unwrap_or("consolidate");
eprintln!(
"[zhiyi] msg type = {:?}, raw keys = {:?}",
msg_type,
msg.as_object().map(|o| o.keys().collect::<Vec<_>>())
);
match msg_type {
"encode" => {
let texts: Vec<String> = msg["texts"]
.as_array()
.map(|a| {
a.iter()
.filter_map(|v| v.as_str())
.map(|s| s.to_string())
.collect()
})
.unwrap_or_default();
if texts.is_empty() {
send_error(&mut stream, "encode", "empty texts");
return;
}
let encoder = match &state.encoder {
Some(e) => e,
None => {
send_error(&mut stream, "encode", "encoder not initialized");
return;
}
};
match encoder.encode_batch(&texts) {
Ok(vectors) => {
let json = serde_json::to_string(&vectors).unwrap_or_default();
send_ok(&mut stream, &json);
}
Err(e) => send_error(&mut stream, "encode", &e.to_string()),
}
}
"lancedb_search" => {
let v: Vec<f32> = msg["vector"].as_array()
.map(|a| a.iter().filter_map(|v| v.as_f64()).map(|v| v as f32).collect())
.unwrap_or_default();
let tk = msg["top_k"].as_u64().unwrap_or(10) as usize;
// 空 namespace = 不过滤(匹配所有)
let ns = msg["namespace"].as_str().filter(|s| !s.is_empty());
match lancedb.search(&v, tk, ns) {
Ok(r) => send_ok(&mut stream, &serde_json::to_string(&r).unwrap_or_default()),
Err(e) => send_error(&mut stream, "lancedb_search", &e.to_string()),
}
}
"lancedb_insert" => {
let table = msg["table"].as_str().unwrap_or("memories");
let records = msg["records"].as_str().unwrap_or("[]");
match lancedb.insert_batch(table, records) {
Ok(n) => send_ok(&mut stream, &format!(r#"{{"inserted":{}}}"#, n)),
Err(e) => send_error(&mut stream, "lancedb_insert", &e.to_string()),
}
}
"lancedb_stats" => {
match lancedb.stats() {
Ok(s) => send_ok(&mut stream, &serde_json::to_string(&s).unwrap_or_default()),
Err(e) => send_error(&mut stream, "lancedb_stats", &e.to_string()),
}
}
"lancedb_update" => {
let table = msg["table"].as_str().unwrap_or("memories");
let id = msg["id"].as_str().unwrap_or("");
let fields = msg["fields"].as_str().unwrap_or("[]");
if id.is_empty() {
send_error(&mut stream, "lancedb_update", "id required");
} else {
match lancedb.update(table, id, fields) {
Ok(n) => send_ok(&mut stream, &format!(r#"{{"updated":{}}}"#, n)),
Err(e) => send_error(&mut stream, "lancedb_update", &e.to_string()),
}
}
}
_ => {
let req: ConsolidateRequest = match serde_json::from_value(msg) {
Ok(r) => r, Err(e) => {
send_error(&mut stream, "parse", &e.to_string()); return;
}
};
let run_args = Args {
mode: req.task.clone(),
data_dir: req.lancedb_path.clone(),
sqlite_path: req.sqlite_path.clone(),
llm_endpoint: req.llm_endpoint.clone(),
llm_model: req.llm_model.clone(),
llm_api_key: req.llm_api_key.clone(),
llm_budget: req.llm_budget,
socket: None,
model_dir: String::new(),
text: None,
texts: None,
epsilon: req.epsilon,
min_points: req.min_points,
};
let report = run_consolidation(&run_args, &lancedb);
send_ok(&mut stream, &serde_json::to_string(&report).unwrap_or_default());
}
}
}
fn send_ok(stream: &mut UnixStream, json: &str) {
let resp = ConsolidateResponse { status: "ok".into(), report_json: json.to_string(), failure_step: "".into(), error_detail: "".into() };
send_response(stream, &resp);
}
fn send_error(stream: &mut UnixStream, step: &str, detail: &str) {
let resp = ConsolidateResponse { status: "error".into(), report_json: "".into(), failure_step: step.to_string(), error_detail: detail.to_string() };
send_response(stream, &resp);
}
fn send_response(stream: &mut UnixStream, resp: &ConsolidateResponse) {
let json = serde_json::to_vec(resp).unwrap_or_default();
let len = (json.len() as u32).to_be_bytes();
stream.write_all(&len).ok();
stream.write_all(&json).ok();
}
/// 执行完整整合流程
fn run_consolidation(args: &Args, lancedb: &LanceDBOps) -> ConsolidationReport {
let start = Instant::now();
let mut clusters_found = 0usize;
let mut noise_points = 0usize;
let mut duplicate_pairs = 0usize;
let mut pruned_stats: Option<report::PruneStats> = None;
let mut new_decay_rates: HashMap<String, f64> = HashMap::new();
let mut r_squared_values: HashMap<String, f64> = HashMap::new();
let mut quality_score = 0.0f64;
let mut low_info = 0usize;
let mut hallucinations = 0usize;
// Step 1: DBSCAN 聚类 — 从 LanceDB 加载向量
if args.mode == "full" || args.mode == "cluster_only" {
let clusterer = Clusterer::new(args.epsilon, args.min_points);
// 从 LanceDB 加载所有记忆向量(用零向量搜索,高 limit)
let zero_vec = vec![0.0f32; 1024];
let vectors: Vec<Vec<f32>>;
let ids: Vec<String>;
match lancedb.search(&zero_vec, 10000, None) {
Ok(records) => {
ids = records.iter().map(|r| r.id.clone()).collect();
vectors = records.iter().map(|r| r.vector.clone()).collect();
}
Err(e) => {
eprintln!("[consolidate] Step 1: failed to load vectors from LanceDB: {}", e);
vectors = Vec::new(); ids = Vec::new();
}
}
match clusterer.cluster(&vectors, &ids) {
Ok(result) => {
clusters_found = result.num_clusters;
noise_points = result.num_noise;
if !vectors.is_empty() {
let centroids = Clusterer::extract_centroids(&vectors, &result.labels, result.num_clusters);
let sizes: Vec<usize> = result.cluster_sizes.clone();
duplicate_pairs = Clusterer::find_duplicates(&centroids, &sizes).len();
}
}
Err(e) => {
eprintln!("[consolidate] Step 1 (cluster) failed: {}", e);
}
}
}
// Step 2: 图谱修剪
if args.mode == "full" || args.mode == "prune_only" {
let pruner = GraphPruner::new(&args.sqlite_path);
match pruner.prune() {
Ok(stats) => {
pruned_stats = Some(report::PruneStats {
isolated_nodes_removed: stats.isolated_nodes_removed,
low_weight_edges_removed: stats.low_weight_edges_removed,
redundant_edges_merged: stats.redundant_edges_merged,
nodes_before: stats.nodes_before,
nodes_after: stats.nodes_after,
edges_before: stats.edges_before,
edges_after: stats.edges_after,
});
}
Err(e) => {
eprintln!("[consolidate] Step 2 (prune) failed: {}", e);
}
}
}
// Step 3: 衰减校准
if args.mode == "full" {
let calibrator = DecayCalibrator::new();
let old_rates = decay_calibrate::default_decay_rates();
// 从 LanceDB 加载真实样本
let all_records = lancedb.scan_all().unwrap_or_default();
let now_secs = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_secs() as f64;
let samples: Vec<DecaySample> = all_records
.iter()
.filter_map(|r| {
let created = chrono::DateTime::parse_from_rfc3339(&r.created_at).ok()
.map(|dt| dt.timestamp() as f64)
.unwrap_or(0.0);
let days = if created > 0.0 { (now_secs - created) / 86400.0 } else { 0.0 };
if days < 0.0 { return None; }
Some(DecaySample {
category: if r.category.is_empty() { "general".into() } else { r.category.clone() },
days_old: days,
current_score: r.quality_score as f64,
original_score: r.importance as f64,
})
})
.collect();
eprintln!("[consolidate] Step 3: {} decay samples from LanceDB", samples.len());
let result = calibrator.calibrate(&samples, &old_rates);
new_decay_rates = result.rates;
r_squared_values = result.r_squared;
}
// Step 4: 质量回溯
if args.mode == "full" && !args.llm_endpoint.is_empty() {
let backtracer = QualityBacktracer::new(&args.llm_endpoint, &args.llm_model, &args.llm_api_key);
// 从 LanceDB 加载(content = distilled, source metadata = original)
let all_records = lancedb.scan_all().unwrap_or_default();
let samples: Vec<QualitySample> = all_records
.iter()
.filter(|r| !r.content.is_empty())
.map(|r| QualitySample {
memory_id: r.id.clone(),
distilled_content: r.content.clone(),
original_episode: if !r.source.is_empty() { r.source.clone() } else { r.content.clone() },
category: if r.category.is_empty() { "general".into() } else { r.category.clone() },
tier: if r.tier.is_empty() { "normal".into() } else { r.tier.clone() },
})
.collect();
eprintln!(
"[consolidate] Step 4: {} quality samples from LanceDB",
samples.len()
);
let sample_20 = quality_backtrace::stratified_sample(&samples, 5, 5, 5, 5);
// 用本地 BGE HTTP 服务编码
let encoder = match BGEEncoder::new(&args.model_dir) {
Ok(e) => Some(e),
Err(e) => {
eprintln!("[consolidate] Step 4: BGE encoder init failed: {} — skipping quality backtrace", e);
None
}
};
if let Some(ref enc) = encoder {
let embed_fn = &|text: &str| -> Result<Vec<f32>, Box<dyn std::error::Error>> {
enc.encode(text)
};
match backtracer.backtrace(&sample_20, embed_fn) {
Ok(result) => {
quality_score = result.score;
low_info = result.low_info_loss;
hallucinations = result.hallucinations;
}
Err(e) => {
eprintln!("[consolidate] Step 4 (quality) failed: {}", e);
}
}
}
}
// Step 5: 生成报告
let dashboard = DashboardSnapshot {
recall_useful_rate: 0.85,
recall_hit_rate: 0.90,
gap_closure_rate: 0.75,
cascade_propagation_rate: 0.60,
deprecation_rate: 3.0,
distill_loss_rate: quality_score.max(0.0),
auto_resolve_rate: 0.70,
};
let gen = ReportGenerator::new();
let mut report = gen.generate(
clusters_found,
noise_points,
duplicate_pairs,
pruned_stats,
new_decay_rates,
r_squared_values,
quality_score,
low_info,
hallucinations,
&dashboard,
);
report.duration_secs = start.elapsed().as_secs_f64();
eprintln!(
"[consolidate] completed in {:.1}s: mode={}, clusters={}, quality={:.3}",
report.duration_secs, args.mode, clusters_found, quality_score
);
report
}