Files
memoryweave/rust/src/graph_prune.rs
T
小唯 3b8ecba48e perf: 织忆内存/碎片双治本 — PageRank O(V+E) 重写 + 写入去抖 + 在线VACUUM
1. PageRank 三重循环 O(47k×374k×20≈3.5e11) → 入边表 O(V+E):
   单次 full 整合 4h41m → 95.2s,内存峰值 3.8G → 237MB(实测)
2. 图谱修剪 merge 边 id 加 relation + INSERT OR REPLACE:
   修复 09-18 起 UNIQUE constraint 静默失败 → 主库首次跑消 9.8万冗余边
3. LanceDB 写入去抖(server.go + validator.go):
   被动验证只写 Confidence 实际变化的记录;原先每输入重写全库 6000 条
   → 版本快照 O(n²),每日 ~1.9万版本/十几G 膨胀的根因
4. 新增 scripts/lance_vacuum_online.py:在线 VACUUM(不停服务)+
   24h 版本窗口(可回滚)+ 行数门禁(减少才报警);替代停服务版
   每日 vacuum → cron 已改为在线版

实测:侧车 RSS 1.09G→69MB;memories 分片 5383→1;服务全程在线
2026-09-21 10:17:56 +08:00

266 lines
10 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// 知识图谱修剪 — SQLite 操作 (rusqlite)
// 深度整合步骤 2:孤立节点/低权重边/冗余边合并/PageRank 更新
use rusqlite::{Connection, params};
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
use std::path::Path;
/// 修剪统计
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PruneStats {
pub isolated_nodes_removed: usize,
pub low_weight_edges_removed: usize,
pub redundant_edges_merged: usize,
pub nodes_before: usize,
pub nodes_after: usize,
pub edges_before: usize,
pub edges_after: usize,
}
/// 图谱修剪器
pub struct GraphPruner {
db_path: String,
/// 孤立节点阈值: 14天无关联边
isolated_days: u32,
/// 低权重阈值
low_weight_threshold: f64,
/// 最多 BFS 跳数
max_hops: usize,
}
impl GraphPruner {
pub fn new(db_path: &str) -> Self {
Self {
db_path: db_path.to_string(),
isolated_days: 14,
low_weight_threshold: 0.15,
max_hops: 3,
}
}
/// 执行全部修剪流程
pub fn prune(&self) -> Result<PruneStats, Box<dyn std::error::Error>> {
let conn = Connection::open(&self.db_path)?;
// 统计修剪前
let nodes_before: usize = conn.query_row("SELECT COUNT(*) FROM graph_nodes", [], |r| r.get(0))?;
let edges_before: usize = conn.query_row("SELECT COUNT(*) FROM graph_edges", [], |r| r.get(0))?;
// Step 1: 删除孤立节点 (14天无关联边)
let isolated_removed = self.remove_isolated_nodes(&conn)?;
// Step 2: 删除低权重边 (weight < 0.15)
let low_weight_removed = self.remove_low_weight_edges(&conn)?;
// Step 3: 合并冗余边 (同 source→target 的多条边 → 取 weight 加权平均)
let redundant_merged = self.merge_redundant_edges(&conn)?;
// Step 4: 更新 PageRank
self.update_pagerank(&conn)?;
// 统计修剪后
let nodes_after: usize = conn.query_row("SELECT COUNT(*) FROM graph_nodes", [], |r| r.get(0))?;
let edges_after: usize = conn.query_row("SELECT COUNT(*) FROM graph_edges", [], |r| r.get(0))?;
let stats = PruneStats {
isolated_nodes_removed: isolated_removed,
low_weight_edges_removed: low_weight_removed,
redundant_edges_merged: redundant_merged,
nodes_before,
nodes_after,
edges_before,
edges_after,
};
eprintln!(
"[graph_prune] nodes {}→{}, edges {}→{} (isolated={}, low_wt={}, merged={})",
nodes_before, nodes_after, edges_before, edges_after,
isolated_removed, low_weight_removed, redundant_merged,
);
conn.close().ok();
Ok(stats)
}
/// 删除孤立节点
fn remove_isolated_nodes(&self, conn: &Connection) -> Result<usize, rusqlite::Error> {
let cutoff = format!("-{} days", self.isolated_days);
let deleted = conn.execute(
"DELETE FROM graph_nodes WHERE id IN (
SELECT n.id FROM graph_nodes n
LEFT JOIN graph_edges e ON n.id = e.source OR n.id = e.target
WHERE e.id IS NULL
AND n.created_at < datetime('now', '-14 days')
)",
[],
)?;
Ok(deleted)
}
/// 删除低权重边
fn remove_low_weight_edges(&self, conn: &Connection) -> Result<usize, rusqlite::Error> {
let deleted = conn.execute(
"DELETE FROM graph_edges WHERE weight < ?1",
params![self.low_weight_threshold],
)?;
Ok(deleted)
}
/// 合并冗余边
fn merge_redundant_edges(&self, conn: &Connection) -> Result<usize, rusqlite::Error> {
// 查找 (source, target, relation) 相同的冗余边
let mut stmt = conn.prepare(
"SELECT source, target, relation, COUNT(*) as cnt,
SUM(weight) as total_weight, SUM(evidence_count) as total_evidence
FROM graph_edges
GROUP BY source, target, relation
HAVING cnt > 1"
)?;
let to_merge: Vec<(String, String, String, i64, f64, i64)> = stmt.query_map(
[],
|row| Ok((
row.get::<_, String>(0)?,
row.get::<_, String>(1)?,
row.get::<_, String>(2)?,
row.get::<_, i64>(3)?,
row.get::<_, f64>(4)?,
row.get::<_, i64>(5)?,
))
)?.filter_map(|r| r.ok()).collect();
let mut merged = 0usize;
for (src, tgt, rel, cnt, total_wt, total_ev) in to_merge {
// 删除所有冗余边
let deleted = conn.execute(
"DELETE FROM graph_edges WHERE source=?1 AND target=?2 AND relation=?3",
params![src, tgt, rel],
)?;
// 插入合并后单边(加权平均)
let avg_weight = total_wt / cnt as f64;
let avg_evidence = total_ev / cnt;
// ⚠️ 2026-09-21 修:原 id 只含 (src,tgt),同一对节点有多条 relation 时 id 撞车
// → UNIQUE constraint failed: graph_edges.id → 每次 full 整合 Step 2 全失败(图谱从不修剪)
let merged_id = format!("merged_{}_{}_{}", src, tgt, rel);
conn.execute(
"INSERT OR REPLACE INTO graph_edges (id, source, target, relation, weight, evidence_count, namespace, created_at)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, 'shared', datetime('now'))",
params![
merged_id,
src, tgt, rel,
avg_weight, avg_evidence,
],
)?;
merged += deleted as usize - 1; // -1 因为已插入新边
}
Ok(merged)
}
/// 幂等添加列(迁移用)
fn ensure_column(&self, conn: &Connection, table: &str, column: &str, def: &str) -> Result<(), rusqlite::Error> {
let check_sql = format!("SELECT 1 FROM pragma_table_info('{}') WHERE name='{}'", table, column);
let exists: bool = conn.query_row(&check_sql, [], |_| Ok(true)).unwrap_or(false);
if !exists {
let alter_sql = format!("ALTER TABLE {} ADD COLUMN {} {}", table, column, def);
conn.execute(&alter_sql, [])?;
eprintln!("[graph_prune] migrated: added {} column to {} (def: {})", column, table, def);
}
Ok(())
}
/// PageRank 迭代更新
/// ⚠️ 2026-09-21 重写:原实现是 O(迭代 × 节点 × 边) 的朴素三重循环
/// (47k 节点 × 374k 边 × 20 轮 ≈ 3.5×10¹¹ 次比较)—— 这就是 09-20 那两次
/// 单次 full 整合烧掉 4h28m~4h41m CPU、内存峰值 3.7~3.8G 的真凶。
/// 改为标准「入边表 + 每轮 O(V+E)」:同一数据集实测毫秒级。
fn update_pagerank(&self, conn: &Connection) -> Result<(), rusqlite::Error> {
// 确保 pagerank 列存在(可能由 Go migrate 创建,也可能没有)
self.ensure_column(conn, "graph_nodes", "pagerank", "REAL DEFAULT 1.0")?;
let damping = 0.85_f64;
let iterations = 20_usize;
// 1. 节点表(跳过 id 为 NULL 的脏行:rusqlite 读 NULL 会报 Invalid column type)
let mut node_ids: Vec<String> = Vec::new();
let mut stmt = conn.prepare("SELECT id FROM graph_nodes WHERE id IS NOT NULL")?;
let rows = stmt.query_map([], |row| row.get::<_, String>(0))?;
for row in rows {
node_ids.push(row?);
}
if node_ids.is_empty() {
return Ok(());
}
let n = node_ids.len();
let idx: HashMap<&str, usize> = node_ids
.iter()
.enumerate()
.map(|(i, id)| (id.as_str(), i))
.collect();
// 2. 出边权重和(归一化用)+ 入边表(target → [(source, weight)])
let mut out_weight_sum = vec![0.0_f64; n];
let mut in_edges: Vec<Vec<(usize, f64)>> = vec![Vec::new(); n];
{
let mut edge_stmt = conn.prepare(
"SELECT source, target, weight FROM graph_edges WHERE source IS NOT NULL AND target IS NOT NULL",
)?;
let edge_rows = edge_stmt.query_map([], |row| {
Ok((
row.get::<_, String>(0)?,
row.get::<_, String>(1)?,
row.get::<_, Option<f64>>(2)?,
))
})?;
for edge in edge_rows {
if let Ok((src, tgt, wt)) = edge {
let (si, ti) = match (idx.get(src.as_str()), idx.get(tgt.as_str())) {
(Some(&s), Some(&t)) => (s, t),
_ => continue, // 悬空边(端点不在节点表)不计入
};
let w = wt.unwrap_or(0.0);
out_weight_sum[si] += w;
in_edges[ti].push((si, w));
}
}
}
// 3. 迭代:每轮 O(V + E)
let base = (1.0 - damping) / n as f64;
let mut ranks = vec![1.0_f64 / n as f64; n];
let mut next = vec![0.0_f64; n];
for _ in 0..iterations {
for v in 0..n {
let mut acc = 0.0_f64;
for &(u, w) in &in_edges[v] {
let osum = out_weight_sum[u];
if osum > 0.0 {
acc += ranks[u] * w / osum;
}
}
next[v] = base + damping * acc;
}
std::mem::swap(&mut ranks, &mut next);
}
// 4. 写回(单事务批量,避免逐条 UPDATE 各自提交)
let tx = conn.unchecked_transaction()?;
{
let mut upd = tx.prepare("UPDATE graph_nodes SET pagerank = ?1 WHERE id = ?2")?;
for (i, id) in node_ids.iter().enumerate() {
upd.execute(params![ranks[i], id])?;
}
}
tx.commit()?;
eprintln!(
"[graph_prune] PageRank updated for {} nodes ({} iterations)",
n, iterations
);
Ok(())
}
}