1. PageRank 三重循环 O(47k×374k×20≈3.5e11) → 入边表 O(V+E): 单次 full 整合 4h41m → 95.2s,内存峰值 3.8G → 237MB(实测) 2. 图谱修剪 merge 边 id 加 relation + INSERT OR REPLACE: 修复 09-18 起 UNIQUE constraint 静默失败 → 主库首次跑消 9.8万冗余边 3. LanceDB 写入去抖(server.go + validator.go): 被动验证只写 Confidence 实际变化的记录;原先每输入重写全库 6000 条 → 版本快照 O(n²),每日 ~1.9万版本/十几G 膨胀的根因 4. 新增 scripts/lance_vacuum_online.py:在线 VACUUM(不停服务)+ 24h 版本窗口(可回滚)+ 行数门禁(减少才报警);替代停服务版 每日 vacuum → cron 已改为在线版 实测:侧车 RSS 1.09G→69MB;memories 分片 5383→1;服务全程在线
266 lines
10 KiB
Rust
266 lines
10 KiB
Rust
// 知识图谱修剪 — SQLite 操作 (rusqlite)
|
||
// 深度整合步骤 2:孤立节点/低权重边/冗余边合并/PageRank 更新
|
||
|
||
use rusqlite::{Connection, params};
|
||
use serde::{Deserialize, Serialize};
|
||
use std::collections::{HashMap, HashSet};
|
||
use std::path::Path;
|
||
|
||
/// 修剪统计
|
||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||
pub struct PruneStats {
|
||
pub isolated_nodes_removed: usize,
|
||
pub low_weight_edges_removed: usize,
|
||
pub redundant_edges_merged: usize,
|
||
pub nodes_before: usize,
|
||
pub nodes_after: usize,
|
||
pub edges_before: usize,
|
||
pub edges_after: usize,
|
||
}
|
||
|
||
/// 图谱修剪器
|
||
pub struct GraphPruner {
|
||
db_path: String,
|
||
/// 孤立节点阈值: 14天无关联边
|
||
isolated_days: u32,
|
||
/// 低权重阈值
|
||
low_weight_threshold: f64,
|
||
/// 最多 BFS 跳数
|
||
max_hops: usize,
|
||
}
|
||
|
||
impl GraphPruner {
|
||
pub fn new(db_path: &str) -> Self {
|
||
Self {
|
||
db_path: db_path.to_string(),
|
||
isolated_days: 14,
|
||
low_weight_threshold: 0.15,
|
||
max_hops: 3,
|
||
}
|
||
}
|
||
|
||
/// 执行全部修剪流程
|
||
pub fn prune(&self) -> Result<PruneStats, Box<dyn std::error::Error>> {
|
||
let conn = Connection::open(&self.db_path)?;
|
||
|
||
// 统计修剪前
|
||
let nodes_before: usize = conn.query_row("SELECT COUNT(*) FROM graph_nodes", [], |r| r.get(0))?;
|
||
let edges_before: usize = conn.query_row("SELECT COUNT(*) FROM graph_edges", [], |r| r.get(0))?;
|
||
|
||
// Step 1: 删除孤立节点 (14天无关联边)
|
||
let isolated_removed = self.remove_isolated_nodes(&conn)?;
|
||
|
||
// Step 2: 删除低权重边 (weight < 0.15)
|
||
let low_weight_removed = self.remove_low_weight_edges(&conn)?;
|
||
|
||
// Step 3: 合并冗余边 (同 source→target 的多条边 → 取 weight 加权平均)
|
||
let redundant_merged = self.merge_redundant_edges(&conn)?;
|
||
|
||
// Step 4: 更新 PageRank
|
||
self.update_pagerank(&conn)?;
|
||
|
||
// 统计修剪后
|
||
let nodes_after: usize = conn.query_row("SELECT COUNT(*) FROM graph_nodes", [], |r| r.get(0))?;
|
||
let edges_after: usize = conn.query_row("SELECT COUNT(*) FROM graph_edges", [], |r| r.get(0))?;
|
||
|
||
let stats = PruneStats {
|
||
isolated_nodes_removed: isolated_removed,
|
||
low_weight_edges_removed: low_weight_removed,
|
||
redundant_edges_merged: redundant_merged,
|
||
nodes_before,
|
||
nodes_after,
|
||
edges_before,
|
||
edges_after,
|
||
};
|
||
|
||
eprintln!(
|
||
"[graph_prune] nodes {}→{}, edges {}→{} (isolated={}, low_wt={}, merged={})",
|
||
nodes_before, nodes_after, edges_before, edges_after,
|
||
isolated_removed, low_weight_removed, redundant_merged,
|
||
);
|
||
|
||
conn.close().ok();
|
||
Ok(stats)
|
||
}
|
||
|
||
/// 删除孤立节点
|
||
fn remove_isolated_nodes(&self, conn: &Connection) -> Result<usize, rusqlite::Error> {
|
||
let cutoff = format!("-{} days", self.isolated_days);
|
||
let deleted = conn.execute(
|
||
"DELETE FROM graph_nodes WHERE id IN (
|
||
SELECT n.id FROM graph_nodes n
|
||
LEFT JOIN graph_edges e ON n.id = e.source OR n.id = e.target
|
||
WHERE e.id IS NULL
|
||
AND n.created_at < datetime('now', '-14 days')
|
||
)",
|
||
[],
|
||
)?;
|
||
Ok(deleted)
|
||
}
|
||
|
||
/// 删除低权重边
|
||
fn remove_low_weight_edges(&self, conn: &Connection) -> Result<usize, rusqlite::Error> {
|
||
let deleted = conn.execute(
|
||
"DELETE FROM graph_edges WHERE weight < ?1",
|
||
params![self.low_weight_threshold],
|
||
)?;
|
||
Ok(deleted)
|
||
}
|
||
|
||
/// 合并冗余边
|
||
fn merge_redundant_edges(&self, conn: &Connection) -> Result<usize, rusqlite::Error> {
|
||
// 查找 (source, target, relation) 相同的冗余边
|
||
let mut stmt = conn.prepare(
|
||
"SELECT source, target, relation, COUNT(*) as cnt,
|
||
SUM(weight) as total_weight, SUM(evidence_count) as total_evidence
|
||
FROM graph_edges
|
||
GROUP BY source, target, relation
|
||
HAVING cnt > 1"
|
||
)?;
|
||
|
||
let to_merge: Vec<(String, String, String, i64, f64, i64)> = stmt.query_map(
|
||
[],
|
||
|row| Ok((
|
||
row.get::<_, String>(0)?,
|
||
row.get::<_, String>(1)?,
|
||
row.get::<_, String>(2)?,
|
||
row.get::<_, i64>(3)?,
|
||
row.get::<_, f64>(4)?,
|
||
row.get::<_, i64>(5)?,
|
||
))
|
||
)?.filter_map(|r| r.ok()).collect();
|
||
|
||
let mut merged = 0usize;
|
||
for (src, tgt, rel, cnt, total_wt, total_ev) in to_merge {
|
||
// 删除所有冗余边
|
||
let deleted = conn.execute(
|
||
"DELETE FROM graph_edges WHERE source=?1 AND target=?2 AND relation=?3",
|
||
params![src, tgt, rel],
|
||
)?;
|
||
|
||
// 插入合并后单边(加权平均)
|
||
let avg_weight = total_wt / cnt as f64;
|
||
let avg_evidence = total_ev / cnt;
|
||
// ⚠️ 2026-09-21 修:原 id 只含 (src,tgt),同一对节点有多条 relation 时 id 撞车
|
||
// → UNIQUE constraint failed: graph_edges.id → 每次 full 整合 Step 2 全失败(图谱从不修剪)
|
||
let merged_id = format!("merged_{}_{}_{}", src, tgt, rel);
|
||
conn.execute(
|
||
"INSERT OR REPLACE INTO graph_edges (id, source, target, relation, weight, evidence_count, namespace, created_at)
|
||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, 'shared', datetime('now'))",
|
||
params![
|
||
merged_id,
|
||
src, tgt, rel,
|
||
avg_weight, avg_evidence,
|
||
],
|
||
)?;
|
||
|
||
merged += deleted as usize - 1; // -1 因为已插入新边
|
||
}
|
||
|
||
Ok(merged)
|
||
}
|
||
|
||
/// 幂等添加列(迁移用)
|
||
fn ensure_column(&self, conn: &Connection, table: &str, column: &str, def: &str) -> Result<(), rusqlite::Error> {
|
||
let check_sql = format!("SELECT 1 FROM pragma_table_info('{}') WHERE name='{}'", table, column);
|
||
let exists: bool = conn.query_row(&check_sql, [], |_| Ok(true)).unwrap_or(false);
|
||
if !exists {
|
||
let alter_sql = format!("ALTER TABLE {} ADD COLUMN {} {}", table, column, def);
|
||
conn.execute(&alter_sql, [])?;
|
||
eprintln!("[graph_prune] migrated: added {} column to {} (def: {})", column, table, def);
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
/// PageRank 迭代更新
|
||
/// ⚠️ 2026-09-21 重写:原实现是 O(迭代 × 节点 × 边) 的朴素三重循环
|
||
/// (47k 节点 × 374k 边 × 20 轮 ≈ 3.5×10¹¹ 次比较)—— 这就是 09-20 那两次
|
||
/// 单次 full 整合烧掉 4h28m~4h41m CPU、内存峰值 3.7~3.8G 的真凶。
|
||
/// 改为标准「入边表 + 每轮 O(V+E)」:同一数据集实测毫秒级。
|
||
fn update_pagerank(&self, conn: &Connection) -> Result<(), rusqlite::Error> {
|
||
// 确保 pagerank 列存在(可能由 Go migrate 创建,也可能没有)
|
||
self.ensure_column(conn, "graph_nodes", "pagerank", "REAL DEFAULT 1.0")?;
|
||
|
||
let damping = 0.85_f64;
|
||
let iterations = 20_usize;
|
||
|
||
// 1. 节点表(跳过 id 为 NULL 的脏行:rusqlite 读 NULL 会报 Invalid column type)
|
||
let mut node_ids: Vec<String> = Vec::new();
|
||
let mut stmt = conn.prepare("SELECT id FROM graph_nodes WHERE id IS NOT NULL")?;
|
||
let rows = stmt.query_map([], |row| row.get::<_, String>(0))?;
|
||
for row in rows {
|
||
node_ids.push(row?);
|
||
}
|
||
if node_ids.is_empty() {
|
||
return Ok(());
|
||
}
|
||
let n = node_ids.len();
|
||
let idx: HashMap<&str, usize> = node_ids
|
||
.iter()
|
||
.enumerate()
|
||
.map(|(i, id)| (id.as_str(), i))
|
||
.collect();
|
||
|
||
// 2. 出边权重和(归一化用)+ 入边表(target → [(source, weight)])
|
||
let mut out_weight_sum = vec![0.0_f64; n];
|
||
let mut in_edges: Vec<Vec<(usize, f64)>> = vec![Vec::new(); n];
|
||
{
|
||
let mut edge_stmt = conn.prepare(
|
||
"SELECT source, target, weight FROM graph_edges WHERE source IS NOT NULL AND target IS NOT NULL",
|
||
)?;
|
||
let edge_rows = edge_stmt.query_map([], |row| {
|
||
Ok((
|
||
row.get::<_, String>(0)?,
|
||
row.get::<_, String>(1)?,
|
||
row.get::<_, Option<f64>>(2)?,
|
||
))
|
||
})?;
|
||
for edge in edge_rows {
|
||
if let Ok((src, tgt, wt)) = edge {
|
||
let (si, ti) = match (idx.get(src.as_str()), idx.get(tgt.as_str())) {
|
||
(Some(&s), Some(&t)) => (s, t),
|
||
_ => continue, // 悬空边(端点不在节点表)不计入
|
||
};
|
||
let w = wt.unwrap_or(0.0);
|
||
out_weight_sum[si] += w;
|
||
in_edges[ti].push((si, w));
|
||
}
|
||
}
|
||
}
|
||
|
||
// 3. 迭代:每轮 O(V + E)
|
||
let base = (1.0 - damping) / n as f64;
|
||
let mut ranks = vec![1.0_f64 / n as f64; n];
|
||
let mut next = vec![0.0_f64; n];
|
||
for _ in 0..iterations {
|
||
for v in 0..n {
|
||
let mut acc = 0.0_f64;
|
||
for &(u, w) in &in_edges[v] {
|
||
let osum = out_weight_sum[u];
|
||
if osum > 0.0 {
|
||
acc += ranks[u] * w / osum;
|
||
}
|
||
}
|
||
next[v] = base + damping * acc;
|
||
}
|
||
std::mem::swap(&mut ranks, &mut next);
|
||
}
|
||
|
||
// 4. 写回(单事务批量,避免逐条 UPDATE 各自提交)
|
||
let tx = conn.unchecked_transaction()?;
|
||
{
|
||
let mut upd = tx.prepare("UPDATE graph_nodes SET pagerank = ?1 WHERE id = ?2")?;
|
||
for (i, id) in node_ids.iter().enumerate() {
|
||
upd.execute(params![ranks[i], id])?;
|
||
}
|
||
}
|
||
tx.commit()?;
|
||
|
||
eprintln!(
|
||
"[graph_prune] PageRank updated for {} nodes ({} iterations)",
|
||
n, iterations
|
||
);
|
||
Ok(())
|
||
}
|
||
}
|