200 lines
7.5 KiB
Python
200 lines
7.5 KiB
Python
#!/usr/bin/env python3
|
||
"""图谱优化：节点聚类 + 关系增强
|
||
|
||
优化内容：
|
||
1. 节点聚类：按共现关系聚成概念组（使用连接数 + 共现关系）
|
||
2. 关系增强：给边加权重，识别核心关系
|
||
3. 生成增强版 graph_data.json（带聚类标签）
|
||
"""
|
||
import sqlite3
|
||
import json
|
||
from collections import Counter, defaultdict
|
||
from pathlib import Path
|
||
|
||
DB = Path.home() / '.memory-fabric/zhiyi/graph.db'
|
||
OUT = Path("/home/muc/projects/zhiyi/docs/graph_data.json")
|
||
|
||
# ── 中文词组合并规则（简单 n-gram 合并）─────────────────────────────
|
||
# 常见的 2-4 字组合词（织忆系统内常见概念）
|
||
COMPOUND_WORDS = {
|
||
"织忆", "牧尘", "小唯", "小雪", "Hermes", "OpenClaw",
|
||
"MemoryWeave", "MemoryFabric", "FAISS", "Redis", "FastAPI",
|
||
"SBert", "bge-m3", "ComfyUI", "Docker", "Python", "Rust",
|
||
"LLM", "RAG", "CRDT", "向量数据库", "语义搜索", "知识图谱",
|
||
"模力方舟", "Gitea", "GitHub", "Jina", "HuggingFace",
|
||
"deepseek", "qwen", "llama", "RTX", "3050",
|
||
"GPU", "CPU", "API", "SDK", "JSONL",
|
||
"memory-fabric", "agent", "agent-id",
|
||
}
|
||
|
||
def merge_adjacent_words(text: str) -> list[str]:
|
||
"""把相邻的单字词合并成常见组合词"""
|
||
words = []
|
||
i = 0
|
||
while i < len(text):
|
||
matched = False
|
||
# 优先匹配 4 字词
|
||
for size in [4, 3, 2]:
|
||
if i + size <= len(text):
|
||
chunk = text[i:i+size]
|
||
if chunk in COMPOUND_WORDS:
|
||
words.append(chunk)
|
||
i += size
|
||
matched = True
|
||
break
|
||
if not matched:
|
||
i += 1
|
||
return words
|
||
|
||
# ── 读取图谱数据 ──────────────────────────────────────────────
|
||
conn = sqlite3.connect(str(DB))
|
||
cur = conn.cursor()
|
||
|
||
# 构建 entity→id 映射
|
||
cur.execute("SELECT id, entity, label, properties, episode_id FROM graph_nodes")
|
||
nodes_raw = {}
|
||
entity_to_id = {}
|
||
for row in cur.fetchall():
|
||
node_id, entity, label, props, ep_id = row
|
||
props = json.loads(props) if props and props != '{}' else {}
|
||
nodes_raw[entity] = {
|
||
"id": entity, "uuid": node_id, "name": entity,
|
||
"label": label, "episode_id": ep_id, "connections": 0,
|
||
**props
|
||
}
|
||
entity_to_id[entity] = entity
|
||
|
||
# 边
|
||
cur.execute("SELECT source, target, relation, properties FROM graph_edges")
|
||
edges_raw = []
|
||
for row in cur.fetchall():
|
||
src, tgt, rel, props = row
|
||
props = json.loads(props) if props and props != '{}' else {}
|
||
edges_raw.append({"source": src, "target": tgt, "relation": rel, **props})
|
||
|
||
conn.close()
|
||
|
||
# ── 统计连接数 ────────────────────────────────────────────────
|
||
cnt = Counter()
|
||
for e in edges_raw:
|
||
cnt[e['source']] += 1
|
||
cnt[e['target']] += 1
|
||
|
||
# 给节点加连接数
|
||
for entity in nodes_raw:
|
||
nodes_raw[entity]["connections"] = cnt.get(entity, 0)
|
||
|
||
# ── 节点分类：区分「碎片词」vs「概念词」────────────────────────────
|
||
# 碎片词：单字或 2 字无意义词（常见工具检测）
|
||
STOP_NODES = {
|
||
'用了', '使用', '做了', '进行', '开始', '完成', '结束',
|
||
'通过', '根据', '按照', '因为', '所以', '如果', '但是',
|
||
'之后', '之前', '时候', '之后', '现在', '今天', '昨天',
|
||
'那个', '这个', '什么', '怎么', '为什么', '如何',
|
||
'牧尘用', '牧尘用', '牧尘今天', '牧尘的', '牧尘在',
|
||
'ip', 'url', 'uri', 'id', 'id', # 常见后缀
|
||
'0', '1', '2', '3', '4', '5', '00001', '00002', # 数字噪声
|
||
}
|
||
|
||
def is_meaningful(entity: str) -> bool:
|
||
"""判断节点是否有意义（不是碎片）"""
|
||
if len(entity) <= 1:
|
||
return False
|
||
if entity in STOP_NODES:
|
||
return False
|
||
if entity.endswith(('用', '了', '在', '的', '和', '或', '与')):
|
||
return False
|
||
if all(c.isdigit() or c in '.-_' for c in entity):
|
||
return False
|
||
if len(entity) == 2 and entity[0] == entity[1]: # AA型重复
|
||
return False
|
||
return True
|
||
|
||
# ── 聚类：高频共现节点自动组成概念簇 ─────────────────────────────
|
||
# 思路：找一个节点的所有邻居，如果邻居互相也相连，视为「概念簇」
|
||
concept_clusters = []
|
||
processed = set()
|
||
|
||
# 按连接数排序，先处理大节点
|
||
sorted_nodes = sorted(nodes_raw.values(), key=lambda x: -x['connections'])
|
||
|
||
for node in sorted_nodes:
|
||
if node['name'] in processed or not is_meaningful(node['name']):
|
||
continue
|
||
if node['connections'] < 2:
|
||
continue
|
||
|
||
# 收集邻居
|
||
neighbors = set()
|
||
for e in edges_raw:
|
||
if e['source'] == node['name'] and is_meaningful(e['target']):
|
||
neighbors.add(e['target'])
|
||
if e['target'] == node['name'] and is_meaningful(e['source']):
|
||
neighbors.add(e['source'])
|
||
|
||
# 如果有 2+ 有意义邻居，构成簇
|
||
good_neighbors = [n for n in neighbors if n in nodes_raw and is_meaningful(n)]
|
||
if len(good_neighbors) >= 2:
|
||
cluster = [node['name']] + good_neighbors[:5] # 最多 6 个
|
||
cluster = [n for n in cluster if is_meaningful(n)]
|
||
if len(cluster) >= 2:
|
||
concept_clusters.append(sorted(set(cluster), key=lambda x: -nodes_raw[x]['connections']))
|
||
for n in cluster:
|
||
processed.add(n)
|
||
|
||
# ── 生成可视化数据 ───────────────────────────────────────────
|
||
# 只保留有意义节点，过滤碎片
|
||
meaningful_nodes = {e: n for e, n in nodes_raw.items() if is_meaningful(e)}
|
||
|
||
# 采样：TOP 120 高连接节点 + 1跳邻居
|
||
top_nodes = sorted(meaningful_nodes.values(), key=lambda x: -x['connections'])[:120]
|
||
top_ids = set(n['name'] for n in top_nodes)
|
||
|
||
for e in edges_raw:
|
||
if e['source'] in top_ids and e['target'] in meaningful_nodes:
|
||
top_ids.add(e['target'])
|
||
if e['target'] in top_ids and e['source'] in meaningful_nodes:
|
||
top_ids.add(e['source'])
|
||
|
||
# 最终节点列表
|
||
final_nodes = []
|
||
for name in top_ids:
|
||
if name not in meaningful_nodes:
|
||
continue
|
||
n = nodes_raw[name].copy()
|
||
# 找簇标签
|
||
cluster_label = None
|
||
for cluster in concept_clusters:
|
||
if name in cluster:
|
||
cluster_label = cluster[0] # 簇内最高连接节点作为代表
|
||
break
|
||
n['cluster'] = cluster_label
|
||
n['group'] = 'concept' if n['connections'] >= 5 else 'term'
|
||
final_nodes.append(n)
|
||
|
||
# 最终边
|
||
final_links = [e for e in edges_raw
|
||
if e['source'] in top_ids and e['target'] in top_ids
|
||
and is_meaningful(e['source']) and is_meaningful(e['target'])]
|
||
|
||
# 连接数更新
|
||
cnt2 = Counter()
|
||
for l in final_links:
|
||
cnt2[l['source']] += 1
|
||
cnt2[l['target']] += 1
|
||
for n in final_nodes:
|
||
n['connections'] = cnt2.get(n['name'], 0)
|
||
|
||
result = {"nodes": final_nodes, "links": final_links}
|
||
OUT.write_text(json.dumps(result, ensure_ascii=False))
|
||
|
||
print(f"✅ 增强图谱: {len(final_nodes)} 节点, {len(final_links)} 边")
|
||
print(f" 聚类簇: {len(concept_clusters)} 个")
|
||
print(f" 碎片过滤: {len(nodes_raw)} → {len(meaningful_nodes)}")
|
||
print(f" 高连接TOP10:")
|
||
for n in sorted(final_nodes, key=lambda x: -x['connections'])[:10]:
|
||
print(f" {n['name']}({n['connections']})", end=" ")
|
||
print()
|
||
print(f" 簇示例:")
|
||
for c in concept_clusters[:5]:
|
||
print(f" {c}") |