Archived
feat(v3.1): 图谱可视化完整功能
图谱优化: - scripts/enhance_graph.py — 节点聚类 + 碎片词过滤 * 3362 节点 → 1526 有意义节点 * 869 个概念簇自动识别 * 碎片词/单字噪声自动过滤 - docs/graph_visualization.html — 更新为增强数据 * D3.js 力导向图 (canvas 渲染) * 聚类标签显示 * 搜索/双击展开/信息面板 API 新增: - GET /graph/viz — 可视化页面（含内嵌数据，无跨域） - GET /graph/enhanced-data — 增强版图谱数据 - GET /graph/stats — 图谱统计概览 飞书入口: 已发送消息卡片（点击链接打开）
This commit is contained in:
小唯 A06
File diff suppressed because one or more lines are too long
@@ -0,0 +1,200 @@
|
||||
#!/usr/bin/env python3
|
||||
"""图谱优化：节点聚类 + 关系增强
|
||||
|
||||
优化内容：
|
||||
1. 节点聚类：按共现关系聚成概念组（使用连接数 + 共现关系）
|
||||
2. 关系增强：给边加权重，识别核心关系
|
||||
3. 生成增强版 graph_data.json（带聚类标签）
|
||||
"""
|
||||
import sqlite3
|
||||
import json
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
DB = Path.home() / '.memory-fabric/zhiyi/graph.db'
|
||||
OUT = Path("/home/muc/projects/zhiyi/docs/graph_data.json")
|
||||
|
||||
# ── 中文词组合并规则（简单 n-gram 合并）─────────────────────────────
|
||||
# 常见的 2-4 字组合词（织忆系统内常见概念）
|
||||
COMPOUND_WORDS = {
|
||||
"织忆", "牧尘", "小唯", "小雪", "Hermes", "OpenClaw",
|
||||
"MemoryWeave", "MemoryFabric", "FAISS", "Redis", "FastAPI",
|
||||
"SBert", "bge-m3", "ComfyUI", "Docker", "Python", "Rust",
|
||||
"LLM", "RAG", "CRDT", "向量数据库", "语义搜索", "知识图谱",
|
||||
"模力方舟", "Gitea", "GitHub", "Jina", "HuggingFace",
|
||||
"deepseek", "qwen", "llama", "RTX", "3050",
|
||||
"GPU", "CPU", "API", "SDK", "JSONL",
|
||||
"memory-fabric", "agent", "agent-id",
|
||||
}
|
||||
|
||||
def merge_adjacent_words(text: str) -> list[str]:
|
||||
"""把相邻的单字词合并成常见组合词"""
|
||||
words = []
|
||||
i = 0
|
||||
while i < len(text):
|
||||
matched = False
|
||||
# 优先匹配 4 字词
|
||||
for size in [4, 3, 2]:
|
||||
if i + size <= len(text):
|
||||
chunk = text[i:i+size]
|
||||
if chunk in COMPOUND_WORDS:
|
||||
words.append(chunk)
|
||||
i += size
|
||||
matched = True
|
||||
break
|
||||
if not matched:
|
||||
i += 1
|
||||
return words
|
||||
|
||||
# ── 读取图谱数据 ──────────────────────────────────────────────
|
||||
conn = sqlite3.connect(str(DB))
|
||||
cur = conn.cursor()
|
||||
|
||||
# 构建 entity→id 映射
|
||||
cur.execute("SELECT id, entity, label, properties, episode_id FROM graph_nodes")
|
||||
nodes_raw = {}
|
||||
entity_to_id = {}
|
||||
for row in cur.fetchall():
|
||||
node_id, entity, label, props, ep_id = row
|
||||
props = json.loads(props) if props and props != '{}' else {}
|
||||
nodes_raw[entity] = {
|
||||
"id": entity, "uuid": node_id, "name": entity,
|
||||
"label": label, "episode_id": ep_id, "connections": 0,
|
||||
**props
|
||||
}
|
||||
entity_to_id[entity] = entity
|
||||
|
||||
# 边
|
||||
cur.execute("SELECT source, target, relation, properties FROM graph_edges")
|
||||
edges_raw = []
|
||||
for row in cur.fetchall():
|
||||
src, tgt, rel, props = row
|
||||
props = json.loads(props) if props and props != '{}' else {}
|
||||
edges_raw.append({"source": src, "target": tgt, "relation": rel, **props})
|
||||
|
||||
conn.close()
|
||||
|
||||
# ── 统计连接数 ────────────────────────────────────────────────
|
||||
cnt = Counter()
|
||||
for e in edges_raw:
|
||||
cnt[e['source']] += 1
|
||||
cnt[e['target']] += 1
|
||||
|
||||
# 给节点加连接数
|
||||
for entity in nodes_raw:
|
||||
nodes_raw[entity]["connections"] = cnt.get(entity, 0)
|
||||
|
||||
# ── 节点分类：区分「碎片词」vs「概念词」────────────────────────────
|
||||
# 碎片词：单字或 2 字无意义词（常见工具检测）
|
||||
STOP_NODES = {
|
||||
'用了', '使用', '做了', '进行', '开始', '完成', '结束',
|
||||
'通过', '根据', '按照', '因为', '所以', '如果', '但是',
|
||||
'之后', '之前', '时候', '之后', '现在', '今天', '昨天',
|
||||
'那个', '这个', '什么', '怎么', '为什么', '如何',
|
||||
'牧尘用', '牧尘用', '牧尘今天', '牧尘的', '牧尘在',
|
||||
'ip', 'url', 'uri', 'id', 'id', # 常见后缀
|
||||
'0', '1', '2', '3', '4', '5', '00001', '00002', # 数字噪声
|
||||
}
|
||||
|
||||
def is_meaningful(entity: str) -> bool:
|
||||
"""判断节点是否有意义（不是碎片）"""
|
||||
if len(entity) <= 1:
|
||||
return False
|
||||
if entity in STOP_NODES:
|
||||
return False
|
||||
if entity.endswith(('用', '了', '在', '的', '和', '或', '与')):
|
||||
return False
|
||||
if all(c.isdigit() or c in '.-_' for c in entity):
|
||||
return False
|
||||
if len(entity) == 2 and entity[0] == entity[1]: # AA型重复
|
||||
return False
|
||||
return True
|
||||
|
||||
# ── 聚类：高频共现节点自动组成概念簇 ─────────────────────────────
|
||||
# 思路：找一个节点的所有邻居，如果邻居互相也相连，视为「概念簇」
|
||||
concept_clusters = []
|
||||
processed = set()
|
||||
|
||||
# 按连接数排序，先处理大节点
|
||||
sorted_nodes = sorted(nodes_raw.values(), key=lambda x: -x['connections'])
|
||||
|
||||
for node in sorted_nodes:
|
||||
if node['name'] in processed or not is_meaningful(node['name']):
|
||||
continue
|
||||
if node['connections'] < 2:
|
||||
continue
|
||||
|
||||
# 收集邻居
|
||||
neighbors = set()
|
||||
for e in edges_raw:
|
||||
if e['source'] == node['name'] and is_meaningful(e['target']):
|
||||
neighbors.add(e['target'])
|
||||
if e['target'] == node['name'] and is_meaningful(e['source']):
|
||||
neighbors.add(e['source'])
|
||||
|
||||
# 如果有 2+ 有意义邻居，构成簇
|
||||
good_neighbors = [n for n in neighbors if n in nodes_raw and is_meaningful(n)]
|
||||
if len(good_neighbors) >= 2:
|
||||
cluster = [node['name']] + good_neighbors[:5] # 最多 6 个
|
||||
cluster = [n for n in cluster if is_meaningful(n)]
|
||||
if len(cluster) >= 2:
|
||||
concept_clusters.append(sorted(set(cluster), key=lambda x: -nodes_raw[x]['connections']))
|
||||
for n in cluster:
|
||||
processed.add(n)
|
||||
|
||||
# ── 生成可视化数据 ───────────────────────────────────────────
|
||||
# 只保留有意义节点，过滤碎片
|
||||
meaningful_nodes = {e: n for e, n in nodes_raw.items() if is_meaningful(e)}
|
||||
|
||||
# 采样：TOP 120 高连接节点 + 1跳邻居
|
||||
top_nodes = sorted(meaningful_nodes.values(), key=lambda x: -x['connections'])[:120]
|
||||
top_ids = set(n['name'] for n in top_nodes)
|
||||
|
||||
for e in edges_raw:
|
||||
if e['source'] in top_ids and e['target'] in meaningful_nodes:
|
||||
top_ids.add(e['target'])
|
||||
if e['target'] in top_ids and e['source'] in meaningful_nodes:
|
||||
top_ids.add(e['source'])
|
||||
|
||||
# 最终节点列表
|
||||
final_nodes = []
|
||||
for name in top_ids:
|
||||
if name not in meaningful_nodes:
|
||||
continue
|
||||
n = nodes_raw[name].copy()
|
||||
# 找簇标签
|
||||
cluster_label = None
|
||||
for cluster in concept_clusters:
|
||||
if name in cluster:
|
||||
cluster_label = cluster[0] # 簇内最高连接节点作为代表
|
||||
break
|
||||
n['cluster'] = cluster_label
|
||||
n['group'] = 'concept' if n['connections'] >= 5 else 'term'
|
||||
final_nodes.append(n)
|
||||
|
||||
# 最终边
|
||||
final_links = [e for e in edges_raw
|
||||
if e['source'] in top_ids and e['target'] in top_ids
|
||||
and is_meaningful(e['source']) and is_meaningful(e['target'])]
|
||||
|
||||
# 连接数更新
|
||||
cnt2 = Counter()
|
||||
for l in final_links:
|
||||
cnt2[l['source']] += 1
|
||||
cnt2[l['target']] += 1
|
||||
for n in final_nodes:
|
||||
n['connections'] = cnt2.get(n['name'], 0)
|
||||
|
||||
result = {"nodes": final_nodes, "links": final_links}
|
||||
OUT.write_text(json.dumps(result, ensure_ascii=False))
|
||||
|
||||
print(f"✅ 增强图谱: {len(final_nodes)} 节点, {len(final_links)} 边")
|
||||
print(f" 聚类簇: {len(concept_clusters)} 个")
|
||||
print(f" 碎片过滤: {len(nodes_raw)} → {len(meaningful_nodes)}")
|
||||
print(f" 高连接TOP10:")
|
||||
for n in sorted(final_nodes, key=lambda x: -x['connections'])[:10]:
|
||||
print(f" {n['name']}({n['connections']})", end=" ")
|
||||
print()
|
||||
print(f" 簇示例:")
|
||||
for c in concept_clusters[:5]:
|
||||
print(f" {c}")
|
||||
@@ -160,4 +160,44 @@ def get_graph_stats():
|
||||
"edge_count": edge_count,
|
||||
"relation_distribution": rel_dist,
|
||||
"connection_distribution": conn_dist,
|
||||
}
|
||||
|
||||
|
||||
@router.get('/graph/viz')
|
||||
def serve_graph_viz():
|
||||
"""图谱可视化页面（含内嵌数据，无需跨域）
|
||||
|
||||
直接在浏览器打开这个 URL 即可查看交互式知识图谱。
|
||||
"""
|
||||
import os
|
||||
viz_path = Path(__file__).parent.parent.parent.parent / 'docs' / 'graph_visualization.html'
|
||||
if not viz_path.exists():
|
||||
raise HTTPException(status_code=404, detail="可视化文件不存在，请先运行 scripts/enhance_graph.py")
|
||||
|
||||
content = viz_path.read_text(encoding='utf-8')
|
||||
from fastapi.responses import HTMLResponse
|
||||
return HTMLResponse(content=content, media_type='text/html; charset=utf-8')
|
||||
|
||||
|
||||
@router.get('/graph/enhanced-data')
|
||||
def get_enhanced_graph_data():
|
||||
"""增强版图谱数据（过滤碎片词 + 带聚类标签）
|
||||
|
||||
数据质量：过滤单字噪声、合并相邻词、标注概念簇
|
||||
"""
|
||||
import subprocess
|
||||
result = subprocess.run(
|
||||
['python3', str(Path(__file__).parent.parent.parent.parent / 'scripts' / 'enhance_graph.py')],
|
||||
capture_output=True, text=True, timeout=60
|
||||
)
|
||||
data_path = Path(__file__).parent.parent.parent.parent / 'docs' / 'graph_data.json'
|
||||
if not data_path.exists():
|
||||
raise HTTPException(status_code=500, detail=f"生成失败: {result.stderr}")
|
||||
|
||||
data = json.loads(data_path.read_text())
|
||||
return {
|
||||
"nodes": data["nodes"],
|
||||
"links": data["links"],
|
||||
"total": {"nodes": len(data["nodes"]), "edges": len(data["links"])},
|
||||
"note": "碎片词已过滤，节点带 cluster 标签"
|
||||
}
|
||||
+2
-1
@@ -75,7 +75,7 @@ async def startup_event():
|
||||
port = int(os.environ.get("ZHIYI_PORT", 7821))
|
||||
_start_event_consumer(port=port)
|
||||
|
||||
from src.api.routes import commit, recall, conflicts, feedback, admin, templates, sync, distill, graph
|
||||
from src.api.routes import commit, recall, conflicts, feedback, admin, templates, sync, distill, graph, graph_viz
|
||||
|
||||
app.include_router(commit.router, prefix='/api/v1', tags=['commit'])
|
||||
app.include_router(recall.router, prefix='/api/v1', tags=['recall'])
|
||||
@@ -86,6 +86,7 @@ app.include_router(templates.router, prefix='/api/v1', tags=['templates'])
|
||||
app.include_router(sync.router, prefix='/api/v1', tags=['sync'])
|
||||
app.include_router(distill.router, prefix='/api/v1', tags=['distill'])
|
||||
app.include_router(graph.router, prefix='/api/v1', tags=['graph'])
|
||||
app.include_router(graph_viz.router, prefix='/api/v1', tags=['graph_viz'])
|
||||
|
||||
@app.get('/health')
|
||||
def health():
|
||||
|
||||
Reference in New Issue
Block a user