#!/usr/bin/env python3 """图谱优化：节点聚类 + 关系增强 优化内容： 1. 节点聚类：按共现关系聚成概念组（使用连接数 + 共现关系） 2. 关系增强：给边加权重，识别核心关系 3. 生成增强版 graph_data.json（带聚类标签） """ import sqlite3 import json from collections import Counter, defaultdict from pathlib import Path DB = Path.home() / '.memory-fabric/zhiyi/graph.db' OUT = Path("/home/muc/projects/zhiyi/docs/graph_data.json") # ── 中文词组合并规则（简单 n-gram 合并）───────────────────────────── # 常见的 2-4 字组合词（织忆系统内常见概念） COMPOUND_WORDS = { "织忆", "牧尘", "小唯", "小雪", "Hermes", "OpenClaw", "MemoryWeave", "MemoryFabric", "FAISS", "Redis", "FastAPI", "SBert", "bge-m3", "ComfyUI", "Docker", "Python", "Rust", "LLM", "RAG", "CRDT", "向量数据库", "语义搜索", "知识图谱", "模力方舟", "Gitea", "GitHub", "Jina", "HuggingFace", "deepseek", "qwen", "llama", "RTX", "3050", "GPU", "CPU", "API", "SDK", "JSONL", "memory-fabric", "agent", "agent-id", } def merge_adjacent_words(text: str) -> list[str]: """把相邻的单字词合并成常见组合词""" words = [] i = 0 while i < len(text): matched = False # 优先匹配 4 字词 for size in [4, 3, 2]: if i + size <= len(text): chunk = text[i:i+size] if chunk in COMPOUND_WORDS: words.append(chunk) i += size matched = True break if not matched: i += 1 return words # ── 读取图谱数据 ────────────────────────────────────────────── conn = sqlite3.connect(str(DB)) cur = conn.cursor() # 构建 entity→id 映射 cur.execute("SELECT id, entity, label, properties, episode_id FROM graph_nodes") nodes_raw = {} entity_to_id = {} for row in cur.fetchall(): node_id, entity, label, props, ep_id = row props = json.loads(props) if props and props != '{}' else {} nodes_raw[entity] = { "id": entity, "uuid": node_id, "name": entity, "label": label, "episode_id": ep_id, "connections": 0, **props } entity_to_id[entity] = entity # 边 cur.execute("SELECT source, target, relation, properties FROM graph_edges") edges_raw = [] for row in cur.fetchall(): src, tgt, rel, props = row props = json.loads(props) if props and props != '{}' else {} edges_raw.append({"source": src, "target": tgt, "relation": rel, **props}) conn.close() # ── 统计连接数 ──────────────────────────────────────────────── cnt = Counter() for e in edges_raw: cnt[e['source']] += 1 cnt[e['target']] += 1 # 给节点加连接数 for entity in nodes_raw: nodes_raw[entity]["connections"] = cnt.get(entity, 0) # ── 节点分类：区分「碎片词」vs「概念词」──────────────────────────── # 碎片词：单字或 2 字无意义词（常见工具检测） STOP_NODES = { '用了', '使用', '做了', '进行', '开始', '完成', '结束', '通过', '根据', '按照', '因为', '所以', '如果', '但是', '之后', '之前', '时候', '之后', '现在', '今天', '昨天', '那个', '这个', '什么', '怎么', '为什么', '如何', '牧尘用', '牧尘用', '牧尘今天', '牧尘的', '牧尘在', 'ip', 'url', 'uri', 'id', 'id', # 常见后缀 '0', '1', '2', '3', '4', '5', '00001', '00002', # 数字噪声 } def is_meaningful(entity: str) -> bool: """判断节点是否有意义（不是碎片）""" if len(entity) <= 1: return False if entity in STOP_NODES: return False if entity.endswith(('用', '了', '在', '的', '和', '或', '与')): return False if all(c.isdigit() or c in '.-_' for c in entity): return False if len(entity) == 2 and entity[0] == entity[1]: # AA型重复 return False return True # ── 聚类：高频共现节点自动组成概念簇 ───────────────────────────── # 思路：找一个节点的所有邻居，如果邻居互相也相连，视为「概念簇」 concept_clusters = [] processed = set() # 按连接数排序，先处理大节点 sorted_nodes = sorted(nodes_raw.values(), key=lambda x: -x['connections']) for node in sorted_nodes: if node['name'] in processed or not is_meaningful(node['name']): continue if node['connections'] < 2: continue # 收集邻居 neighbors = set() for e in edges_raw: if e['source'] == node['name'] and is_meaningful(e['target']): neighbors.add(e['target']) if e['target'] == node['name'] and is_meaningful(e['source']): neighbors.add(e['source']) # 如果有 2+ 有意义邻居，构成簇 good_neighbors = [n for n in neighbors if n in nodes_raw and is_meaningful(n)] if len(good_neighbors) >= 2: cluster = [node['name']] + good_neighbors[:5] # 最多 6 个 cluster = [n for n in cluster if is_meaningful(n)] if len(cluster) >= 2: concept_clusters.append(sorted(set(cluster), key=lambda x: -nodes_raw[x]['connections'])) for n in cluster: processed.add(n) # ── 生成可视化数据 ─────────────────────────────────────────── # 只保留有意义节点，过滤碎片 meaningful_nodes = {e: n for e, n in nodes_raw.items() if is_meaningful(e)} # 采样：TOP 120 高连接节点 + 1跳邻居 top_nodes = sorted(meaningful_nodes.values(), key=lambda x: -x['connections'])[:120] top_ids = set(n['name'] for n in top_nodes) for e in edges_raw: if e['source'] in top_ids and e['target'] in meaningful_nodes: top_ids.add(e['target']) if e['target'] in top_ids and e['source'] in meaningful_nodes: top_ids.add(e['source']) # 最终节点列表 final_nodes = [] for name in top_ids: if name not in meaningful_nodes: continue n = nodes_raw[name].copy() # 找簇标签 cluster_label = None for cluster in concept_clusters: if name in cluster: cluster_label = cluster[0] # 簇内最高连接节点作为代表 break n['cluster'] = cluster_label n['group'] = 'concept' if n['connections'] >= 5 else 'term' final_nodes.append(n) # 最终边 final_links = [e for e in edges_raw if e['source'] in top_ids and e['target'] in top_ids and is_meaningful(e['source']) and is_meaningful(e['target'])] # 连接数更新 cnt2 = Counter() for l in final_links: cnt2[l['source']] += 1 cnt2[l['target']] += 1 for n in final_nodes: n['connections'] = cnt2.get(n['name'], 0) result = {"nodes": final_nodes, "links": final_links} OUT.write_text(json.dumps(result, ensure_ascii=False)) print(f"✅ 增强图谱: {len(final_nodes)} 节点, {len(final_links)} 边") print(f" 聚类簇: {len(concept_clusters)} 个") print(f" 碎片过滤: {len(nodes_raw)} → {len(meaningful_nodes)}") print(f" 高连接TOP10:") for n in sorted(final_nodes, key=lambda x: -x['connections'])[:10]: print(f" {n['name']}({n['connections']})", end=" ") print() print(f" 簇示例:") for c in concept_clusters[:5]: print(f" {c}")