memoryweave/go/internal/governance/graph_file.go

778 lines
20 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

// 织忆 MemoryWeave — 文件持久化知识图谱(多 Agent 共享,零外部依赖)
// 用 JSON + flock 实现跨进程并发安全,替代 InMemoryGraph 和 SQLite
package governance
import (
"encoding/json"
"fmt"
"math"
"os"
"strings"
"sync"
"syscall"
"time"
"github.com/xiaoxue/memoryweave/internal/models"
)
// ─── 持久化结构 ──────────────────────────────────────────
// FileGraphNode 带 pagerank + evidence_count 的节点
type FileGraphNode struct {
ID string `json:"id"`
Name string `json:"name"`
Type string `json:"type"`
Namespace string `json:"namespace"`
PageRank float64 `json:"pagerank"`
EvidenceCount int `json:"evidence_count"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
}
// FileGraphEdge 带权重的边
type FileGraphEdge struct {
ID string `json:"id"`
Source string `json:"source"`
Target string `json:"target"`
Relation string `json:"relation"`
Weight float64 `json:"weight"`
Namespace string `json:"namespace"`
CreatedAt string `json:"created_at"`
}
// FileGraphData 持久化到磁盘的完整数据结构
type FileGraphData struct {
Version int `json:"version"`
Nodes []*FileGraphNode `json:"nodes"`
Edges []*FileGraphEdge `json:"edges"`
}
// ─── FileGraph ───────────────────────────────────────────
// FileGraph 基于 JSON 文件 + flock 的多 Agent 共享知识图谱
// 所有写操作获取排他锁,所有读操作获取共享锁
type FileGraph struct {
mu sync.RWMutex // 进程内并发控制
filePath string // JSON 文件路径
nodes map[string]*FileGraphNode
edges []*FileGraphEdge
}
// NewFileGraph 创建或加载图谱文件
func NewFileGraph(filePath string) (*FileGraph, error) {
fg := &FileGraph{
filePath: filePath,
nodes: make(map[string]*FileGraphNode),
}
// 尝试加载已有数据
if err := fg.load(); err != nil && !os.IsNotExist(err) {
return nil, fmt.Errorf("load graph: %w", err)
}
// 自动定期保存
go fg.autoSave(5 * time.Minute)
return fg, nil
}
// ─── 文件锁 ──────────────────────────────────────────────
func (fg *FileGraph) lockFile(fd *os.File, exclusive bool) error {
how := syscall.LOCK_SH
if exclusive {
how = syscall.LOCK_EX
}
return syscall.Flock(int(fd.Fd()), how)
}
func (fg *FileGraph) unlockFile(fd *os.File) {
syscall.Flock(int(fd.Fd()), syscall.LOCK_UN)
}
// ─── 持久化 ──────────────────────────────────────────────
func (fg *FileGraph) load() error {
fd, err := os.OpenFile(fg.filePath, os.O_RDONLY|os.O_CREATE, 0644)
if err != nil {
return err
}
defer fd.Close()
if err := fg.lockFile(fd, false); err != nil {
return err
}
defer fg.unlockFile(fd)
stat, err := fd.Stat()
if err != nil {
return err
}
if stat.Size() == 0 {
return nil // 空文件,正常
}
var data FileGraphData
if err := json.NewDecoder(fd).Decode(&data); err != nil {
return err
}
fg.mu.Lock()
for _, n := range data.Nodes {
fg.nodes[n.ID] = n
}
fg.edges = data.Edges
fg.mu.Unlock()
return nil
}
func (fg *FileGraph) save() error {
fd, err := os.OpenFile(fg.filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
return err
}
defer fd.Close()
if err := fg.lockFile(fd, true); err != nil {
return err
}
defer fg.unlockFile(fd)
fg.mu.RLock()
data := FileGraphData{Version: 2}
for _, n := range fg.nodes {
data.Nodes = append(data.Nodes, n)
}
data.Edges = fg.edges
fg.mu.RUnlock()
return json.NewEncoder(fd).Encode(data)
}
func (fg *FileGraph) autoSave(interval time.Duration) {
ticker := time.NewTicker(interval)
for range ticker.C {
fg.save()
}
}
// ─── GraphStore 接口实现 ─────────────────────────────────
func (fg *FileGraph) AddNode(id, name, nodeType, namespace string) error {
fg.mu.Lock()
defer fg.mu.Unlock()
now := time.Now().Format(time.RFC3339)
if existing, ok := fg.nodes[id]; ok {
existing.Name = name
existing.Type = nodeType
existing.EvidenceCount++
existing.UpdatedAt = now
} else {
fg.nodes[id] = &FileGraphNode{
ID: id,
Name: name,
Type: nodeType,
Namespace: namespace,
EvidenceCount: 1,
PageRank: 0.15, // 初始 PageRank
CreatedAt: now,
UpdatedAt: now,
}
}
return fg.save()
}
func (fg *FileGraph) AddEdge(id, source, target, relation, namespace string, weight float64) error {
fg.mu.Lock()
defer fg.mu.Unlock()
// 更新 source/target 节点的 evidence_count
if s, ok := fg.nodes[source]; ok {
s.EvidenceCount++
}
if t, ok := fg.nodes[target]; ok {
t.EvidenceCount++
}
now := time.Now().Format(time.RFC3339)
fg.edges = append(fg.edges, &FileGraphEdge{
ID: id,
Source: source,
Target: target,
Relation: relation,
Weight: weight,
Namespace: namespace,
CreatedAt: now,
})
return fg.save()
}
// Navigate 多跳 BFS 导航E1.4: relationFilter 支持)
func (fg *FileGraph) Navigate(entity string, maxHops int, namespace string, relFilter []string) ([]map[string]interface{}, error) {
return fg.NavigateBiDir(entity, "", maxHops, namespace, relFilter)
}
// bfsNode 双向 BFS 节点(包级类型)
type bfsNode struct {
node string
parent string
hop int
edgeID string
weight float64
rel string
}
// NavigateBiDir 双向 BFS — 从 source 和目标同时扩展相遇时合并路径E1.1/E1.4
func (fg *FileGraph) NavigateBiDir(source, target string, maxHops int, namespace string, relFilter []string) ([]map[string]interface{}, error) {
fg.mu.RLock()
defer fg.mu.RUnlock()
if maxHops <= 0 {
maxHops = 2
}
// 构建邻接表
adj := make(map[string][]struct {
neighbor string
edge *FileGraphEdge
})
for _, e := range fg.edges {
if e.Namespace != namespace {
continue
}
adj[e.Source] = append(adj[e.Source], struct {
neighbor string
edge *FileGraphEdge
}{e.Target, e})
adj[e.Target] = append(adj[e.Target], struct {
neighbor string
edge *FileGraphEdge
}{e.Source, e})
}
if target == "" {
return fg.singleBFS(source, adj, maxHops), nil
}
// 双向forward 从 source 出发backward 从 target 出发
forwardVisited := map[string]*bfsNode{source: {node: source, hop: 0}}
backwardVisited := map[string]*bfsNode{target: {node: target, hop: 0}}
forwardQueue := []string{source}
backwardQueue := []string{target}
for hop := 1; hop <= maxHops; hop++ {
if len(forwardQueue) == 0 && len(backwardQueue) == 0 {
break
}
var nextForward []string
for _, current := range forwardQueue {
for _, nb := range adj[current] {
if _, seen := forwardVisited[nb.neighbor]; seen {
continue
}
bn := &bfsNode{node: nb.neighbor, parent: current, hop: hop,
edgeID: nb.edge.ID, weight: nb.edge.Weight, rel: nb.edge.Relation}
forwardVisited[nb.neighbor] = bn
nextForward = append(nextForward, nb.neighbor)
if bw, ok := backwardVisited[nb.neighbor]; ok {
return fg.mergePaths(forwardVisited, backwardVisited, bn, bw), nil
}
}
}
forwardQueue = nextForward
var nextBackward []string
for _, current := range backwardQueue {
for _, nb := range adj[current] {
if _, seen := backwardVisited[nb.neighbor]; seen {
continue
}
bn := &bfsNode{node: nb.neighbor, parent: current, hop: hop,
edgeID: nb.edge.ID, weight: nb.edge.Weight, rel: nb.edge.Relation}
backwardVisited[nb.neighbor] = bn
nextBackward = append(nextBackward, nb.neighbor)
if fw, ok := forwardVisited[nb.neighbor]; ok {
return fg.mergePaths(forwardVisited, backwardVisited, fw, bn), nil
}
}
}
backwardQueue = nextBackward
}
return fg.singleBFS(source, adj, maxHops), nil
}
func (fg *FileGraph) singleBFS(start string, adj map[string][]struct {
neighbor string
edge *FileGraphEdge
}, maxHops int) []map[string]interface{} {
visited := map[string]bool{start: true}
queue := []string{start}
var paths []map[string]interface{}
for hop := 1; hop <= maxHops && len(queue) > 0; hop++ {
var nextQueue []string
for _, current := range queue {
for _, nb := range adj[current] {
if visited[nb.neighbor] {
continue
}
visited[nb.neighbor] = true
nextQueue = append(nextQueue, nb.neighbor)
paths = append(paths, map[string]interface{}{
"edge_id": nb.edge.ID,
"source": current,
"target": nb.neighbor,
"relation": nb.edge.Relation,
"weight": nb.edge.Weight,
"hop": hop,
})
}
}
queue = nextQueue
}
return paths
}
func (fg *FileGraph) mergePaths(forward, backward map[string]*bfsNode, fw, bw *bfsNode) []map[string]interface{} {
var paths []map[string]interface{}
// 从 meeting point 沿 forward 回溯到 source
cur := fw
for cur != nil && cur.parent != "" {
paths = append(paths, map[string]interface{}{
"edge_id": cur.edgeID,
"source": cur.parent,
"target": cur.node,
"relation": cur.rel,
"weight": cur.weight,
"hop": cur.hop,
"direction": "forward",
})
cur = forward[cur.parent]
}
// 从 meeting point 沿 backward 回溯到 target反转方向
cur = bw
for cur != nil && cur.parent != "" {
paths = append(paths, map[string]interface{}{
"edge_id": cur.edgeID,
"source": cur.node, // 反转
"target": cur.parent,
"relation": cur.rel,
"weight": cur.weight,
"hop": cur.hop,
"direction": "backward",
})
cur = backward[cur.parent]
}
return paths
}
func (fg *FileGraph) Query(entity, relation, namespace string) []map[string]interface{} {
fg.mu.RLock()
defer fg.mu.RUnlock()
var results []map[string]interface{}
for _, e := range fg.edges {
if e.Namespace != namespace {
continue
}
if (e.Source == entity || e.Target == entity) &&
(relation == "" || e.Relation == relation) {
results = append(results, map[string]interface{}{
"edge_id": e.ID,
"source": e.Source,
"target": e.Target,
"relation": e.Relation,
"weight": e.Weight,
})
}
}
return results
}
func (fg *FileGraph) Stats() (nodeCount, edgeCount int, density float64) {
fg.mu.RLock()
defer fg.mu.RUnlock()
nodeCount = len(fg.nodes)
edgeCount = len(fg.edges)
if nodeCount > 1 {
density = float64(edgeCount) / float64(nodeCount*(nodeCount-1))
}
return
}
func (fg *FileGraph) Prune(minWeight float64) {
fg.mu.Lock()
defer fg.mu.Unlock()
var kept []*FileGraphEdge
for _, e := range fg.edges {
if e.Weight >= minWeight {
kept = append(kept, e)
}
}
fg.edges = kept
// 删除孤立节点
connected := make(map[string]bool)
for _, e := range fg.edges {
connected[e.Source] = true
connected[e.Target] = true
}
for id := range fg.nodes {
if !connected[id] {
delete(fg.nodes, id)
}
}
fg.save()
}
func (fg *FileGraph) CleanupNoiseNodes(dryRun bool) (int, []string, error) {
fg.mu.Lock()
defer fg.mu.Unlock()
// FileGraph 不需要脏数据清理(已迁移到 SQLite
return 0, nil, nil
}
// P0: FallbackTextSearch FileGraph stub已迁移到 SQLite
func (fg *FileGraph) FallbackTextSearch(query, namespace string, limit int) []map[string]interface{} {
return nil
}
// P2: 信任评分 stubFileGraph 不持久化信任数据)
func (fg *FileGraph) AddEdgeFeedback(edgeID string, helpful bool) error { return nil }
func (fg *FileGraph) IncrementEdgeRetrieval(edgeID string) error { return nil }
func (fg *FileGraph) UpdateEdgeTrustScores() error { return nil }
// ─── 图谱扩展 ────────────────────────────────────────────
func (fg *FileGraph) ExpandFromResults(results []models.RecallResult, namespace string, maxHops int) []models.RecallResult {
var expanded []models.RecallResult
seen := make(map[string]bool)
for _, r := range results {
seen[r.ID] = true
}
for _, r := range results {
paths, err := fg.Navigate(r.Category, maxHops, namespace, nil)
if err != nil {
continue
}
for _, p := range paths {
target, _ := p["target"].(string)
source, _ := p["source"].(string)
for _, id := range []string{target, source} {
if id != "" && !seen[id] {
seen[id] = true
expanded = append(expanded, models.RecallResult{
ID: id,
Category: "graph_expanded",
Score: 0.5,
})
}
}
}
}
return expanded
}
// ExpandWithSummary BFS 扩展 + 生成汇总语句 — E1 图谱导航增强
func (fg *FileGraph) ExpandWithSummary(results []models.RecallResult, namespace string, maxHops int) models.GraphBFSResult {
if maxHops <= 0 {
maxHops = 2
}
seenEntities := make(map[string]bool)
var relations []models.ExpandedRelation
for _, r := range results {
entities := extractFileGraphEntities(r.Content)
for _, entity := range entities {
if seenEntities[entity] {
continue
}
seenEntities[entity] = true
nodeID := normalizeFileGraphEntityID(entity)
paths, _ := fg.Navigate(nodeID, maxHops, namespace, nil)
for _, p := range paths {
from, _ := p["source"].(string)
to, _ := p["target"].(string)
rel, _ := p["relation"].(string)
weight, _ := p["weight"].(float64)
hop, _ := p["hop"].(int)
fromName := strings.TrimPrefix(from, "n_")
toName := strings.TrimPrefix(to, "n_")
rel = strings.TrimSpace(rel)
if rel == "" {
rel = "RELATED_TO"
}
relations = append(relations, models.ExpandedRelation{
From: fromName,
To: toName,
Relation: rel,
Hops: hop,
Weight: weight,
Score: r.Score * weight,
})
}
}
}
summary := buildBFSSummary(relations)
return models.GraphBFSResult{
ExpandedRelations: relations,
Summary: summary,
}
}
// extractFileGraphEntities 从文本提取实体FileGraph 用)
func extractFileGraphEntities(text string) []string {
var entities []string
seen := make(map[string]bool)
runes := []rune(text)
for i := 0; i < len(runes); {
r := runes[i]
// 中文字符
if r >= 0x4E00 && r <= 0x9FFF {
start := i
i++
for i < len(runes) && runes[i] >= 0x4E00 && runes[i] <= 0x9FFF {
i++
}
chinese := string(runes[start:i])
if len(chinese) >= 2 && len(chinese) <= 8 && !seen[chinese] {
seen[chinese] = true
entities = append(entities, chinese)
}
continue
}
// 英文/其他
start := i
for i < len(runes) {
r2 := runes[i]
if r2 >= 0x4E00 && r2 <= 0x9FFF {
break
}
i++
}
if i-start < 2 {
continue
}
w := string(runes[start:i])
w = strings.Trim(w, ",.;:!?,。;:!?、\"'()[]【】")
if len(w) < 2 {
continue
}
first := []rune(w)
if len(first) > 0 && first[0] >= 'A' && first[0] <= 'Z' {
lower := strings.ToLower(w)
if !seen[lower] {
seen[lower] = true
entities = append(entities, w)
}
}
}
return entities
}
// normalizeFileGraphEntityID 将自由文本转为实体 ID 格式
func normalizeFileGraphEntityID(name string) string {
clean := strings.Map(func(r rune) rune {
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' || r == '-' || r == ' ' {
return r
}
if r >= 0x4E00 && r <= 0x9FFF {
return r
}
return '_'
}, strings.TrimSpace(name))
clean = strings.ToLower(clean)
clean = strings.ReplaceAll(clean, " ", "_")
for strings.Contains(clean, "__") {
clean = strings.ReplaceAll(clean, "__", "_")
}
clean = strings.Trim(clean, "_")
if clean == "" {
return "n_unknown"
}
return "n_" + clean
}
// ─── 多 Agent 分析 ───────────────────────────────────────
// PageRank 计算所有节点的 PageRank
func (fg *FileGraph) PageRank(damping float64, iterations int) map[string]float64 {
fg.mu.RLock()
defer fg.mu.RUnlock()
if damping <= 0 {
damping = 0.85
}
if iterations <= 0 {
iterations = 20
}
N := float64(len(fg.nodes))
if N == 0 {
return nil
}
// 初始化
rank := make(map[string]float64)
for id := range fg.nodes {
rank[id] = 1.0 / N
}
// 出边计数
outDegree := make(map[string]int)
for _, e := range fg.edges {
outDegree[e.Source]++
}
// 迭代
for iter := 0; iter < iterations; iter++ {
newRank := make(map[string]float64)
var sinkRank float64
// 收集 dangling 节点(无出边的)的 rank
for id := range fg.nodes {
if outDegree[id] == 0 {
sinkRank += rank[id]
}
}
sinkContrib := sinkRank / N
for id := range fg.nodes {
newRank[id] = (1.0 - damping) / N
newRank[id] += damping * sinkContrib
}
// 沿边传播
for _, e := range fg.edges {
if outDegree[e.Source] > 0 {
contrib := damping * rank[e.Source] / float64(outDegree[e.Source])
newRank[e.Target] += contrib
}
}
rank = newRank
}
// 更新节点 PageRank
for id, r := range rank {
if n, ok := fg.nodes[id]; ok {
n.PageRank = math.Round(r*100000) / 100000
}
}
return rank
}
// EvidenceCount 返回某实体的证据数(被多少其他节点引用)
func (fg *FileGraph) EvidenceCount(entity string) int {
fg.mu.RLock()
defer fg.mu.RUnlock()
count := 0
for _, e := range fg.edges {
if e.Source == entity || e.Target == entity {
count++
}
}
return count
}
// GetEntityDegree E4.3: 返回实体的图谱度(入度+出度),度越高越优先保留
func (fg *FileGraph) GetEntityDegree(entity string) int {
return fg.EvidenceCount(entity) // 与 EvidenceCount 相同逻辑:统计 entity 作为 source 或 target 的边数
}
// ─── 强制保存 ────────────────────────────────────────────
func (fg *FileGraph) Save() error {
return fg.save()
}
// SearchNodes 按 label 模糊搜索节点
func (fg *FileGraph) SearchNodes(label, namespace string) []map[string]interface{} {
fg.mu.RLock()
defer fg.mu.RUnlock()
var out []map[string]interface{}
for _, n := range fg.nodes {
if namespace != "" && n.Namespace != namespace {
continue
}
if searchSubstring(n.Name, label) {
out = append(out, map[string]interface{}{
"id": n.ID, "name": n.Name, "type": n.Type, "namespace": n.Namespace,
})
}
}
return out
}
// ListNodesByType 按 type 列出节点
func (fg *FileGraph) ListNodesByType(nodeType, namespace string) []map[string]interface{} {
fg.mu.RLock()
defer fg.mu.RUnlock()
var out []map[string]interface{}
for _, n := range fg.nodes {
if namespace != "" && n.Namespace != namespace {
continue
}
if nodeType != "" && n.Type != nodeType {
continue
}
out = append(out, map[string]interface{}{
"id": n.ID, "name": n.Name, "type": n.Type, "namespace": n.Namespace,
})
}
return out
}
// ListNodes 列出所有节点
func (fg *FileGraph) ListNodes(namespace string) []map[string]interface{} {
return fg.ListNodesByType("", namespace)
}
// GetGraph 导出完整图谱供可视化limit≤0 时不限制
func (fg *FileGraph) GetGraph(namespace string, limit int) ([]map[string]interface{}, []map[string]interface{}) {
fg.mu.RLock()
defer fg.mu.RUnlock()
var nodes, edges []map[string]interface{}
for _, n := range fg.nodes {
if namespace == "" || n.Namespace == namespace {
nodes = append(nodes, map[string]interface{}{
"id": n.ID, "name": n.Name, "type": n.Type, "namespace": n.Namespace,
"pagerank": n.PageRank, "evidence_count": n.EvidenceCount,
})
if limit > 0 && len(nodes) >= limit {
break
}
}
}
for _, e := range fg.edges {
if namespace == "" || e.Namespace == namespace {
edges = append(edges, map[string]interface{}{
"id": e.ID, "source": e.Source, "target": e.Target,
"relation": e.Relation, "weight": e.Weight, "namespace": e.Namespace,
})
}
}
return nodes, edges
}