From f26126765c96e01a07eba4fa05d439fe6b45e33b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=94=AF?= Date: Sun, 27 Sep 2026 09:23:03 +0800 Subject: [PATCH] =?UTF-8?q?graph:=20=E8=B7=A8=E5=B1=82=E5=AE=9E=E4=BD=93?= =?UTF-8?q?=E5=B9=B6=E9=9B=86=20+=20PathEnds=20=E5=8F=8C=E9=94=AE=E5=AE=B9?= =?UTF-8?q?=E9=94=99=20+=20=E9=82=BB=E5=9F=9F=E5=B1=95=E5=BC=80=E4=B8=8A?= =?UTF-8?q?=E9=99=90=EF=BC=88=E4=BF=AE=20CPU=20=E6=89=93=E7=88=86=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - entity_resolve.go: 新增 ResolveEntityIDs / MergePaths / PathEnds / ExtractEntities - graph_sqlite.go: 新增 NodeIDsByName 精确同名查询(SearchNodes 是 name LIKE '%x%' LIMIT 100,'织忆' 有 834 条子串命中会把精确同名挤出去 → 并集静默失效) - graph_cache.go: 转发 NodeIDsByName(包装器不转发 → 类型断言失败 → 静默降级回 LIKE 路径) - server.go: notes 路径接并集(union=1 opt-in / 0 关闭,响应带 union + entity_ids_used); navigate 结果统一走 routes.PathEnds —— governance.Navigate 返回的是 from/to, 此前读 source/target 导致并集路径与「邻居进笔记匹配」被整条静默丢弃; 邻域进笔记匹配加 maxMatchEntities=24 上限(2026-09-27 事故:无上限邻域展开 → 数千实体 × 全库笔记扫描 → CPU 159% / HTTP 超时) - 单测 5 条,含 from/to 键容错回归 --- go/internal/api/routes/entity_resolve.go | 120 +++++++++++++ go/internal/api/routes/entity_resolve_test.go | 170 ++++++++++++++++++ go/internal/api/server.go | 94 ++++++++-- go/internal/governance/graph_sqlite.go | 24 +++ go/internal/storage/graph_cache.go | 11 ++ 5 files changed, 408 insertions(+), 11 deletions(-) create mode 100644 go/internal/api/routes/entity_resolve.go create mode 100644 go/internal/api/routes/entity_resolve_test.go diff --git a/go/internal/api/routes/entity_resolve.go b/go/internal/api/routes/entity_resolve.go new file mode 100644 index 0000000..72ba44d --- /dev/null +++ b/go/internal/api/routes/entity_resolve.go @@ -0,0 +1,120 @@ +// 织忆 MemoryWeave — 实体名字 → 同名全部节点 id 的并集解析 +// +// 背景(2026-09-27 图谱重名成因报告):图谱里同一实体可能有两层节点 —— +// +// · 旧 entity 层:id = 名字(namespace=wiki 等,06-16 起写入) +// · 新 concept 层:id = "n_" + 名字(namespace=hermes-main 等,09-26 起写入,边多得多) +// +// 而 Navigate 按 id 匹配(`WHERE e.source = ''`),PageRank 词表键也是 id +// ⇒ 按名字导航只会走旧层,新层的连接形同隐形(织忆:旧层 6 出边 vs 新层 1674 出边)。 +// +// 本文件提供并集解析:给定名字,返回**全部**同名节点 id,让调用方对并集展开, +// 从而两层数据都能进入笔记匹配。 +package routes + +import ( + "fmt" + "strings" +) + +// NodeSearcher 只要实现 SearchNodes 即可(governance.GraphStore 满足)。 +// 收窄成小接口是为了单测里能塞假实现,不必实现整个图谱接口。 +type NodeSearcher interface { + SearchNodes(label string, namespace string) []map[string]interface{} +} + +// ExactNodeFinder 精确同名解析(governance.SQLiteGraphStore 实现 NodeIDsByName)。 +// 有它就用它 —— SearchNodes 的 LIKE+LIMIT 语义会把精确同名的节点挤出结果。 +type ExactNodeFinder interface { + NodeIDsByName(name string) []string +} + +// ResolveEntityIDs 返回与 name 对应的全部节点 id(同名两层都返回)。 +// +// 语义: +// - 优先精确匹配:节点的 name 或 id 与 name 相等(忽略大小写)才算命中, +// 避免 SearchNodes 的 LIKE 子串语义把「织忆」带出「织忆召回」这类噪声节点。 +// - 若 gs 实现了 ExactNodeFinder 则走精确 SQL(首选,见 NodeIDsByName 注释)。 +// - 名字本身始终作为回退候选放在结果里(图谱里查不到、但词表命中的名字仍可按 id 直查)。 +// - 结果去重,稳定有序:先旧层(id=名字)后新层(id=n_名字),最后是名字回退。 +func ResolveEntityIDs(gs NodeSearcher, name string) []string { + name = strings.TrimSpace(name) + if name == "" { + return nil + } + ids := make([]string, 0, 2) + seen := make(map[string]bool, 4) + add := func(id string) { + if id == "" || seen[id] { + return + } + seen[id] = true + ids = append(ids, id) + } + if ex, ok := gs.(ExactNodeFinder); ok { + for _, id := range ex.NodeIDsByName(name) { + add(id) + } + } else if gs != nil { + for _, node := range gs.SearchNodes(name, "") { + nid, _ := node["id"].(string) + nname, _ := node["name"].(string) + if nid == "" { + continue + } + if strings.EqualFold(nname, name) || strings.EqualFold(nid, name) { + add(nid) + } + } + } + add(name) // 回退:按 id 直查(id == 名字) + return ids +} + +// MergePaths 把并集展开得到的路径合并进 dst,按 source|relation|target 去重。 +// 返回合并后的切片(可能原地扩容);limit > 0 时最多保留 limit 条。 +func MergePaths(dst []map[string]interface{}, src []map[string]interface{}, seen map[string]bool, limit int) []map[string]interface{} { + if seen == nil { + seen = make(map[string]bool, len(dst)+len(src)) + } + for _, p := range dst { + seen[pathKey(p)] = true + } + for _, p := range src { + if limit > 0 && len(dst) >= limit { + break + } + k := pathKey(p) + if k == "" || seen[k] { + continue + } + seen[k] = true + dst = append(dst, p) + } + return dst +} + +// PathEnds 取出路径的 (source, target, relation)。 +// 兼容两套键名:governance.Navigate 返回的是 from/to,而其它构造处用 source/target—— +// 只读一套会静默丢路径(2026-09-27 实测:并集路径、以及「图谱邻居进笔记匹配」整条链路 +// 都因为读错键名而失效,全程无报错)。 +func PathEnds(p map[string]interface{}) (string, string, string) { + s, _ := p["source"].(string) + if s == "" { + s, _ = p["from"].(string) + } + t, _ := p["target"].(string) + if t == "" { + t, _ = p["to"].(string) + } + r, _ := p["relation"].(string) + return s, t, r +} + +func pathKey(p map[string]interface{}) string { + s, t, r := PathEnds(p) + if s == "" && t == "" { + return "" + } + return fmt.Sprintf("%s|%s|%s", s, r, t) +} diff --git a/go/internal/api/routes/entity_resolve_test.go b/go/internal/api/routes/entity_resolve_test.go new file mode 100644 index 0000000..e6d6ca8 --- /dev/null +++ b/go/internal/api/routes/entity_resolve_test.go @@ -0,0 +1,170 @@ +package routes + +import ( + "strings" + "testing" +) + +// fakeSearcher 模拟 SearchNodes 的 LIKE 语义返回,用于验证精确匹配过滤。 +type fakeSearcher struct { + nodes []map[string]interface{} + calls int +} + +func (f *fakeSearcher) SearchNodes(label, namespace string) []map[string]interface{} { + f.calls++ + out := []map[string]interface{}{} + for _, n := range f.nodes { + nm, _ := n["name"].(string) + id, _ := n["id"].(string) + if strings.Contains(nm, label) || strings.Contains(id, label) { + out = append(out, n) + } + } + return out +} + +// 同名两层:旧 entity(id=名字)+ 新 concept(id=n_名字)→ 都应返回,且顺序稳定。 +func TestResolveEntityIDs_TwoLayers(t *testing.T) { + gs := &fakeSearcher{nodes: []map[string]interface{}{ + {"id": "织忆", "name": "织忆", "type": "entity"}, + {"id": "n_织忆", "name": "织忆", "type": "concept"}, + }} + ids := ResolveEntityIDs(gs, "织忆") + if len(ids) != 2 { + t.Fatalf("want 2 ids (两层), got %v", ids) + } + if ids[0] != "织忆" || ids[1] != "n_织忆" { + t.Fatalf("unexpected order: %v", ids) + } +} + +// LIKE 子串噪声必须被挡掉:查「织忆」不应带出「织忆召回」。 +func TestResolveEntityIDs_RejectsSubstringNoise(t *testing.T) { + gs := &fakeSearcher{nodes: []map[string]interface{}{ + {"id": "织忆", "name": "织忆"}, + {"id": "织忆召回机制", "name": "织忆召回机制"}, + }} + ids := ResolveEntityIDs(gs, "织忆") + for _, id := range ids { + if id == "织忆召回机制" { + t.Fatalf("子串噪声未被过滤: %v", ids) + } + } + if len(ids) != 1 { + t.Fatalf("want 1 (仅精确), got %v", ids) + } +} + +// 图谱查不到时回退为名字本身(按 id 直查仍可用)。 +func TestResolveEntityIDs_FallbackToName(t *testing.T) { + gs := &fakeSearcher{} + ids := ResolveEntityIDs(gs, "不存在的实体") + if len(ids) != 1 || ids[0] != "不存在的实体" { + t.Fatalf("want fallback to name, got %v", ids) + } +} + +// nil store / 空名字不外抛。 +func TestResolveEntityIDs_NilAndEmpty(t *testing.T) { + if got := ResolveEntityIDs(nil, "织忆"); len(got) != 1 || got[0] != "织忆" { + t.Fatalf("nil store 应回退名字: %v", got) + } + if got := ResolveEntityIDs(nil, " "); got != nil { + t.Fatalf("空名字应返回 nil, got %v", got) + } +} + +// 去重:SearchNodes 同时按 name 与 id 命中同一节点时只返回一次。 +func TestResolveEntityIDs_Dedup(t *testing.T) { + gs := &fakeSearcher{nodes: []map[string]interface{}{ + {"id": "n_织忆", "name": "织忆"}, + {"id": "n_织忆", "name": "n_织忆"}, + }} + ids := ResolveEntityIDs(gs, "织忆") + // n_织忆 命中(name 相等);名字回退也加入 ⇒ 共 2 条且不重复 + if len(ids) != 2 || ids[0] != "n_织忆" || ids[1] != "织忆" { + t.Fatalf("dedup 结果异常: %v", ids) + } +} + +// fakeExact 同时实现 NodeSearcher 与 ExactNodeFinder:验证优先走精确 SQL。 +// 模拟真实场景:SearchNodes(LIKE)被 834 条子串命中淹没、精确改名拿不到 ⇒ 必须用精确路径。 +type fakeExact struct { + exactIDs []string + likeRows []map[string]interface{} +} + +func (f *fakeExact) SearchNodes(label, namespace string) []map[string]interface{} { return f.likeRows } +func (f *fakeExact) NodeIDsByName(name string) []string { return f.exactIDs } + +func TestResolveEntityIDs_PrefersExactFinder(t *testing.T) { + gs := &fakeExact{ + exactIDs: []string{"织忆", "n_织忆"}, + likeRows: []map[string]interface{}{ // LIKE 噪声:精确同名的 n_织忆 不在其中 + {"id": "n_织忆召回机制", "name": "织忆召回机制"}, + {"id": "n_织忆架构", "name": "织忆架构"}, + }, + } + ids := ResolveEntityIDs(gs, "织忆") + want := []string{"织忆", "n_织忆"} + if len(ids) != len(want) || ids[0] != want[0] || ids[1] != want[1] { + t.Fatalf("want %v, got %v", want, ids) + } +} + +// MergePaths:按 source|relation|target 去重,并尊重上限。 +func TestMergePaths_TolerantToFromToKeys(t *testing.T) { + // governance.Navigate 返回 from/to;MergePaths 必须认,否则路径被静默丢弃 + nav := []map[string]interface{}{ + {"from": "织忆", "to": "Hermes", "relation": "USES"}, + {"from": "织忆", "to": "LanceDB", "relation": "DEPENDS_ON"}, + } + out := MergePaths([]map[string]interface{}{}, nav, nil, 0) + if len(out) != 2 { + t.Fatalf("Navigate 风格 from/to 路径被丢弃:want 2, got %d", len(out)) + } + // 去重:同一 from|relation|to 再来一次不重复 + out = MergePaths(out, nav, nil, 0) + if len(out) != 2 { + t.Fatalf("去重失败:want 2, got %d", len(out)) + } + // limit 生效 + out = MergePaths([]map[string]interface{}{}, nav, nil, 1) + if len(out) != 1 { + t.Fatalf("limit 未生效:want 1, got %d", len(out)) + } +} + +func TestPathEnds_BothKeySets(t *testing.T) { + if s, tt, r := PathEnds(map[string]interface{}{"from": "a", "to": "b", "relation": "R"}); s != "a" || tt != "b" || r != "R" { + t.Fatalf("from/to 未识别: %s %s %s", s, tt, r) + } + if s, tt, _ := PathEnds(map[string]interface{}{"source": "a", "target": "b"}); s != "a" || tt != "b" { + t.Fatalf("source/target 未识别: %s %s", s, tt) + } +} + +func TestMergePaths_DedupAndLimit(t *testing.T) { + p := func(s, r, tg string) map[string]interface{} { + return map[string]interface{}{"source": s, "relation": r, "target": tg} + } + dst := []map[string]interface{}{p("a", "is", "b")} + seen := map[string]bool{} + dst = MergePaths(dst, []map[string]interface{}{ + p("a", "is", "b"), // 重复 + p("n_a", "is", "c"), + p("n_a", "is", "d"), + }, seen, 2) + if len(dst) != 2 { + t.Fatalf("want limit 2, got %d: %v", len(dst), dst) + } + if dst[1]["target"] != "c" { + t.Fatalf("第二条应为新路径 c, got %v", dst[1]) + } + // 再次合并应识别已有 key(含 dst 内建索引) + dst = MergePaths(dst, []map[string]interface{}{p("n_a", "is", "c")}, seen, 0) + if len(dst) != 2 { + t.Fatalf("重复路径未被去重: %v", dst) + } +} diff --git a/go/internal/api/server.go b/go/internal/api/server.go index 26f3022..edd9c6b 100644 --- a/go/internal/api/server.go +++ b/go/internal/api/server.go @@ -556,18 +556,69 @@ func NewServer() http.Handler { // 不读 graph_paths(2026-09-27 全仓核对 3 处调用点)。而单实体 2 跳可爆到 1.9 万条 // ≈1.7MB/次 → 截断,仅作可观测样本。 const maxGraphPaths = 50 + // union:同名两层并集展开(id=名字 的旧 entity 层 + id="n_"+名字 的新 concept 层)。 + // 图谱重名报告(2026-09-27):Navigate 按 id 匹配 ⇒ 只传名字时新层完全用不上(织忆: + // 旧层 6 出边 vs 新层 1674 出边)。兜底路径(整句提问)默认并集;直查路径用 union=1 显式开 + // —— 灰度:直查是稳定路径,先不动,等对比数据再决定默认值。 + unionParam := r.URL.Query().Get("union") + union := unionParam == "1" // 直查路径:显式 union=1 才开 + unionDefault := unionParam != "0" // 兜底路径:默认开,union=0 可关(验收对照用) + unionApplied := false + var unionIDs []string // 并集展开实际用到的节点 id(可观测:是否真的吃到新层) + addUnionID := func(id string) { + for _, x := range unionIDs { + if x == id { + return + } + } + unionIDs = append(unionIDs, id) + } if entity != "" { norm := routes.NormalizeEntity(entity) p, err := graphStore.Navigate(norm, maxHops, "", nil) if err == nil { - paths = p - entities := []string{norm} - for _, path := range p { - if s, ok := path["source"].(string); ok { - entities = append(entities, s) + // union=1:同名跨层并集。必须按**原始名字**解析:norm 已带 n_ 前缀 + // (NormalizeEntity 行为),用它解析只得到新 concept 层(含 n_n_ 伪影), + // 恰恰漏掉旧 wiki 语义层(id=名字,7,466 条边,HTTP 上完全不可达)。 + if union { + seenP := make(map[string]bool, len(p)) + extra := []map[string]interface{}{} + for _, id := range routes.ResolveEntityIDs(graphStore, strings.TrimPrefix(norm, "n_")) { + addUnionID(id) + if id == norm { + continue + } + if p2, err2 := graphStore.Navigate(id, maxHops, "", nil); err2 == nil { + extra = routes.MergePaths(extra, p2, nil, 0) + } } - if t, ok := path["target"].(string); ok { - entities = append(entities, t) + // 跨层(旧 wiki 语义层)边放前面:主层 1.9 万条机器边会把 50 条样本窗口 + // 占满,排在后面等于被截断——并集就白做了。 + p = routes.MergePaths(extra, p, seenP, 0) + unionApplied = true + } + paths = p + // 邻居进笔记匹配:必须带上限 + 去重。2026-09-27 上线实测教训——不加限会把 + // 并集展开出的上万个邻居全丢给 SearchNotesByEntities(实体 × 全库笔记扫描), + // CPU 直接打满、HTTP 45s 超时。上限与兜底路径的 maxSearchEntities 同思路。 + const maxMatchEntities = 24 + entities := make([]string, 0, maxMatchEntities) + entsSeen := map[string]bool{norm: true} + entities = append(entities, norm) + for _, path := range p { + if len(entities) >= maxMatchEntities { + break + } + s, t, _ := routes.PathEnds(path) // 兼容 from/to 与 source/target 两套键名 + for _, e := range [2]string{s, t} { + if e == "" || entsSeen[e] { + continue + } + entsSeen[e] = true + entities = append(entities, e) + if len(entities) >= maxMatchEntities { + break + } } } if obsidian != nil { @@ -592,22 +643,37 @@ func NewServer() http.Handler { } seen[ne] = true ents = append(ents, ne) - if p, err := graphStore.Navigate(ne, maxHops, "", nil); err == nil { + // 并集:同名跨层(旧 wiki 语义层 + 新 concept 层)都展开, + // 按原始名字解析——ne 带 n_ 前缀会漏掉旧层(同直查路径注释)。 + ids := []string{ne} + if unionDefault { + ids = routes.ResolveEntityIDs(graphStore, strings.TrimPrefix(ne, "n_")) + unionApplied = true + for _, id := range ids { + addUnionID(id) + } + } + for _, id := range ids { + p, err := graphStore.Navigate(id, maxHops, "", nil) + if err != nil { + continue + } if len(p) > 20 { // 单跳邻居即可,避免把整个二跳子图塞进响应 p = p[:20] } - if len(paths) < 50 { + if len(paths) < maxGraphPaths { paths = append(paths, p...) } for _, path := range p { if len(ents) >= maxSearchEntities { break } - if s, ok := path["source"].(string); ok && !seen[s] { + s, t, _ := routes.PathEnds(path) // 同上:兼容两套键名 + if s != "" && !seen[s] { seen[s] = true ents = append(ents, s) } - if t, ok := path["target"].(string); ok && !seen[t] { + if t != "" && !seen[t] { seen[t] = true ents = append(ents, t) } @@ -635,6 +701,12 @@ func NewServer() http.Handler { if len(usedEntities) > 0 { resp["entities_used"] = usedEntities } + if unionApplied { + resp["union"] = true // 本次响应走了同名两层并集展开 + } + if len(unionIDs) > 0 { + resp["entity_ids_used"] = unionIDs + } respondJSON(w, 200, resp) }) diff --git a/go/internal/governance/graph_sqlite.go b/go/internal/governance/graph_sqlite.go index 3936118..4173615 100644 --- a/go/internal/governance/graph_sqlite.go +++ b/go/internal/governance/graph_sqlite.go @@ -933,6 +933,30 @@ func normalizeEntityID(name string) string { return "n_" + clean } +// NodeIDsByName 精确同名的全部节点 id(同名两层:旧 entity 层 id=名字 + 新 concept 层 id="n_"+名字)。 +// +// 不复用 SearchNodes:它的语义是 `name LIKE '%x%' LIMIT 100`,子串命中一多就把精确同名的 +// 节点挤出结果(2026-09-27 实测:'织忆' 有 834 条子串命中,n_织忆 因此永远拿不到 → 并集静默失效)。 +func (gs *SQLiteGraphStore) NodeIDsByName(name string) []string { + name = strings.TrimSpace(name) + if name == "" { + return nil + } + gs.mu.RLock() + defer gs.mu.RUnlock() + esc := escape(name) + sql := fmt.Sprintf( + "SELECT id FROM graph_nodes WHERE name COLLATE NOCASE = '%s' OR id COLLATE NOCASE = '%s' ORDER BY id", + esc, esc) + var ids []string + for _, r := range queryRows(gs.db, sql) { + if id, ok := r["id"].(string); ok && id != "" { + ids = append(ids, id) + } + } + return ids +} + // SearchNodes 按 label 模糊搜索节点(§2.5.4 match 格式兼容) func (gs *SQLiteGraphStore) SearchNodes(label, namespace string) []map[string]interface{} { gs.mu.RLock() diff --git a/go/internal/storage/graph_cache.go b/go/internal/storage/graph_cache.go index 4f3e066..a496aaa 100644 --- a/go/internal/storage/graph_cache.go +++ b/go/internal/storage/graph_cache.go @@ -51,6 +51,17 @@ func (r *GraphCacheRef) InvalidateAll() { r.cs.InvalidateAll() } // Stats 返回缓存统计 func (r *GraphCacheRef) Stats() map[string]interface{} { return r.cs.CacheStats() } +// NodeIDsByName 转发到内层(精确同名解析,见 governance.SQLiteGraphStore.NodeIDsByName)。 +// 必须转发:上层用类型断言探测「有没有精确解析能力」,包装器不转发就会静默退回 +// SearchNodes 的 LIKE+LIMIT 语义——子串命中一多就把精确同名的节点挤出结果 +// (2026-09-27 实测:'织忆' 834 条子串命中,n_织忆 拿不到 ⇒ 并集静默失效)。 +func (c *cachedGraphStore) NodeIDsByName(name string) []string { + if ex, ok := c.inner.(interface{ NodeIDsByName(string) []string }); ok { + return ex.NodeIDsByName(name) + } + return nil +} + // ── governance.GraphStore 接口实现(只有 Navigate 被缓存) ── func (c *cachedGraphStore) AddNode(id, name, nodeType, namespace string) error {