From 57bdf7fa03837d1c85ee06dd14a0e25ed836df87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=94=AF?= Date: Wed, 14 Oct 2026 01:41:52 +0800 Subject: [PATCH] =?UTF-8?q?feat(zhiyi):=20=E2=91=A1L2=20=E6=A8=A1=E5=BC=8F?= =?UTF-8?q?=E6=A3=80=E6=B5=8B+=E5=8F=AC=E5=9B=9E=E6=B3=A8=E5=85=A5=20(t=5F?= =?UTF-8?q?d32e5f8a)=20=E2=80=94=20=E5=A4=8D=E6=B4=BB=20L1=E2=86=92L2=20pa?= =?UTF-8?q?ttern=20=E8=92=B8=E9=A6=8F:=20=E6=96=B0=20internal/pattern=20?= =?UTF-8?q?=E5=8C=85(=E5=B7=A5=E5=85=B7=E7=BB=84=E5=90=88=E9=A2=91?= =?UTF-8?q?=E7=8E=87=E2=89=A53=20+=20=E8=B7=AF=E5=BE=84=20cosine=E2=89=A50?= =?UTF-8?q?.85=20=E8=81=9A=E7=B1=BB,=E8=AE=BE=E8=AE=A1=C2=A74.1)=E2=86=92?= =?UTF-8?q?=20upsert=20graph=5Fnodes(type=3D'pattern',ns=3Ddaemon-distill)?= =?UTF-8?q?(=E4=B8=8E=2085=20=E5=AD=98=E9=87=8F=E5=90=8C=E8=A1=A8=E4=B8=8D?= =?UTF-8?q?=E5=BC=80=E6=96=B0=E8=A1=A8=C2=B7modernc=20sqlite=20=E7=9F=AD?= =?UTF-8?q?=E5=91=BD=E5=BC=80/=E5=85=B3=C2=B7=E4=B8=8D=E7=A2=B0=20state.db?= =?UTF-8?q?);=20API=20POST=20/patterns/detect=20+=20GET=20/patterns=20+=20?= =?UTF-8?q?zhiyi-cli=20pattern=20detect/list(cron=20=E5=8F=AF=E5=8D=95?= =?UTF-8?q?=E5=8F=91);=20=E5=8F=AC=E5=9B=9E=E6=B3=A8=E5=85=A5=20env=20?= =?UTF-8?q?=E9=97=A8=E6=8E=A7=20ZHIYI=5FPATTERN=5FRECALL=20=E9=BB=98?= =?UTF-8?q?=E8=AE=A4=E5=85=B3=E2=87=92=E9=9B=B6=E8=A1=8C=E4=B8=BA=E5=8F=98?= =?UTF-8?q?=E5=8C=96;=20=E5=8D=95=E6=B5=8B=2012=20=E7=94=A8=E4=BE=8B(detec?= =?UTF-8?q?t5+store4+=E6=B3=A8=E5=85=A53+route4=20=E5=90=AB=E7=A9=BA?= =?UTF-8?q?=E5=BA=93=E5=B9=82=E7=AD=89)PASS;=20=E9=9B=B6=E6=96=B0=E4=BE=9D?= =?UTF-8?q?=E8=B5=96=E4=B8=8D=E7=A2=B0=20state.db?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go/cmd/zhiyi-cli/main.go | 83 ++++ go/internal/api/routes/patterns.go | 107 +++++ go/internal/api/routes/patterns_test.go | 132 ++++++ go/internal/api/server.go | 24 ++ go/internal/pattern/pattern.go | 449 +++++++++++++++++++++ go/internal/pattern/pattern_test.go | 111 +++++ go/internal/pattern/provider.go | 138 +++++++ go/internal/pattern/store.go | 255 ++++++++++++ go/internal/pattern/store_test.go | 116 ++++++ go/internal/storage/recall.go | 36 ++ go/internal/storage/recall_pattern_test.go | 146 +++++++ 11 files changed, 1597 insertions(+) create mode 100644 go/internal/api/routes/patterns.go create mode 100644 go/internal/api/routes/patterns_test.go create mode 100644 go/internal/pattern/pattern.go create mode 100644 go/internal/pattern/pattern_test.go create mode 100644 go/internal/pattern/provider.go create mode 100644 go/internal/pattern/store.go create mode 100644 go/internal/pattern/store_test.go create mode 100644 go/internal/storage/recall_pattern_test.go diff --git a/go/cmd/zhiyi-cli/main.go b/go/cmd/zhiyi-cli/main.go index 7bc0435..da9e1e7 100644 --- a/go/cmd/zhiyi-cli/main.go +++ b/go/cmd/zhiyi-cli/main.go @@ -10,6 +10,7 @@ import ( "net/http" "os" "sort" + "strconv" "strings" "unicode/utf8" ) @@ -92,6 +93,8 @@ func main() { runErr = runStats(apiKeyVal) case "entity": runErr = runEntity(args, apiKeyVal) + case "pattern": + runErr = runPattern(args, apiKeyVal) case "help", "--help", "-h": flag.Usage() os.Exit(0) @@ -130,6 +133,7 @@ func usage() { recall 语义搜索,返回 top-10 结果 stats 显示系统统计(记忆数、蒸馏状态等) entity 查询实体详情(出现次数、关联记忆) + pattern L2 模式检测/列出(detect=手动触发检测,list=列出已检测) 示例: zhiyi tree @@ -461,6 +465,85 @@ func runStats(apiKey string) error { return nil } +// runPattern L2 模式检测 CLI 子命令(t_d32e5f8a,cron 单发)。 +// 子命令: +// zhiyi pattern detect [--limit N] 触发检测(扫 L1 蒸馏记忆 → 工具组合/路径聚类 → 写 graph_nodes) +// zhiyi pattern list [N] 列出已检测的 pattern(默认 20) +func runPattern(args []string, apiKey string) error { + // 子命令(默认 detect) + sub := "detect" + rest := []string{} + if len(args) > 0 { + // 允许 "detect" / "list" 显式指定,或把第一个参数当 limit(list N) + switch args[0] { + case "detect", "list": + sub = args[0] + rest = args[1:] + default: + // 数字参数 = list 的 limit + if n, err := strconv.Atoi(args[0]); err == nil && n > 0 { + sub = "list" + rest = []string{args[0]} + } else { + rest = args + } + } + } + + if sub == "list" { + limit := "20" + if len(rest) > 0 { + limit = rest[0] + } + var resp struct { + Patterns []struct { + ID string `json:"id"` + Type string `json:"type"` + Name string `json:"name"` + Confidence float64 `json:"confidence"` + Tools []string `json:"tools"` + } `json:"patterns"` + Count int `json:"count"` + } + if err := apiGet("/api/v1/patterns?limit="+limit, &resp, apiKey); err != nil { + return err + } + fmt.Printf("📐 已检测模式 [%s] (%d)\n", *ns, resp.Count) + fmt.Println(strings.Repeat("─", 50)) + for i, p := range resp.Patterns { + if i >= 30 { + fmt.Printf(" … 还有 %d 个\n", len(resp.Patterns)-30) + break + } + tools := "" + if len(p.Tools) > 0 { + tools = strings.Join(p.Tools, ",") + } + fmt.Printf(" [%s] %s conf=%.2f %s\n", p.Type, p.Name, p.Confidence, tools) + } + if resp.Count == 0 { + fmt.Println(" (暂无已检测模式)") + } + return nil + } + + // detect + var resp struct { + Scanned int `json:"scanned"` + Created int `json:"created"` + Updated int `json:"updated"` + Detected int `json:"detected"` + } + if err := apiPost("/api/v1/patterns/detect", map[string]interface{}{ + "namespace": *ns, + }, &resp, apiKey); err != nil { + return err + } + fmt.Printf("🔬 L2 模式检测 [%s] 扫描 %d 条 → 命中 %d(新建 %d / 更新 %d)\n", + *ns, resp.Scanned, resp.Detected, resp.Created, resp.Updated) + return nil +} + func runEntity(args []string, apiKey string) error { if len(args) == 0 { return fmt.Errorf("用法: zhiyi entity ") diff --git a/go/internal/api/routes/patterns.go b/go/internal/api/routes/patterns.go new file mode 100644 index 0000000..278c9b7 --- /dev/null +++ b/go/internal/api/routes/patterns.go @@ -0,0 +1,107 @@ +// 织忆 MemoryWeave — L2 模式 API(t_d32e5f8a) +// +// 设计 §4.1 端点: +// POST /api/v1/patterns/detect 手动触发模式检测(扫 L1 蒸馏记忆 → 工具组合频率 + 路径聚类 +// → 写 graph_nodes type='pattern')。短命单发:modernc sqlite +// 开/关不占长连接、不碰 state.db(规避当年 daemon main_loop +// 退役根因)—— 可挂 cron 单发。 +// GET /api/v1/patterns 列出检测到的模式(按 source_tasks 长度降序)。 +package routes + +import ( + "encoding/json" + "net/http" + "os" + "strconv" + + "github.com/xiaoxue/memoryweave/internal/models" + "github.com/xiaoxue/memoryweave/internal/pattern" + "github.com/xiaoxue/memoryweave/internal/storage" +) + +// PatternAPI L2 模式端点。 +type PatternAPI struct { + Store *pattern.GraphPatternStore + Provider *pattern.PatternProvider + LDB func() storage.LanceDB // 扫 L1 记忆(distilled) + Cfg pattern.Config +} + +// NewPatternAPI 构造 L2 模式端点。 +func NewPatternAPI(store *pattern.GraphPatternStore, provider *pattern.PatternProvider, ldb func() storage.LanceDB) *PatternAPI { + cfg := pattern.DefaultConfig() + // cron 单发跑大批量时可放宽扫描上限 + if v := os.Getenv("ZHIYI_PATTERN_MAX_RECORDS"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + cfg.MaxRecords = n + } + } + return &PatternAPI{Store: store, Provider: provider, LDB: ldb, Cfg: cfg} +} + +// GET /api/v1/patterns — 列出所有检测到的模式(AC1:空库返回 count=0 不报错)。 +func (pa *PatternAPI) List(w http.ResponseWriter, r *http.Request) { + limit := pa.Cfg.TopKPatternList + if v := r.URL.Query().Get("limit"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + limit = n + } + } + pats, err := pa.Store.ListPatterns(limit) + if err != nil { + respondError(w, 500, "list patterns: "+err.Error()) + return + } + respond(w, 200, map[string]interface{}{ + "patterns": pats, + "count": len(pats), + }) +} + +// POST /api/v1/patterns/detect — 手动触发模式检测。 +// body 可选 {"limit": N, "namespace": "hermes-main"}。 +// 流程:LanceDB Scan(limit, ns) → pattern.DetectFromRecords → UpsertPatterns(graph_nodes)。 +// 空库/无命中 → created/updated=0,不报错(AC1)。 +func (pa *PatternAPI) Detect(w http.ResponseWriter, r *http.Request) { + var req struct { + Limit int `json:"limit"` + Namespace string `json:"namespace"` + } + if r.Body != nil { + _ = json.NewDecoder(r.Body).Decode(&req) // 空 body 也合法(用默认) + } + if req.Limit <= 0 { + req.Limit = pa.Cfg.MaxRecords + } + ldb := pa.LDB() + if ldb == nil { + respondError(w, 503, "lancedb unavailable") + return + } + records, err := ldb.Scan(req.Limit, req.Namespace) + if err != nil { + respondError(w, 500, "scan memories: "+err.Error()) + return + } + // 只拿蒸馏产物(L1 蒸馏记忆)做检测——原始对话/episode 不是「已验证」的解决路径 + // (与召回管线 2.5 步同源判据:raw/episodes 类不参与模式沉淀)。 + distilled := make([]models.MemoryRecord, 0, len(records)) + for _, rec := range records { + if rec.Category == "distilled" || rec.Category == "" { + distilled = append(distilled, rec) + } + } + pats := pattern.DetectFromRecords(distilled, pa.Cfg) + created, updated, uerr := pa.Store.UpsertPatterns(pats) + if uerr != nil { + respondError(w, 500, "upsert patterns: "+uerr.Error()) + return + } + respond(w, 200, map[string]interface{}{ + "scanned": len(records), + "created": created, + "updated": updated, + "detected": len(pats), + "patterns": pats, + }) +} diff --git a/go/internal/api/routes/patterns_test.go b/go/internal/api/routes/patterns_test.go new file mode 100644 index 0000000..2c7c5de --- /dev/null +++ b/go/internal/api/routes/patterns_test.go @@ -0,0 +1,132 @@ +package routes + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/xiaoxue/memoryweave/internal/models" + "github.com/xiaoxue/memoryweave/internal/pattern" + "github.com/xiaoxue/memoryweave/internal/storage" +) + +// 织忆 — L2 模式 API 路由测试(t_d32e5f8a) +// AC1: 空库 detect/list → 不报错(count=0) +// AC2: 3 条同工具组合记忆 → detect 后 list 能查到 tool_combo pattern + +// newPatternTestAPI 构造一个用临时 graph.db + 假 LanceDB 后端的 PatternAPI(无网络、无 CGO)。 +// 假后端只实现 PatternAPI 实际调用的 Scan(其余方法嵌 nil 接口,测试路径不会触达)。 +func newPatternTestAPI(t *testing.T, mems []models.MemoryRecord) *PatternAPI { + t.Helper() + store := pattern.NewGraphPatternStore(t.TempDir() + "/graph.db") + t.Cleanup(store.Close) + mlc := &patternScanLance{rows: mems} + pemb := pattern.NewEmbedderProvider(&fakePatternEmbedder{}) + return NewPatternAPI(store, pattern.NewPatternProvider(store, pemb), func() storage.LanceDB { return mlc }) +} + +// patternScanLance 覆写 Scan(PatternAPI.Detect 唯一用到的 LanceDB 方法); +// 其余方法嵌 nil 接口 —— 本测试路径不会调用,触达即 panic(暴露误用)。 +type patternScanLance struct { + storage.LanceDB + rows []models.MemoryRecord +} + +func (f *patternScanLance) Scan(limit int, namespaceFilter string) ([]models.MemoryRecord, error) { + if limit > 0 && len(f.rows) > limit { + return f.rows[:limit], nil + } + return f.rows, nil +} + +// fakePatternEmbedder 确定性假向量(pattern.Provider 匹配测试用;注入测试走 storage 侧不走这里)。 +type fakePatternEmbedder struct{} + +func (fakePatternEmbedder) EncodeSingle(text string) ([]float32, error) { + v := make([]float32, 8) + for i := 0; i < 8; i++ { + v[i] = 0.1 + } + return v, nil +} + +// TestPatterns_ListEmpty —— AC1:空库 list → 200 + count=0(不报错)。 +func TestPatterns_ListEmpty(t *testing.T) { + api := newPatternTestAPI(t, nil) + rec := httptest.NewRecorder() + api.List(rec, httptest.NewRequest("GET", "/api/v1/patterns", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("空库 list 期望 200,实际 %d body=%s", rec.Code, rec.Body.String()) + } + var resp map[string]interface{} + _ = json.Unmarshal(rec.Body.Bytes(), &resp) + if int(resp["count"].(float64)) != 0 { + t.Fatalf("空库 count 期望 0,实际 %v", resp["count"]) + } +} + +// TestPatterns_DetectEmpty —— AC1:空库 detect → 200 + scanned=0/created=0(不报错)。 +func TestPatterns_DetectEmpty(t *testing.T) { + api := newPatternTestAPI(t, nil) + rec := httptest.NewRecorder() + api.Detect(rec, httptest.NewRequest("POST", "/api/v1/patterns/detect", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("空库 detect 期望 200,实际 %d body=%s", rec.Code, rec.Body.String()) + } + var resp map[string]interface{} + _ = json.Unmarshal(rec.Body.Bytes(), &resp) + if int(resp["scanned"].(float64)) != 0 || int(resp["created"].(float64)) != 0 { + t.Fatalf("空库 detect 期望 0/0,实际 %v/%v", resp["scanned"], resp["created"]) + } +} + +// TestPatterns_DetectToolCombo —— AC2:3 条同工具组合蒸馏记忆 → detect 产出 1 个 tool_combo, +// 且 list 能查到(sqlite graph_nodes 判据)。 +func TestPatterns_DetectToolCombo(t *testing.T) { + mems := make([]models.MemoryRecord, 0, 3) + for i, id := range []string{"pc_a", "pc_b", "pc_c"} { + mems = append(mems, models.MemoryRecord{ + ID: id, + Content: "用 jq git curl 处理第" + string(rune('一'+i)) + "项", + Category: "distilled", + }) + } + api := newPatternTestAPI(t, mems) + + rec := httptest.NewRecorder() + api.Detect(rec, httptest.NewRequest("POST", "/api/v1/patterns/detect", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("detect 期望 200,实际 %d body=%s", rec.Code, rec.Body.String()) + } + var dresp map[string]interface{} + _ = json.Unmarshal(rec.Body.Bytes(), &dresp) + if int(dresp["detected"].(float64)) < 1 { + t.Fatalf("3 条同组合应检出 ≥1 个 pattern,实际 detected=%v", dresp["detected"]) + } + // 用 list 再核一遍 sqlite 判据:确有 tool_combo 行 + lrec := httptest.NewRecorder() + api.List(lrec, httptest.NewRequest("GET", "/api/v1/patterns", nil)) + var lresp struct { + Count int `json:"count"` + Patterns []struct { + Type string `json:"type"` + Name string `json:"name"` + } `json:"patterns"` + } + if err := json.Unmarshal(lrec.Body.Bytes(), &lresp); err != nil { + t.Fatalf("list 解析: %v (body=%s)", err, lrec.Body.String()) + } + if lresp.Count < 1 { + t.Fatalf("AC2 判据:detect 后 list count 应 ≥1,实际 %d (detected=%v)", lresp.Count, dresp["detected"]) + } + var foundToolCombo bool + for _, p := range lresp.Patterns { + if p.Type == "tool_combo" { + foundToolCombo = true + } + } + if !foundToolCombo { + t.Fatalf("AC2 判据:list 应含 tool_combo pattern,实际 %d 条均非 tool_combo", lresp.Count) + } +} diff --git a/go/internal/api/server.go b/go/internal/api/server.go index 5bc8649..e73418d 100644 --- a/go/internal/api/server.go +++ b/go/internal/api/server.go @@ -23,6 +23,7 @@ import ( "github.com/xiaoxue/memoryweave/internal/governance" "github.com/xiaoxue/memoryweave/internal/metrics" "github.com/xiaoxue/memoryweave/internal/models" + "github.com/xiaoxue/memoryweave/internal/pattern" "github.com/xiaoxue/memoryweave/internal/redact" "github.com/xiaoxue/memoryweave/internal/selfoptimize" "github.com/xiaoxue/memoryweave/internal/storage" @@ -147,6 +148,26 @@ func NewServer() http.Handler { consolPipe := routes.NewConsolidationPipeline(ldb, graphStore, conflictDetector) consolPipe.SetDataDir("/var/lib/memoryweave/lancedb", "/var/lib/memoryweave/graph.db") + // ─── t_d32e5f8a L2 模式检测(MemOS 血统)───────────────────────── + // 复活 L1→L2 pattern 蒸馏(原 Python daemon main_loop 退役后停摆): + // pattern 节点存 graph_nodes(type='pattern'),与现有 85 存量同表,不另开新表。 + // modernc sqlite 短命单发开/关(UpsertPatterns/ListPatterns 开一次用一次即关), + // 不持长连接、不碰 state.db —— 规避当年退役根因。 + // 召回注入 env 门控 ZHIYI_PATTERN_RECALL,默认 0 ⇒ 不注册 Provider,行为零变化。 + graphPath := os.Getenv("GRAPH_PATH") + if graphPath == "" { + graphPath = "/var/lib/memoryweave/graph.db" + } + patternStore := pattern.NewGraphPatternStore(graphPath) + patternProvider := pattern.NewPatternProvider(patternStore, pattern.NewEmbedderProvider(emb)) + patternAPI := routes.NewPatternAPI(patternStore, patternProvider, func() storage.LanceDB { return ldb }) + if os.Getenv("ZHIYI_PATTERN_RECALL") == "1" { + storage.PatternRecallProvider = patternProvider.Match + log.Printf("[zhiyid] L2 模式召回注入已启用(ZHIYI_PATTERN_RECALL=1)") + } else { + log.Printf("[zhiyid] L2 模式召回注入默认关(ZHIYI_PATTERN_RECALL 未设/≠1 ⇒ 零行为变化)") + } + // Obsidian obsidian := routes.NewObsidianSyncer("", ldb) @@ -1068,6 +1089,9 @@ func NewServer() http.Handler { // 缺口 mux.HandleFunc("/api/v1/gaps", gapAPI.List) mux.HandleFunc("/api/v1/gaps/detect", gapAPI.Detect) + // t_d32e5f8a L2 模式检测(MemOS 血统) + mux.HandleFunc("/api/v1/patterns", patternAPI.List) + mux.HandleFunc("/api/v1/patterns/detect", patternAPI.Detect) mux.HandleFunc("/api/v1/gaps/close/", func(w http.ResponseWriter, r *http.Request) { gapAPI.Close(w, r) topic := r.URL.Path[len("/api/v1/gaps/close/"):] diff --git a/go/internal/pattern/pattern.go b/go/internal/pattern/pattern.go new file mode 100644 index 0000000..0bb0ce9 --- /dev/null +++ b/go/internal/pattern/pattern.go @@ -0,0 +1,449 @@ +// 织忆 MemoryWeave — L2 模式检测引擎(t_d32e5f8a) +// +// 背景:MemOS 血统的 L2「跨任务重复模式」检测。原实现活在 Python daemon 的 +// main_loop(30s tick),2026-09-24 为 state-db 写安全整体退役后 L2 pattern 蒸馏 +// 停摆(graph_nodes type='pattern' 停在 85 个 / 2026-09-18)。本包把 L1→L2 蒸馏 +// 复活成**短命单发**(modernc sqlite 直连 graph.db,开/关不占长连接,不碰 state.db), +// 并按设计 §4.1 叠加两条检测规则: +// - 工具组合频率:同一 tool 组合出现 ≥3 次 → tool_combo pattern +// (confidence = min(freq*0.1, 1.0)) +// - 相似解决路径聚类:cosine ≥0.85 成簇且簇内 ≥3 条 → workflow pattern +// +// 存储:复用现有 graph_nodes(type='pattern'),**不另开新表**(避免与 85 存量 + 现有 +// 图谱查询割裂)。namespace='daemon-distill' 沿用旧 daemon 约定。 +package pattern + +import ( + "fmt" + "math" + "sort" + "strings" + + "github.com/xiaoxue/memoryweave/internal/models" +) + +// Pattern 一个被检测出的 L2 模式。 +type Pattern struct { + ID string + Type string // "tool_combo" | "workflow" + Name string // 稳定命名键(同组合去重) + Data string // tool_combo=排序后工具串;workflow=共同步骤/代表路径摘要 + Confidence float64 + SourceTasks []string // 支撑该 pattern 的记忆 ID(设计 source_tasks) + Tools []string // tool_combo 专用 +} + +// Config 检测参数(设计 §4.1)。 +type Config struct { + MinComboFreq int // 工具组合出现 ≥ N 次才产 pattern(默认 3) + ClusterThresh float64 // 相似路径聚类 cosine 阈值(默认 0.85) + MinClusterSize int // 簇内 ≥ N 条才产 workflow pattern(默认 3) + MaxRecords int // 单次扫描参与检测的记忆上限(防 O(n²) 爆炸,默认 500) + TopKPatternList int // 召回/列出取前 N(默认 10) +} + +// DefaultConfig 设计参数。 +func DefaultConfig() Config { + return Config{ + MinComboFreq: 3, + ClusterThresh: 0.85, + MinClusterSize: 3, + MaxRecords: 500, + TopKPatternList: 10, + } +} + +// knownTools 常见工具/CLI/命令名(小写)。工具组合频率分析从这里识别 + 兜底正则。 +var knownTools = map[string]bool{ + "jq": true, "yq": true, "git": true, "grep": true, "sed": true, "awk": true, + "curl": true, "wget": true, "tar": true, "zip": true, "unzip": true, + "docker": true, "kubectl": true, "helm": true, "npm": true, "pip": true, + "go": true, "python": true, "python3": true, "node": true, "pnpm": true, + "make": true, "cmake": true, "gcc": true, "gcc-12": true, "clang": true, + "sqlite3": true, "psql": true, "mysql": true, "redis-cli": true, + "ssh": true, "scp": true, "rsync": true, "systemctl": true, "journalctl": true, + "systemd-run": true, "crontab": true, "find": true, "rg": true, "ripgrep": true, + "fd": true, "fzf": true, "gh": true, "gcloud": true, "aws": true, "jq2": true, + "hermes": true, "zhiyi": true, "obsidian": true, "esbuild": true, "ts": true, + "tsc": true, "cargo": true, "rustc": true, "swift": true, "brew": true, +} + +// DetectFromRecords 纯检测核心:从 L1 记忆列表发现重复模式。 +// 不读 LanceDB / 不写库,只做「记录 → pattern 候选」,便于确定性单测。 +// 返回合并去重后的 pattern 列表(同 Name 合并 source_tasks、取更高 confidence)。 +func DetectFromRecords(records []models.MemoryRecord, cfg Config) []Pattern { + if cfg.MinComboFreq <= 0 { + cfg.MinComboFreq = 3 + } + if cfg.MinClusterSize <= 0 { + cfg.MinClusterSize = 3 + } + if cfg.ClusterThresh <= 0 { + cfg.ClusterThresh = 0.85 + } + lim := cfg.MaxRecords + if lim <= 0 { + lim = 500 + } + if len(records) > lim { + records = records[:lim] + } + + combined := make(map[string]*Pattern) // Name → merged pattern + order := []string{} + + merge := func(p Pattern) { + if ex, ok := combined[p.Name]; ok { + ex.SourceTasks = append(ex.SourceTasks, p.SourceTasks...) + if p.Confidence > ex.Confidence { + ex.Confidence = p.Confidence + } + return + } + combined[p.Name] = &p + order = append(order, p.Name) + } + + // 1. 工具组合频率分析 + comboCount := map[string]*struct { + tools []string + tasks []string + }{} + for _, rec := range records { + tools := ExtractTools(rec.Content) + if len(tools) == 0 { + continue + } + tools = dedupeSorted(tools) + key := "tool_combo:" + strings.Join(tools, "+") + c := comboCount[key] + if c == nil { + c = &struct { + tools []string + tasks []string + }{tools: tools} + comboCount[key] = c + } + c.tasks = append(c.tasks, rec.ID) + } + for name, c := range comboCount { + if len(c.tasks) < cfg.MinComboFreq { + continue + } + merge(Pattern{ + Type: "tool_combo", + Name: name, + Data: strings.Join(c.tools, "+"), + Tools: c.tools, + Confidence: confidenceFromFreq(len(c.tasks)), + SourceTasks: c.tasks, + }) + } + + // 2. 相似解决路径聚类(cosine ≥ 阈值 成簇,簇内 ≥ N 条) + paths := clusterPaths(records, cfg.ClusterThresh, cfg.MinClusterSize) + for _, cluster := range paths { + // 代表路径 = 簇内最长 content(共同步骤近似) + rep := "" + var ids []string + for _, r := range cluster { + ids = append(ids, r.ID) + if len(r.Content) > len(rep) { + rep = r.Content + } + } + conf := clusterConfidence(cluster) // 平均两两 cosine,封顶 1.0 + if conf > 1.0 { + conf = 1.0 + } + // 簇名带代表内容哈希前 8 位 —— 同 size 不同内容簇不碰撞(否则 cluster-3 撞簇) + name := fmt.Sprintf("workflow:cluster-%d-%s", len(cluster), contentFingerprint(rep)) + merge(Pattern{ + Type: "workflow", + Name: name, + Data: truncateForPattern(rep, 120), + Confidence: conf, + SourceTasks: ids, + }) + } + + out := make([]Pattern, 0, len(order)) + for _, n := range order { + p := *combined[n] + p.SourceTasks = dedupeStrings(p.SourceTasks) + if p.Type == "tool_combo" { + p.ID = "pat_" + p.Name + } else { + p.ID = "pat_" + p.Name + } + out = append(out, p) + } + sort.SliceStable(out, func(i, j int) bool { + if out[i].Type != out[j].Type { + return out[i].Type < out[j].Type + } + return out[i].Name < out[j].Name + }) + return out +} + +// confidenceFromFreq 设计公式:min(freq*0.1, 1.0)。 +func confidenceFromFreq(freq int) float64 { + c := float64(freq) * 0.1 + if c > 1.0 { + c = 1.0 + } + return c +} + +// clusterPaths 贪心成簇:为每条未分配记忆开一个簇,把与其 cosine≥阈值 的其余未分配 +// 记忆并入;最终只保留 size≥minSize 的簇。向量缺失时用记录自带 Vector,全缺则退到 +// 文本 bigram Jaccard(确定性,不依赖网络)。 +func clusterPaths(records []models.MemoryRecord, thresh float64, minSize int) [][]models.MemoryRecord { + n := len(records) + if n < minSize { + return nil + } + type sig struct { + vec []float32 + big map[string]bool + text string + } + sigs := make([]sig, n) + for i, r := range records { + sigs[i] = sig{vec: r.Vector, text: r.Content} + } + sim := func(a, b int) float64 { + sa, sb := sigs[a], sigs[b] + if len(sa.vec) > 0 && len(sb.vec) > 0 { + return cosineF32(sa.vec, sb.vec) + } + if len(sa.vec) == 0 && len(sb.vec) == 0 { + return bigramJaccard(sa.text, sb.text) + } + return 0 + } + assigned := make([]bool, n) + var clusters [][]models.MemoryRecord + for i := 0; i < n; i++ { + if assigned[i] { + continue + } + cluster := []models.MemoryRecord{records[i]} + assigned[i] = true + for j := i + 1; j < n; j++ { + if assigned[j] { + continue + } + if sim(i, j) >= thresh { + cluster = append(cluster, records[j]) + assigned[j] = true + } + } + if len(cluster) >= minSize { + clusters = append(clusters, cluster) + } + } + return clusters +} + +// clusterConfidence 簇内平均两两相似度(cosine 或 Jaccard)。 +func clusterConfidence(cluster []models.MemoryRecord) float64 { + n := len(cluster) + if n < 2 { + return 1.0 + } + sum := 0.0 + pairs := 0 + for i := 0; i < n; i++ { + for j := i + 1; j < n; j++ { + var s float64 + a, b := cluster[i], cluster[j] + if len(a.Vector) > 0 && len(b.Vector) > 0 { + s = cosineF32(a.Vector, b.Vector) + } else { + s = bigramJaccard(a.Content, b.Content) + } + sum += s + pairs++ + } + } + if pairs == 0 { + return 1.0 + } + return sum / float64(pairs) +} + +// ExtractTools 从记忆内容提取工具/命令名。识别 knownTools + 兜底 ascii token。 +// 确定性:结果排序去重。 +func ExtractTools(content string) []string { + found := map[string]bool{} + lower := strings.ToLower(content) + for t := range knownTools { + // 词边界匹配(避免 "go" 命中 "google") + if tokenPresent(lower, t) { + found[t] = true + } + } + // 兜底:捕获已知列表外的 ascii 命令式 token(2~14 字符,含 - 的命令风格) + for _, tok := range regexTokenExtract(content) { + lt := strings.ToLower(tok) + if len(lt) < 2 { + continue + } + // 排除明显非工具的常见词(纯小写短词且不在 knownTools 的放行——保持宽松, + // 让工具组合信号进来;过噪由 MinComboFreq 过滤) + found[lt] = true + } + out := make([]string, 0, len(found)) + for t := range found { + out = append(out, t) + } + sort.Strings(out) + return out +} + +// tokenPresent 检查 lower 文本中 t 是否作为独立 token 出现(词边界,避免 "go" 命中 "google")。 +func tokenPresent(lower, t string) bool { + if t == "" { + return false + } + // 切词精确匹配 + for _, w := range strings.Fields(lower) { + // 剥掉包裹的引号/逗号/括号 + w = strings.Trim(w, `"',()[]|;:`) + if w == t { + return true + } + } + // 空格/首尾边界 + if strings.Contains(lower, " "+t+" ") || strings.HasPrefix(lower, t+" ") || strings.HasSuffix(lower, " "+t) { + return true + } + return false +} + +// regexTokenExtract 轻量 token 抽取(无正则依赖,按字符切): +// 返回形如命令/token 的 ascii 串(长度 ≥2,含字母,可能带数字/_.-)。 +func regexTokenExtract(content string) []string { + var out []string + var cur []rune + flush := func() { + if len(cur) >= 2 { + out = append(out, string(cur)) + } + cur = nil + } + for _, r := range content { + isTok := (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || + (r >= '0' && r <= '9') || r == '_' || r == '-' || r == '.' + if isTok { + cur = append(cur, r) + } else { + flush() + } + } + flush() + return out +} + +func dedupeSorted(in []string) []string { + seen := map[string]bool{} + out := make([]string, 0, len(in)) + for _, s := range in { + if !seen[s] { + seen[s] = true + out = append(out, s) + } + } + sort.Strings(out) + return out +} + +func dedupeStrings(in []string) []string { + seen := map[string]bool{} + out := make([]string, 0, len(in)) + for _, s := range in { + if !seen[s] { + seen[s] = true + out = append(out, s) + } + } + return out +} + +func cosineF32(a, b []float32) float64 { + if len(a) == 0 || len(b) == 0 || len(a) != len(b) { + return 0 + } + var dot, na, nb float64 + for i := range a { + x, y := float64(a[i]), float64(b[i]) + dot += x * y + na += x * x + nb += y * y + } + if na == 0 || nb == 0 { + return 0 + } + c := dot / (float64(math.Sqrt(na)) * float64(math.Sqrt(nb))) + if c > 1.0 { + c = 1.0 + } + if c < -1.0 { + c = -1.0 + } + return c +} + +// bigramJaccard 字符 bigram Jaccard 相似度(无向量时的确定性回退)。 +func bigramJaccard(a, b string) float64 { + bg := func(s string) map[string]bool { + m := make(map[string]bool) + r := []rune(s) + for i := 0; i+1 < len(r); i++ { + m[string(r[i:i+2])] = true + } + return m + } + ma, mb := bg(a), bg(b) + if len(ma) == 0 && len(mb) == 0 { + return 1.0 + } + if len(ma) == 0 || len(mb) == 0 { + return 0.0 + } + inter := 0 + for k := range ma { + if mb[k] { + inter++ + } + } + uni := len(ma) + len(mb) - inter + if uni == 0 { + return 0 + } + return float64(inter) / float64(uni) +} + +// contentFingerprint 取 content 稳定短指纹(FNV 前 8 hex 位)——同 size 不同内容簇去重。 +func contentFingerprint(s string) string { + if s == "" { + return "empty" + } + // FNV-1a 32 + h := uint32(2166136261) + for i := 0; i < len(s); i++ { + h ^= uint32(s[i]) + h *= 16777619 + } + return fmt.Sprintf("%08x", h) +} + +func truncateForPattern(s string, n int) string { + r := []rune(s) + if len(r) <= n { + return s + } + return string(r[:n]) + "…" +} diff --git a/go/internal/pattern/pattern_test.go b/go/internal/pattern/pattern_test.go new file mode 100644 index 0000000..a174294 --- /dev/null +++ b/go/internal/pattern/pattern_test.go @@ -0,0 +1,111 @@ +package pattern + +import ( + "strings" + "testing" + + "github.com/xiaoxue/memoryweave/internal/models" +) + +// 构造 N 条同一工具组合的记忆(内容里带 jq/git/curl 三个已知工具)。 +func comboRecords(n int, combo string) []models.MemoryRecord { + // combo 形如 "jq git curl"——内容里出现这三个工具名(空格分隔,保证词边界命中)。 + // 上下文用纯中文(非 ascii token),避免回退正则把上下文词误当工具混进组合键。 + recs := make([]models.MemoryRecord, 0, n) + for i := 0; i < n; i++ { + recs = append(recs, models.MemoryRecord{ + ID: "m_" + string(rune('a'+i)), + Content: "用 " + combo + " 处理第" + string(rune('一'+i)) + "项", + Category: "distilled", + Vector: nil, // 走 bigram Jaccard 回退(确定性,不依赖网络) + }) + } + return recs +} + +// TestDetect_ToolCombo —— 3 条同工具组合记忆 → 检出 1 个 tool_combo pattern(AC2)。 +func TestDetect_ToolCombo(t *testing.T) { + recs := comboRecords(3, "jq git curl") + pats := DetectFromRecords(recs, DefaultConfig()) + var toolPatterns []Pattern + for _, p := range pats { + if p.Type == "tool_combo" { + toolPatterns = append(toolPatterns, p) + } + } + if len(toolPatterns) != 1 { + t.Fatalf("期望 1 个 tool_combo pattern,实际 %d: %#v", len(toolPatterns), toolPatterns) + } + p := toolPatterns[0] + if p.Name != "tool_combo:curl+git+jq" { + t.Fatalf("命名键不符: got %q (工具应排序去重为 curl,git,jq)", p.Name) + } + if len(p.Tools) != 3 { + t.Fatalf("Tools 应含 3 个,实际 %v", p.Tools) + } + // confidence = min(freq*0.1,1.0) = min(3*0.1,1.0) ≈ 0.3(浮点容差 1e-9) + if p.Confidence < 0.3-1e-9 || p.Confidence > 0.3+1e-9 { + t.Fatalf("confidence 期望 ≈0.3,实际 %f", p.Confidence) + } + if len(p.SourceTasks) != 3 { + t.Fatalf("SourceTasks 应含 3 条记忆 ID,实际 %v", p.SourceTasks) + } +} + +// TestDetect_BelowThreshold —— 2 条同组合(<3)→ 不产 tool_combo(AC2 反例)。 +func TestDetect_BelowThreshold(t *testing.T) { + recs := comboRecords(2, "jq git curl") + pats := DetectFromRecords(recs, DefaultConfig()) + for _, p := range pats { + if p.Type == "tool_combo" { + t.Fatalf("2 条同组合不应产 tool_combo,实际产出: %#v", p) + } + } +} + +// TestDetect_Workflow —— 3 条近同文(向量近似 1.0 或 bigram 高度重合)→ 1 个 workflow 簇。 +func TestDetect_Workflow(t *testing.T) { + // 向量相同(cosine=1.0 ≥ 0.85)且 3 条 → 应聚成 1 簇。 + vec := []float32{1.0, 0.0, 0.0, 0.0} + recs := []models.MemoryRecord{ + {ID: "w1", Content: "重启服务后拉日志", Category: "distilled", Vector: vec}, + {ID: "w2", Content: "重启服务后拉日志", Category: "distilled", Vector: vec}, + {ID: "w3", Content: "重启服务后拉日志", Category: "distilled", Vector: vec}, + } + pats := DetectFromRecords(recs, DefaultConfig()) + var wf []Pattern + for _, p := range pats { + if p.Type == "workflow" { + wf = append(wf, p) + } + } + if len(wf) != 1 { + t.Fatalf("3 条近同文应聚 1 个 workflow pattern,实际 %d: %#v", len(wf), pats) + } + if !strings.HasPrefix(wf[0].Name, "workflow:cluster-3-") { + t.Fatalf("workflow 命名应含 cluster-3-,实际 %q", wf[0].Name) + } +} + +// TestDetect_Empty —— 空输入 → 空输出不报错(AC1 空库前提)。 +func TestDetect_Empty(t *testing.T) { + if pats := DetectFromRecords(nil, DefaultConfig()); len(pats) != 0 { + t.Fatalf("空输入应 0 pattern,实际 %d", len(pats)) + } +} + +// TestExtractTools —— 词边界:「google」不应命中工具 "go"。 +func TestExtractTools_WordBoundary(t *testing.T) { + tools := ExtractTools("I like google and grep the log with jq") + // 应含 grep、jq;不应含 go("google" 被词边界排除) + found := map[string]bool{} + for _, s := range tools { + found[s] = true + } + if !found["grep"] || !found["jq"] { + t.Fatalf("应识别 grep/jq,实际 %v", tools) + } + if found["go"] { + t.Fatalf("'google' 误命中工具 'go'(词边界失效): %v", tools) + } +} diff --git a/go/internal/pattern/provider.go b/go/internal/pattern/provider.go new file mode 100644 index 0000000..dc391e1 --- /dev/null +++ b/go/internal/pattern/provider.go @@ -0,0 +1,138 @@ +// 织忆 MemoryWeave — L2 pattern 召回 Provider(t_d32e5f8a) +// +// 把 graph_nodes(type='pattern') 变成召回可附带的 pattern 命中(设计 §4.1「命中即注入」)。 +// 匹配语义:query 编码成向量 → 对每个 pattern 取「代表向量」(pattern.data 文本的 embedding) +// 算 cosine,取 ≥ 阈值 的前 topK。embedder 不可用 / 无 pattern 时返回空(不报错、零注入)。 +// +// 与 storage.PatternRecallProvider 解耦:本 Provider 是纯函数式匹配器, +// server.go 把它包进 storage.PatternRecallProvider 闭包(再叠 env 门控)。 +package pattern + +import ( + "log" + "strings" + + "github.com/xiaoxue/memoryweave/internal/models" +) + +// EmbedderProvider pattern 文本 → 向量。复用 storage.Embedder.EncodeSingle 语义。 +type EmbedderProvider interface { + Embed(text string) ([]float32, error) +} + +// StorageEmbedderAdapter 把 storage.Embedder(EncodeSingle)适配成 EmbedderProvider, +// 供 server.go 装配 pattern.Provider 用(pattern 包不 import storage,避免循环依赖)。 +type StorageEmbedderAdapter interface { + EncodeSingle(text string) ([]float32, error) +} + +// adapterProvider 实现 EmbedderProvider。 +type adapterProvider struct { + inner StorageEmbedderAdapter +} + +func (a adapterProvider) Embed(text string) ([]float32, error) { + return a.inner.EncodeSingle(text) +} + +// NewEmbedderProvider 从 storage.Embedder 适配(签名匹配 EncodeSingle)。 +func NewEmbedderProvider(inner StorageEmbedderAdapter) EmbedderProvider { + return adapterProvider{inner: inner} +} + +// recallProvider storage 侧注入用的适配签名实现(包一层)。 +// 与 recall.go 的 PatternRecallProvider 保持解耦:pattern 包不 import storage(避免循环)。 + +// PatternProvider 匹配器:query+namespace → top-k pattern(转成 RecallResult,Category="pattern")。 +type PatternProvider struct { + store *GraphPatternStore + embedder EmbedderProvider + threshold float64 // 相似度下限(默认 0.6;低于则不注入,避免噪声 pattern) +} + +// NewPatternProvider 构造匹配器。store 读 graph_nodes(type='pattern')。 +func NewPatternProvider(store *GraphPatternStore, emb EmbedderProvider) *PatternProvider { + return &PatternProvider{store: store, embedder: emb, threshold: 0.6} +} + +// SetThreshold 调整相似度下限。 +func (pp *PatternProvider) SetThreshold(t float64) { pp.threshold = t } + +// Match query+namespace → 命中的 pattern(top-k,cosine≥threshold)。 +// 返回 models.RecallResult(Category="pattern",Content=pattern 摘要), +// 供 storage.PatternRecallProvider 直接附带到召回结果。 +func (pp *PatternProvider) Match(query, namespace string, topK int) []models.RecallResult { + if pp == nil || pp.store == nil || topK <= 0 { + return nil + } + pats, err := pp.store.ListPatterns(topK * 3) // 多取些再按相似度截断 + if err != nil || len(pats) == 0 { + return nil + } + qv, err := pp.embedder.Embed(query) + if err != nil || len(qv) == 0 { + log.Printf("[pattern-recall] query 编码失败,跳过注入: %v", err) + return nil + } + type scored struct { + p Pattern + sim float64 + } + var scoredList []scored + for _, p := range pats { + // namespace 过滤:daemon-distill 是跨 ns 的,namespace 非空时仍保留(pattern 全局复用) + rep := p.Data + if rep == "" { + rep = p.Name + } + // 加 query 里出现的工具名做召回加权(工具组合 pattern 对工具词敏感) + text := rep + " " + strings.Join(p.Tools, " ") + pv, err := pp.embedder.Embed(text) + if err != nil || len(pv) == 0 { + continue + } + sim := cosineF32(qv, pv) + // 工具词命中加成:query 含 pattern 任一工具 ⇒ sim 抬 0.15 + qLower := strings.ToLower(query) + for _, tool := range p.Tools { + if tool != "" && strings.Contains(qLower, strings.ToLower(tool)) { + sim += 0.15 + break + } + } + if sim > 1.0 { + sim = 1.0 + } + if sim >= pp.threshold { + scoredList = append(scoredList, scored{p: p, sim: sim}) + } + } + // 按 sim 降序取 topK + if len(scoredList) > topK { + // 简单选择:O(n^2) 但 n 很小 + for i := 0; i < len(scoredList); i++ { + for j := i + 1; j < len(scoredList); j++ { + if scoredList[j].sim > scoredList[i].sim { + scoredList[i], scoredList[j] = scoredList[j], scoredList[i] + } + } + } + scoredList = scoredList[:topK] + } + out := make([]models.RecallResult, 0, len(scoredList)) + for _, sc := range scoredList { + content := sc.p.Name + if sc.p.Data != "" { + content = sc.p.Name + " — " + sc.p.Data + } + out = append(out, models.RecallResult{ + ID: sc.p.ID, + Content: content, + Category: "pattern", + Score: sc.sim, + Namespace: namespace, + Source: "l2_pattern", + }) + } + return out +} diff --git a/go/internal/pattern/store.go b/go/internal/pattern/store.go new file mode 100644 index 0000000..8c110e9 --- /dev/null +++ b/go/internal/pattern/store.go @@ -0,0 +1,255 @@ +// 织忆 MemoryWeave — L2 pattern 存储(t_d32e5f8a) +// +// 存储沿用现有 graph_nodes(type='pattern'),**不另开新表**(避免与 85 存量 + +// 现有图谱查询割裂)。modernc sqlite 短命单发开/关,不持长连接、不碰 state.db。 +// +// 写入幂等语义(复活 L1→L2 occurrence_count 驱动): +// - 同名 pattern 节点存在 → occurrence_count += 1、last_seen 刷新、props 合并(新键覆盖、旧键保留) +// - 不存在 → INSERT(occurrence_count=1、first_seen=last_seen=now) +// 命名键:id = name = "tool_combo:" / "workflow:cluster-N-", +// namespace = "daemon-distill"(沿用旧 Python daemon 约定)。 +package pattern + +import ( + "database/sql" + "encoding/json" + "log" + "os" + "path/filepath" + "sort" + "sync" + "time" + + _ "modernc.org/sqlite" +) + +// GraphPatternStore pattern 在 graph_nodes(type='pattern') 上的读写。 +// modernc 纯 Go,无 CGO,测试可用 t.TempDir() 的 db 路径。 +type GraphPatternStore struct { + dbPath string + + mu sync.Mutex + once sync.Once + db *sql.DB + closed bool +} + +// NewGraphPatternStore 指向 graph.db(生产 /var/lib/memoryweave/graph.db)。 +// 测试传 t.TempDir() 路径。 +func NewGraphPatternStore(dbPath string) *GraphPatternStore { + if dbPath == "" { + dbPath = "/var/lib/memoryweave/graph.db" + } + return &GraphPatternStore{dbPath: dbPath} +} + +// SetGraphPathForTest 测试专用:改库路径 + 重置单例(下次调用按新路径重开)。 +func (s *GraphPatternStore) SetForTest(p string) { + s.mu.Lock() + defer s.mu.Unlock() + s.dbPath = p + s.once = sync.Once{} + if s.db != nil { + s.db.Close() + s.db = nil + } + s.closed = false +} + +// open 懒开库(WAL 兼容:先清 stale WAL 只在能 stat 到时才动,避免误删新建库)。 +func (s *GraphPatternStore) open() (*sql.DB, error) { + s.once.Do(func() { + // 目录不存在时先建(cron 单发到新环境/临时库路径不炸) + if dir := filepath.Dir(s.dbPath); dir != "" && dir != "." { + if err := os.MkdirAll(dir, 0o755); err != nil { + log.Printf("[pattern] mkdir %s: %v", dir, err) + return + } + } + db, err := sql.Open("sqlite", "file:"+s.dbPath+ + "?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)&_pragma=synchronous(NORMAL)") + if err != nil { + log.Printf("[pattern] open graph.db %s: %v", s.dbPath, err) + return + } + // 建表兜底(graph_nodes 生产已有;测试临时库需要) + _, _ = db.Exec(`CREATE TABLE IF NOT EXISTS graph_nodes ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + type TEXT NOT NULL, + namespace TEXT NOT NULL DEFAULT '', + properties TEXT DEFAULT '{}', + created_at TEXT NOT NULL, + last_updated_at TEXT NOT NULL DEFAULT '')`) + s.db = db + }) + if s.db == nil { + return nil, os.ErrNotExist + } + return s.db, nil +} + +// Close 单发退出时关库。 +func (s *GraphPatternStore) Close() { + s.mu.Lock() + defer s.mu.Unlock() + if s.db != nil { + s.db.Close() + s.db = nil + } + s.closed = true +} + +// propsJSON 一个 pattern 节点的 properties(graph_nodes.properties 列)。 +type propsJSON map[string]interface{} + +// UpsertPatterns 把检测出的 pattern 写回 graph_nodes(type='pattern')。 +// 返回新建数与更新数。空输入 → 0,0(AC1:空库不报错)。 +func (s *GraphPatternStore) UpsertPatterns(patterns []Pattern) (created, updated int, err error) { + if len(patterns) == 0 { + return 0, 0, nil + } + db, oerr := s.open() + if oerr != nil { + return 0, 0, oerr + } + now := time.Now().UTC().Format(time.RFC3339) + + for _, p := range patterns { + // 读现有 + var ( + id string + propsRaw string + hasExisting bool + ) + row := db.QueryRow(`SELECT id, properties FROM graph_nodes WHERE name=? AND type='pattern'`, p.Name) + if err := row.Scan(&id, &propsRaw); err == nil { + hasExisting = true + } + + var props propsJSON + if hasExisting && propsRaw != "" && propsRaw != "null" { + _ = json.Unmarshal([]byte(propsRaw), &props) + } + if props == nil { + props = propsJSON{} + } + + // 合并:新键覆盖、旧键保留 + props["pattern_type"] = p.Type + props["data"] = p.Data + props["confidence"] = p.Confidence + tools := p.Tools + if tools == nil { + tools = []string{} + } + props["tools"] = tools + props["source_tasks"] = p.SourceTasks + + var oldOcc int + if v, ok := props["occurrence_count"].(float64); ok { + oldOcc = int(v) + } + if hasExisting { + props["occurrence_count"] = oldOcc + 1 + props["last_seen"] = now + b, _ := json.Marshal(props) + if _, werr := db.Exec(`UPDATE graph_nodes SET properties=?, last_updated_at=? WHERE id=?`, + string(b), now, id); werr != nil { + err = werr + continue + } + updated++ + } else { + props["occurrence_count"] = 1 + props["first_seen"] = now + props["last_seen"] = now + b, _ := json.Marshal(props) + if _, ierr := db.Exec(`INSERT INTO graph_nodes (id, name, type, namespace, properties, created_at, last_updated_at) + VALUES (?,?,?,?,?,?,?)`, + p.ID, p.Name, "pattern", "daemon-distill", string(b), now, now); ierr != nil { + err = ierr + continue + } + created++ + } + } + if err != nil { + return + } + return +} + +// ListPatterns 列出 graph_nodes(type='pattern'),按 occurrence_count 降序取 top N。 +// 0 / 空库 → 空列表不报错(AC1)。 +func (s *GraphPatternStore) ListPatterns(limit int) ([]Pattern, error) { + if limit <= 0 { + limit = 10 + } + db, oerr := s.open() + if oerr != nil { + return nil, oerr + } + rows, qerr := db.Query(`SELECT id, name, properties FROM graph_nodes WHERE type='pattern' LIMIT ?`, limit*3) + if qerr != nil { + return nil, qerr + } + defer rows.Close() + type raw struct { + ID, Name, Props string + } + var all []raw + for rows.Next() { + var r raw + if serr := rows.Scan(&r.ID, &r.Name, &r.Props); serr != nil { + return nil, serr + } + all = append(all, r) + } + // 解析 props + occurrence 排序(sqlite 无 json 聚合,取 limit*3 再内存排) + out := make([]Pattern, 0, len(all)) + for _, r := range all { + p := Pattern{ID: r.ID, Name: r.Name} + var props propsJSON + if r.Props != "" && r.Props != "null" { + _ = json.Unmarshal([]byte(r.Props), &props) + } + if props != nil { + if t, ok := props["pattern_type"].(string); ok { + p.Type = t + } + if d, ok := props["data"].(string); ok { + p.Data = d + } + if c, ok := props["confidence"].(float64); ok { + p.Confidence = c + } + if o, ok := props["occurrence_count"].(float64); ok { + _ = o + } + if ts, ok := props["source_tasks"].([]interface{}); ok { + for _, v := range ts { + if s2, ok := v.(string); ok { + p.SourceTasks = append(p.SourceTasks, s2) + } + } + } + if tl, ok := props["tools"].([]interface{}); ok { + for _, v := range tl { + if s2, ok := v.(string); ok { + p.Tools = append(p.Tools, s2) + } + } + } + } + out = append(out, p) + } + // occurrence 降序(去重后 source_tasks 长度近似;存量行 source_tasks 可能为空 → 靠 0 兜底) + sort.SliceStable(out, func(i, j int) bool { + return len(out[i].SourceTasks) > len(out[j].SourceTasks) + }) + if len(out) > limit { + out = out[:limit] + } + return out, nil +} diff --git a/go/internal/pattern/store_test.go b/go/internal/pattern/store_test.go new file mode 100644 index 0000000..38cf0cd --- /dev/null +++ b/go/internal/pattern/store_test.go @@ -0,0 +1,116 @@ +package pattern + +import ( + "database/sql" + "encoding/json" + "testing" + + _ "modernc.org/sqlite" +) + +// newTestStore 指向 t.TempDir() 的空库(modernc 纯 Go,无需 CGO/网络)。 +func newTestStore(t *testing.T) *GraphPatternStore { + t.Helper() + dir := t.TempDir() + s := NewGraphPatternStore(dir + "/graph.db") + t.Cleanup(s.Close) + return s +} + +// TestStore_EmptyList —— 空库 ListPatterns → 0 条不报错(AC1 判据:SELECT count ≥0 不报错)。 +func TestStore_EmptyList(t *testing.T) { + s := newTestStore(t) + got, err := s.ListPatterns(10) + if err != nil { + t.Fatalf("空库 List 不应报错: %v", err) + } + if len(got) != 0 { + t.Fatalf("空库应 0 条 pattern,实际 %d", len(got)) + } +} + +// TestStore_UpsertEmpty —— 空输入 Upsert → 0,0,nil(AC1 判据)。 +func TestStore_UpsertEmpty(t *testing.T) { + s := newTestStore(t) + created, updated, err := s.UpsertPatterns(nil) + if err != nil { + t.Fatalf("空输入 Upsert 不应报错: %v", err) + } + if created != 0 || updated != 0 { + t.Fatalf("空输入应 0/0,实际 %d/%d", created, updated) + } +} + +// TestStore_UpsertCreate —— 新 pattern → INSERT 1 条,occurrence_count=1(AC2 前置)。 +func TestStore_UpsertCreate(t *testing.T) { + s := newTestStore(t) + p := Pattern{ + ID: "pat_tool_combo:curl+git+jq", + Type: "tool_combo", + Name: "tool_combo:curl+git+jq", + Data: "curl+git+jq", + Confidence: 0.3, + Tools: []string{"curl", "git", "jq"}, + SourceTasks: []string{"m_a", "m_b", "m_c"}, + } + created, updated, err := s.UpsertPatterns([]Pattern{p}) + if err != nil { + t.Fatalf("upsert 报错: %v", err) + } + if created != 1 || updated != 0 { + t.Fatalf("新 pattern 应 created=1/updated=0,实际 %d/%d", created, updated) + } + listed, _ := s.ListPatterns(10) + if len(listed) != 1 { + t.Fatalf("应 1 条,实际 %d", len(listed)) + } +} + +// TestStore_UpsertIdempotent —— 同名 pattern 二次 upsert → updated=1、occurrence_count=2。 +// 验证幂等语义(occurrence_count 驱动,AC2 sqlite 查判据):跑两次检测,不产生重复行。 +func TestStore_UpsertIdempotent(t *testing.T) { + s := newTestStore(t) + p := Pattern{ + ID: "pat_tool_combo:curl+git+jq", + Type: "tool_combo", + Name: "tool_combo:curl+git+jq", + Data: "curl+git+jq", + Confidence: 0.3, + Tools: []string{"curl", "git", "jq"}, + SourceTasks: []string{"m_a", "m_b", "m_c"}, + } + c1, u1, _ := s.UpsertPatterns([]Pattern{p}) // 新建 + p.Confidence = 0.4 // 第二次检测:同组合再次出现(freq 抬升) + p.SourceTasks = []string{"m_d"} + c2, u2, _ := s.UpsertPatterns([]Pattern{p}) // 应幂等更新(updated=1) + if c1 != 1 || u1 != 0 { + t.Fatalf("首次应 created=1/updated=0,实际 %d/%d", c1, u1) + } + if c2 != 0 || u2 != 1 { + t.Fatalf("二次应 created=0/updated=1,实际 %d/%d", c2, u2) + } + // 读库验:仍是 1 行、occurrence_count=2(幂等,非重复) + db, err := sql.Open("sqlite", s.dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + defer db.Close() + var ( + cnt int + occ int + ) + if err := db.QueryRow(`SELECT count(*) FROM graph_nodes WHERE type='pattern'`).Scan(&cnt); err != nil { + t.Fatal(err) + } + if cnt != 1 { + t.Fatalf("两次同 pattern 检测后应仍 1 行,实际 %d(幂等失败)", cnt) + } + var raw string + _ = db.QueryRow(`SELECT properties FROM graph_nodes WHERE type='pattern'`).Scan(&raw) + var props map[string]interface{} + _ = json.Unmarshal([]byte(raw), &props) + occ = int(props["occurrence_count"].(float64)) + if occ != 2 { + t.Fatalf("occurrence_count 应累计到 2,实际 %d", occ) + } +} diff --git a/go/internal/storage/recall.go b/go/internal/storage/recall.go index 9d6eb60..22cb086 100644 --- a/go/internal/storage/recall.go +++ b/go/internal/storage/recall.go @@ -51,6 +51,23 @@ var ( staleAfterHours = envFloat("ZHIYI_STALE_AFTER_HOURS", 0) ) +// ── t_d32e5f8a L2 模式召回注入(env 门控 ZHIYI_PATTERN_RECALL,默认 0 ⇒ 行为零变化)── +// Provider 由 server.go 装配:读 graph_nodes(type='pattern'),按 query/namespace 匹配 +// 出 top-k pattern,作为 Category="pattern" 的召回条目附带返回(设计 §4.1「命中即注入」)。 +// 未注册 Provider(默认 =0 时 server 不注册)⇒ 注入块整体跳过,召回结果逐条不变。 +// 注册了 Provider 但 ZHIYI_PATTERN_RECALL=0 ⇒ 同样跳过(关时零变化,AC3 判据)。 +var PatternRecallProvider func(query, namespace string, topK int) []models.RecallResult + +// patternRecallEnabled 每次调用读 env(测试可 t.Setenv 翻转,无需重跑 init)。 +// 默认(未设)= 关闭。 +func patternRecallEnabled() bool { + switch strings.ToLower(os.Getenv("ZHIYI_PATTERN_RECALL")) { + case "1", "true", "on": + return true + } + return false +} + func envFloat(key string, def float64) float64 { if v := os.Getenv(key); v != "" { if f, err := strconv.ParseFloat(v, 64); err == nil { @@ -522,6 +539,25 @@ func (p *RecallPipeline) recallUncached(query, namespace string, topK int, diver } results = deduped + // Step 6.2: t_d32e5f8a L2 模式召回注入(env 门控 ZHIYI_PATTERN_RECALL,默认 0) + // 命中 pattern 时以 Category="pattern" 附带 top-k 返回(设计「命中即注入」)。 + // 默认关 ⇒ 整块跳过,results 逐条不变(AC3 判据:固定 query 前后 diff 为空)。 + if patternRecallEnabled() && PatternRecallProvider != nil { + for _, pr := range PatternRecallProvider(query, namespace, 3) { + // 去重:不重复追加已在结果里的条目 + dup := false + for _, r := range results { + if r.ID == pr.ID { + dup = true + break + } + } + if !dup { + results = append(results, pr) + } + } + } + // Step 6.5: 写搜索缓存(§2.6 — t_1c12e000 起 TTL=60s,下次相同 query+topK+diversity // 直接返回;写路径落库后会 Invalidate 本 namespace,陈旧窗口上限=60s 双保险) if SearchCacheInstance != nil { diff --git a/go/internal/storage/recall_pattern_test.go b/go/internal/storage/recall_pattern_test.go new file mode 100644 index 0000000..bd3d58e --- /dev/null +++ b/go/internal/storage/recall_pattern_test.go @@ -0,0 +1,146 @@ +package storage + +import ( + "testing" + + "github.com/xiaoxue/memoryweave/internal/models" +) + +// AC3/AC4:L2 模式召回注入(t_d32e5f8a)的开关行为测试。 +// 判据: +// - 默认(ZHIYI_PATTERN_RECALL 未设/≠1)⇒ 即使注册了 Provider,召回结果也不变(零行为变化) +// - =1 ⇒ 命中 pattern 的 query 召回结果含 Category="pattern" 条目 + +// patternRecallLance 返回固定 2 条记忆 + 记录 Search 实参(注入测试专用假后端)。 +type patternRecallLance struct { + LanceDB + rows []models.MemoryRecord +} + +func (f *patternRecallLance) Search(table string, vector []float32, topK int, namespaceFilter string) ([]models.MemoryRecord, error) { + return f.rows, nil +} +func (f *patternRecallLance) Update(table, id string, fields map[string]any) error { return nil } + +func newPatternRecallPipeline(t *testing.T) (*RecallPipeline, *patternRecallLance) { + t.Helper() + saved := SearchCacheInstance + SearchCacheInstance = nil + t.Cleanup(func() { SearchCacheInstance = saved }) + + fake := &patternRecallLance{rows: []models.MemoryRecord{ + {ID: "m_base1", Content: "蒸馏模型日限计数器跨天重置", Category: "distilled"}, + {ID: "m_base2", Content: "用 jq git curl 处理日志", Category: "distilled"}, + }} + embsaved := VValueProvider + t.Cleanup(func() { VValueProvider = embsaved }) + + vec := make([]float32, 1024) + for i := range vec { + vec[i] = 0.01 + } + emb := &Embedder{endpoint: "http://127.0.0.1:1/never", dim: 1024} + emb.cache.Store("pattern 注入测试", vec) + p := NewRecallPipeline(emb, fake, &Reranker{}) + return p, fake +} + +// patternIDs 提取结果里 pattern 类条目 ID(判据:有无 Category="pattern")。 +func patternIDs(res []models.RecallResult) []string { + var out []string + for _, r := range res { + if r.Category == "pattern" { + out = append(out, r.ID) + } + } + return out +} + +// TestPatternRecall_DefaultOff —— AC3:env 默认(未设)⇒ 注册了 Provider 也不注入, +// 固定 query 召回前后 diff 为空(行为零变化)。 +func TestPatternRecall_DefaultOff(t *testing.T) { + t.Setenv("ZHIYI_PATTERN_RECALL", "") // 清空(模拟未设) + p, _ := newPatternRecallPipeline(t) + + // 注册一个会注入 2 条 pattern 的 Provider + saved := PatternRecallProvider + PatternRecallProvider = func(query, namespace string, topK int) []models.RecallResult { + return []models.RecallResult{ + {ID: "pat_x", Category: "pattern", Content: "tool_combo:curl+git+jq"}, + {ID: "pat_y", Category: "pattern", Content: "workflow:cluster-3-abcdef01"}, + } + } + t.Cleanup(func() { PatternRecallProvider = saved }) + + res, err := p.Recall("pattern 注入测试", "test-ns", 10, 0.0) + if err != nil { + t.Fatalf("recall 报错: %v", err) + } + if ids := patternIDs(res); len(ids) != 0 { + t.Fatalf("ZHIYI_PATTERN_RECALL 默认关时不应注入 pattern,实际注入了 %v", ids) + } + // 固定 query 前后 diff:再跑一次(仍关),结果必须一致 + res2, _ := p.Recall("pattern 注入测试", "test-ns", 10, 0.0) + if len(res) != len(res2) { + t.Fatalf("关时两次召回结果数不一致: %d vs %d", len(res), len(res2)) + } +} + +// TestPatternRecall_On —— AC4:ZHIYI_PATTERN_RECALL=1 ⇒ 召回结果含 pattern 条目(前后对照)。 +func TestPatternRecall_On(t *testing.T) { + t.Setenv("ZHIYI_PATTERN_RECALL", "1") + p, _ := newPatternRecallPipeline(t) + + // 关时(env 先置 0)无 pattern → 作为「前」 + t.Setenv("ZHIYI_PATTERN_RECALL", "0") + saved := PatternRecallProvider + PatternRecallProvider = func(query, namespace string, topK int) []models.RecallResult { + return []models.RecallResult{ + {ID: "pat_hit", Category: "pattern", Content: "tool_combo:curl+git+jq"}, + } + } + t.Cleanup(func() { PatternRecallProvider = saved }) + + before, _ := p.Recall("pattern 注入测试", "test-ns", 10, 0.0) + beforePat := patternIDs(before) + + // 开时(env=1)应有 pattern → 作为「后」 + t.Setenv("ZHIYI_PATTERN_RECALL", "1") + after, _ := p.Recall("pattern 注入测试", "test-ns", 10, 0.0) + afterPat := patternIDs(after) + + if len(beforePat) != 0 { + t.Fatalf("env=0(前)应无 pattern 注入,实际 %v", beforePat) + } + if len(afterPat) == 0 { + t.Fatalf("env=1(后)应注入 pattern,实际 %v", afterPat) + } + if afterPat[0] != "pat_hit" { + t.Fatalf("pattern 条目 ID 不符: %v", afterPat) + } +} + +// TestPatternRecall_Dedup —— 注入的 pattern ID 与既有结果重复时不重复追加。 +func TestPatternRecall_Dedup(t *testing.T) { + t.Setenv("ZHIYI_PATTERN_RECALL", "1") + p, _ := newPatternRecallPipeline(t) + saved := PatternRecallProvider + PatternRecallProvider = func(query, namespace string, topK int) []models.RecallResult { + // 返回一个与既有结果同 ID 的条目(m_base1)—— 不应重复 + return []models.RecallResult{ + {ID: "m_base1", Category: "pattern", Content: "dup-check"}, + } + } + t.Cleanup(func() { PatternRecallProvider = saved }) + + res, _ := p.Recall("pattern 注入测试", "test-ns", 10, 0.0) + cnt := 0 + for _, r := range res { + if r.ID == "m_base1" { + cnt++ + } + } + if cnt != 1 { + t.Fatalf("重复 ID 不应二次追加:m_base1 出现 %d 次", cnt) + } +}