From 714df5d2a8137d1f8eccbc6bc7f6e1e9fed192e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=94=AF?= Date: Mon, 9 Nov 2026 00:13:34 +0800 Subject: [PATCH] =?UTF-8?q?feat(zhiyi):=20L2=E5=85=A8=E8=A1=A8=E6=A3=80?= =?UTF-8?q?=E6=B5=8B=E6=8E=A2=E9=92=88=20(t=5Fd8830204)=20=E2=80=94=20?= =?UTF-8?q?=E6=89=93=E9=80=9A=20L2=20pattern=200=E8=A1=8C=E2=86=92?= =?UTF-8?q?=E9=9D=9E0=20=E4=B8=8E=20L3=20=E8=8D=89=E7=A8=BF=E7=94=9F?= =?UTF-8?q?=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 背景:POST /patterns/detect 被 lancedb_scan 硬限 5000 且 DetectFromRecords 默认 MaxRecords=500 (物理序前500条是'不幸窗口'检出0)→ 已有 pattern-detect-daily cron(limit=500) 空转,graph_nodes type='pattern' 恒 0。本探针走 sidecar IPC lancedb_scan_all(无5000上限)+ 全表 MaxRecords + 诚实阈值: L2 conf>=0.5 写 graph_nodes(type='pattern',ns=daemon-distill),L3 conf>=0.85 才出草稿到 attachments//skill-draft.md([AUTO-DRAFT]标记,status=draft,绝不自动promote)。 -cwrite 才落库,默认 dry-run。 实测(全表45227行/distilled45096):6个tool_combo,仅 hermes+python (conf=1.0,13源) 过 L2/L3 双闸。 写库后 graph_nodes pattern 0→1, skills 0→1(draft);召回 top5 10query 前后逐位一致(零行为变化)。 本地2B(8081)生成草稿落盘。 --- go/cmd/pattern-fullscan-probe/main.go | 174 ++++++++++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 go/cmd/pattern-fullscan-probe/main.go diff --git a/go/cmd/pattern-fullscan-probe/main.go b/go/cmd/pattern-fullscan-probe/main.go new file mode 100644 index 0000000..06099dd --- /dev/null +++ b/go/cmd/pattern-fullscan-probe/main.go @@ -0,0 +1,174 @@ +// pattern-fullscan-probe — 全表 L2 检测探针(t_d8830204) +// 读:经 sidecar IPC lancedb_scan_all 取全量存活行(detect 端点被硬限 5000,物理序前 N 条 +// 是"不幸窗口"→ 检不到;本探针无上限,覆盖全表)。 +// 检:真实 pattern.DetectFromRecords(与生产同一份代码,不改判据、不降阈值)。 +// 写(仅 -write,且仅 conf≥0.85 的诚实过闸项): +// - L2:pattern.GraphPatternStore.UpsertPatterns → graph_nodes(type='pattern') +// - L3:skillgen.DraftSkill → skills(status=draft) + attachments//skill-draft.md +// 红线:L3 草稿绝不自动 promote;不自动进 skills/ 技能库。 +package main + +import ( + "flag" + "fmt" + "os" + "sort" + + "github.com/xiaoxue/memoryweave/internal/models" + "github.com/xiaoxue/memoryweave/internal/pattern" + "github.com/xiaoxue/memoryweave/internal/skillgen" + "github.com/xiaoxue/memoryweave/internal/storage" +) + +func joinPlus(tools []string) string { + out := "" + for i, t := range tools { + if i > 0 { + out += "+" + } + out += t + } + return out +} + +func main() { + write := flag.Bool("write", false, "write L2 patterns (conf>=0.5) + L3 drafts (conf>=0.85); default dry-run") + attRoot := flag.String("attach", "", "attachments root for L3 drafts (default skillgen.DefaultAttachmentsRoot)") + flag.Parse() + + sock := os.Getenv("LANCEDB_SOCKET") + if sock == "" { + sock = "/tmp/zhiyi-ipc.sock" + } + rc := storage.NewRustLanceDBClient(sock, nil) + recs, err := rc.ScanAll() + if err != nil { + fmt.Println("scan_all error:", err) + os.Exit(1) + } + dist := make([]models.MemoryRecord, 0, len(recs)) + for _, r := range recs { + if r.Category == "distilled" || r.Category == "" { + dist = append(dist, r) + } + } + fmt.Printf("scan_all live=%d distilled_scanned=%d\n", len(recs), len(dist)) + + // debug: 抽样看 content 是否非空 + 工具命中 + nonEmpty := 0 + samples := 0 + for _, r := range dist { + if r.Content != "" { + nonEmpty++ + } + if samples < 3 && r.Content != "" { + fmt.Printf(" sample id=%s len=%d tools=%v content=%.80s\n", + r.ID, len(r.Content), pattern.ExtractTools(r.Content), r.Content) + samples++ + } + } + fmt.Printf("non_empty_content=%d/%d\n", nonEmpty, len(dist)) + + // debug: ID 是否非空(DetectFromRecords 要求非空 ID 去重计源) + emptyID := 0 + for _, r := range dist { + if r.ID == "" { + emptyID++ + } + } + fmt.Printf("empty_ID=%d/%d\n", emptyID, len(dist)) + + // debug: 工具命中统计(复刻 combo 计数) + toolRecs := 0 + comboN := map[string]int{} + for _, r := range dist { + tools := pattern.ExtractTools(r.Content) + if len(tools) >= 2 { + toolRecs++ + comboN["tool_combo:"+joinPlus(tools)]++ + } + } + fmt.Printf("records_with_>=2_tools=%d\n--- top combos (records, not dedup src) ---\n", toolRecs) + type kv struct { + k string + v int + } + var arr []kv + for k, v := range comboN { + arr = append(arr, kv{k, v}) + } + sort.Slice(arr, func(i, j int) bool { return arr[i].v > arr[j].v }) + for i, e := range arr { + if i >= 15 { + break + } + fmt.Printf(" %-45s %d\n", e.k, e.v) + } + + cfg := pattern.DefaultConfig() // 不改任何阈值(MinWriteConfidence=0.5, DraftThreshold=0.85) + cfg.MaxRecords = len(dist) // 覆盖全表(DefaultConfig 仅 500;API 端点又硬限 5000) + pats := pattern.DetectFromRecords(dist, cfg) + fmt.Printf("detected=%d\n", len(pats)) + // debug: 打印前 3 个 pattern 的 ID/name/conf/src + for i, p := range pats { + if i >= 3 { + break + } + fmt.Printf(" pat[%d] id=%q name=%q conf=%.2f nsrc=%d\n", i, p.ID, p.Name, p.Confidence, len(p.SourceTasks)) + } + sort.SliceStable(pats, func(i, j int) bool { return len(pats[i].SourceTasks) > len(pats[j].SourceTasks) }) + for _, p := range pats { + fmt.Printf(" %-45s type=%s conf=%.2f nsrc=%d\n", p.Name, p.Type, p.Confidence, len(p.SourceTasks)) + } + + writeOK, draftOK := []pattern.Pattern{}, []pattern.Pattern{} + for _, p := range pats { + if p.Confidence >= pattern.MinWriteConfidence { + writeOK = append(writeOK, p) + } + if p.Confidence >= skillgen.DraftThreshold { + draftOK = append(draftOK, p) + } + } + fmt.Printf("pass write-gate(>=%.1f)=%d pass L3(>=%.2f)=%d\n", + pattern.MinWriteConfidence, len(writeOK), skillgen.DraftThreshold, len(draftOK)) + + if !*write { + fmt.Println("DRY-RUN: 不写库、不落盘(加 -write 才落库)") + return + } + + graphPath := os.Getenv("GRAPH_PATH") + if graphPath == "" { + graphPath = "/var/lib/memoryweave/graph.db" + } + // L2 写入(conf>=0.5,生产同一 UpsertPatterns 含 IsValidPatternName 写闸) + pstore := pattern.NewGraphPatternStore(graphPath) + defer pstore.Close() + c, u, e := pstore.UpsertPatterns(writeOK) + if e != nil { + fmt.Println("UpsertPatterns error:", e) + } + fmt.Printf("L2 upsert: created=%d updated=%d\n", c, u) + + // L3 草稿(仅 conf>=0.85;本地 2B;绝不 promote) + if len(draftOK) > 0 { + sstore := skillgen.NewStore(graphPath) + defer sstore.Close() + gen := skillgen.NewLocal2BGenerator() + root := *attRoot + if root == "" { + root = skillgen.DefaultAttachmentsRoot() + } + for _, p := range draftOK { + sk, path, derr := skillgen.DraftSkill(sstore, gen, p, root) + if derr != nil { + fmt.Printf(" L3 draft FAIL %s: %v\n", p.Name, derr) + continue + } + fmt.Printf(" L3 draft OK %s -> status=%s file=%s\n", p.Name, sk.Status, path) + } + n, _ := sstore.Count("draft") + fmt.Printf("skills draft count=%d\n", n) + } +}