diff --git a/go/cmd/pattern-fullscan-probe/main.go b/go/cmd/pattern-fullscan-probe/main.go new file mode 100644 index 0000000..06099dd --- /dev/null +++ b/go/cmd/pattern-fullscan-probe/main.go @@ -0,0 +1,174 @@ +// pattern-fullscan-probe — 全表 L2 检测探针(t_d8830204) +// 读:经 sidecar IPC lancedb_scan_all 取全量存活行(detect 端点被硬限 5000,物理序前 N 条 +// 是"不幸窗口"→ 检不到;本探针无上限,覆盖全表)。 +// 检:真实 pattern.DetectFromRecords(与生产同一份代码,不改判据、不降阈值)。 +// 写(仅 -write,且仅 conf≥0.85 的诚实过闸项): +// - L2:pattern.GraphPatternStore.UpsertPatterns → graph_nodes(type='pattern') +// - L3:skillgen.DraftSkill → skills(status=draft) + attachments//skill-draft.md +// 红线:L3 草稿绝不自动 promote;不自动进 skills/ 技能库。 +package main + +import ( + "flag" + "fmt" + "os" + "sort" + + "github.com/xiaoxue/memoryweave/internal/models" + "github.com/xiaoxue/memoryweave/internal/pattern" + "github.com/xiaoxue/memoryweave/internal/skillgen" + "github.com/xiaoxue/memoryweave/internal/storage" +) + +func joinPlus(tools []string) string { + out := "" + for i, t := range tools { + if i > 0 { + out += "+" + } + out += t + } + return out +} + +func main() { + write := flag.Bool("write", false, "write L2 patterns (conf>=0.5) + L3 drafts (conf>=0.85); default dry-run") + attRoot := flag.String("attach", "", "attachments root for L3 drafts (default skillgen.DefaultAttachmentsRoot)") + flag.Parse() + + sock := os.Getenv("LANCEDB_SOCKET") + if sock == "" { + sock = "/tmp/zhiyi-ipc.sock" + } + rc := storage.NewRustLanceDBClient(sock, nil) + recs, err := rc.ScanAll() + if err != nil { + fmt.Println("scan_all error:", err) + os.Exit(1) + } + dist := make([]models.MemoryRecord, 0, len(recs)) + for _, r := range recs { + if r.Category == "distilled" || r.Category == "" { + dist = append(dist, r) + } + } + fmt.Printf("scan_all live=%d distilled_scanned=%d\n", len(recs), len(dist)) + + // debug: 抽样看 content 是否非空 + 工具命中 + nonEmpty := 0 + samples := 0 + for _, r := range dist { + if r.Content != "" { + nonEmpty++ + } + if samples < 3 && r.Content != "" { + fmt.Printf(" sample id=%s len=%d tools=%v content=%.80s\n", + r.ID, len(r.Content), pattern.ExtractTools(r.Content), r.Content) + samples++ + } + } + fmt.Printf("non_empty_content=%d/%d\n", nonEmpty, len(dist)) + + // debug: ID 是否非空(DetectFromRecords 要求非空 ID 去重计源) + emptyID := 0 + for _, r := range dist { + if r.ID == "" { + emptyID++ + } + } + fmt.Printf("empty_ID=%d/%d\n", emptyID, len(dist)) + + // debug: 工具命中统计(复刻 combo 计数) + toolRecs := 0 + comboN := map[string]int{} + for _, r := range dist { + tools := pattern.ExtractTools(r.Content) + if len(tools) >= 2 { + toolRecs++ + comboN["tool_combo:"+joinPlus(tools)]++ + } + } + fmt.Printf("records_with_>=2_tools=%d\n--- top combos (records, not dedup src) ---\n", toolRecs) + type kv struct { + k string + v int + } + var arr []kv + for k, v := range comboN { + arr = append(arr, kv{k, v}) + } + sort.Slice(arr, func(i, j int) bool { return arr[i].v > arr[j].v }) + for i, e := range arr { + if i >= 15 { + break + } + fmt.Printf(" %-45s %d\n", e.k, e.v) + } + + cfg := pattern.DefaultConfig() // 不改任何阈值(MinWriteConfidence=0.5, DraftThreshold=0.85) + cfg.MaxRecords = len(dist) // 覆盖全表(DefaultConfig 仅 500;API 端点又硬限 5000) + pats := pattern.DetectFromRecords(dist, cfg) + fmt.Printf("detected=%d\n", len(pats)) + // debug: 打印前 3 个 pattern 的 ID/name/conf/src + for i, p := range pats { + if i >= 3 { + break + } + fmt.Printf(" pat[%d] id=%q name=%q conf=%.2f nsrc=%d\n", i, p.ID, p.Name, p.Confidence, len(p.SourceTasks)) + } + sort.SliceStable(pats, func(i, j int) bool { return len(pats[i].SourceTasks) > len(pats[j].SourceTasks) }) + for _, p := range pats { + fmt.Printf(" %-45s type=%s conf=%.2f nsrc=%d\n", p.Name, p.Type, p.Confidence, len(p.SourceTasks)) + } + + writeOK, draftOK := []pattern.Pattern{}, []pattern.Pattern{} + for _, p := range pats { + if p.Confidence >= pattern.MinWriteConfidence { + writeOK = append(writeOK, p) + } + if p.Confidence >= skillgen.DraftThreshold { + draftOK = append(draftOK, p) + } + } + fmt.Printf("pass write-gate(>=%.1f)=%d pass L3(>=%.2f)=%d\n", + pattern.MinWriteConfidence, len(writeOK), skillgen.DraftThreshold, len(draftOK)) + + if !*write { + fmt.Println("DRY-RUN: 不写库、不落盘(加 -write 才落库)") + return + } + + graphPath := os.Getenv("GRAPH_PATH") + if graphPath == "" { + graphPath = "/var/lib/memoryweave/graph.db" + } + // L2 写入(conf>=0.5,生产同一 UpsertPatterns 含 IsValidPatternName 写闸) + pstore := pattern.NewGraphPatternStore(graphPath) + defer pstore.Close() + c, u, e := pstore.UpsertPatterns(writeOK) + if e != nil { + fmt.Println("UpsertPatterns error:", e) + } + fmt.Printf("L2 upsert: created=%d updated=%d\n", c, u) + + // L3 草稿(仅 conf>=0.85;本地 2B;绝不 promote) + if len(draftOK) > 0 { + sstore := skillgen.NewStore(graphPath) + defer sstore.Close() + gen := skillgen.NewLocal2BGenerator() + root := *attRoot + if root == "" { + root = skillgen.DefaultAttachmentsRoot() + } + for _, p := range draftOK { + sk, path, derr := skillgen.DraftSkill(sstore, gen, p, root) + if derr != nil { + fmt.Printf(" L3 draft FAIL %s: %v\n", p.Name, derr) + continue + } + fmt.Printf(" L3 draft OK %s -> status=%s file=%s\n", p.Name, sk.Status, path) + } + n, _ := sstore.Count("draft") + fmt.Printf("skills draft count=%d\n", n) + } +}