feat(zhiyi): L2全表检测探针 (t_d8830204) — 打通 L2 pattern 0行→非0 与 L3 草稿生成

背景:POST /patterns/detect 被 lancedb_scan 硬限 5000 且 DetectFromRecords 默认 MaxRecords=500
(物理序前500条是'不幸窗口'检出0)→ 已有 pattern-detect-daily cron(limit=500) 空转,graph_nodes
type='pattern' 恒 0。本探针走 sidecar IPC lancedb_scan_all(无5000上限)+ 全表 MaxRecords + 诚实阈值:
L2 conf>=0.5 写 graph_nodes(type='pattern',ns=daemon-distill),L3 conf>=0.85 才出草稿到
attachments/<pattern>/skill-draft.md([AUTO-DRAFT]标记,status=draft,绝不自动promote)。
-cwrite 才落库,默认 dry-run。

实测(全表45227行/distilled45096):6个tool_combo,仅 hermes+python (conf=1.0,13源) 过 L2/L3 双闸。
写库后 graph_nodes pattern 0→1, skills 0→1(draft);召回 top5 10query 前后逐位一致(零行为变化)。
本地2B(8081)生成草稿落盘。
This commit is contained in:
小唯
2026-11-09 00:13:34 +08:00
parent 6cbc49ca0e
commit 714df5d2a8
+174
View File
@@ -0,0 +1,174 @@
// pattern-fullscan-probe — 全表 L2 检测探针(t_d8830204)
// 读:经 sidecar IPC lancedb_scan_all 取全量存活行(detect 端点被硬限 5000,物理序前 N 条
// 是"不幸窗口"→ 检不到;本探针无上限,覆盖全表)。
// 检:真实 pattern.DetectFromRecords(与生产同一份代码,不改判据、不降阈值)。
// 写(仅 -write,且仅 conf≥0.85 的诚实过闸项):
// - L2:pattern.GraphPatternStore.UpsertPatterns → graph_nodes(type='pattern')
// - L3:skillgen.DraftSkill → skills(status=draft) + attachments/<pattern>/skill-draft.md
// 红线:L3 草稿绝不自动 promote;不自动进 skills/ 技能库。
package main
import (
"flag"
"fmt"
"os"
"sort"
"github.com/xiaoxue/memoryweave/internal/models"
"github.com/xiaoxue/memoryweave/internal/pattern"
"github.com/xiaoxue/memoryweave/internal/skillgen"
"github.com/xiaoxue/memoryweave/internal/storage"
)
func joinPlus(tools []string) string {
out := ""
for i, t := range tools {
if i > 0 {
out += "+"
}
out += t
}
return out
}
func main() {
write := flag.Bool("write", false, "write L2 patterns (conf>=0.5) + L3 drafts (conf>=0.85); default dry-run")
attRoot := flag.String("attach", "", "attachments root for L3 drafts (default skillgen.DefaultAttachmentsRoot)")
flag.Parse()
sock := os.Getenv("LANCEDB_SOCKET")
if sock == "" {
sock = "/tmp/zhiyi-ipc.sock"
}
rc := storage.NewRustLanceDBClient(sock, nil)
recs, err := rc.ScanAll()
if err != nil {
fmt.Println("scan_all error:", err)
os.Exit(1)
}
dist := make([]models.MemoryRecord, 0, len(recs))
for _, r := range recs {
if r.Category == "distilled" || r.Category == "" {
dist = append(dist, r)
}
}
fmt.Printf("scan_all live=%d distilled_scanned=%d\n", len(recs), len(dist))
// debug: 抽样看 content 是否非空 + 工具命中
nonEmpty := 0
samples := 0
for _, r := range dist {
if r.Content != "" {
nonEmpty++
}
if samples < 3 && r.Content != "" {
fmt.Printf(" sample id=%s len=%d tools=%v content=%.80s\n",
r.ID, len(r.Content), pattern.ExtractTools(r.Content), r.Content)
samples++
}
}
fmt.Printf("non_empty_content=%d/%d\n", nonEmpty, len(dist))
// debug: ID 是否非空(DetectFromRecords 要求非空 ID 去重计源)
emptyID := 0
for _, r := range dist {
if r.ID == "" {
emptyID++
}
}
fmt.Printf("empty_ID=%d/%d\n", emptyID, len(dist))
// debug: 工具命中统计(复刻 combo 计数)
toolRecs := 0
comboN := map[string]int{}
for _, r := range dist {
tools := pattern.ExtractTools(r.Content)
if len(tools) >= 2 {
toolRecs++
comboN["tool_combo:"+joinPlus(tools)]++
}
}
fmt.Printf("records_with_>=2_tools=%d\n--- top combos (records, not dedup src) ---\n", toolRecs)
type kv struct {
k string
v int
}
var arr []kv
for k, v := range comboN {
arr = append(arr, kv{k, v})
}
sort.Slice(arr, func(i, j int) bool { return arr[i].v > arr[j].v })
for i, e := range arr {
if i >= 15 {
break
}
fmt.Printf(" %-45s %d\n", e.k, e.v)
}
cfg := pattern.DefaultConfig() // 不改任何阈值(MinWriteConfidence=0.5, DraftThreshold=0.85)
cfg.MaxRecords = len(dist) // 覆盖全表(DefaultConfig 仅 500;API 端点又硬限 5000)
pats := pattern.DetectFromRecords(dist, cfg)
fmt.Printf("detected=%d\n", len(pats))
// debug: 打印前 3 个 pattern 的 ID/name/conf/src
for i, p := range pats {
if i >= 3 {
break
}
fmt.Printf(" pat[%d] id=%q name=%q conf=%.2f nsrc=%d\n", i, p.ID, p.Name, p.Confidence, len(p.SourceTasks))
}
sort.SliceStable(pats, func(i, j int) bool { return len(pats[i].SourceTasks) > len(pats[j].SourceTasks) })
for _, p := range pats {
fmt.Printf(" %-45s type=%s conf=%.2f nsrc=%d\n", p.Name, p.Type, p.Confidence, len(p.SourceTasks))
}
writeOK, draftOK := []pattern.Pattern{}, []pattern.Pattern{}
for _, p := range pats {
if p.Confidence >= pattern.MinWriteConfidence {
writeOK = append(writeOK, p)
}
if p.Confidence >= skillgen.DraftThreshold {
draftOK = append(draftOK, p)
}
}
fmt.Printf("pass write-gate(>=%.1f)=%d pass L3(>=%.2f)=%d\n",
pattern.MinWriteConfidence, len(writeOK), skillgen.DraftThreshold, len(draftOK))
if !*write {
fmt.Println("DRY-RUN: 不写库、不落盘(加 -write 才落库)")
return
}
graphPath := os.Getenv("GRAPH_PATH")
if graphPath == "" {
graphPath = "/var/lib/memoryweave/graph.db"
}
// L2 写入(conf>=0.5,生产同一 UpsertPatterns 含 IsValidPatternName 写闸)
pstore := pattern.NewGraphPatternStore(graphPath)
defer pstore.Close()
c, u, e := pstore.UpsertPatterns(writeOK)
if e != nil {
fmt.Println("UpsertPatterns error:", e)
}
fmt.Printf("L2 upsert: created=%d updated=%d\n", c, u)
// L3 草稿(仅 conf>=0.85;本地 2B;绝不 promote)
if len(draftOK) > 0 {
sstore := skillgen.NewStore(graphPath)
defer sstore.Close()
gen := skillgen.NewLocal2BGenerator()
root := *attRoot
if root == "" {
root = skillgen.DefaultAttachmentsRoot()
}
for _, p := range draftOK {
sk, path, derr := skillgen.DraftSkill(sstore, gen, p, root)
if derr != nil {
fmt.Printf(" L3 draft FAIL %s: %v\n", p.Name, derr)
continue
}
fmt.Printf(" L3 draft OK %s -> status=%s file=%s\n", p.Name, sk.Status, path)
}
n, _ := sstore.Count("draft")
fmt.Printf("skills draft count=%d\n", n)
}
}