Files
memoryweave/go/cmd/pattern-fullscan-probe/main.go
T
小唯 714df5d2a8 feat(zhiyi): L2全表检测探针 (t_d8830204) — 打通 L2 pattern 0行→非0 与 L3 草稿生成
背景:POST /patterns/detect 被 lancedb_scan 硬限 5000 且 DetectFromRecords 默认 MaxRecords=500
(物理序前500条是'不幸窗口'检出0)→ 已有 pattern-detect-daily cron(limit=500) 空转,graph_nodes
type='pattern' 恒 0。本探针走 sidecar IPC lancedb_scan_all(无5000上限)+ 全表 MaxRecords + 诚实阈值:
L2 conf>=0.5 写 graph_nodes(type='pattern',ns=daemon-distill),L3 conf>=0.85 才出草稿到
attachments/<pattern>/skill-draft.md([AUTO-DRAFT]标记,status=draft,绝不自动promote)。
-cwrite 才落库,默认 dry-run。

实测(全表45227行/distilled45096):6个tool_combo,仅 hermes+python (conf=1.0,13源) 过 L2/L3 双闸。
写库后 graph_nodes pattern 0→1, skills 0→1(draft);召回 top5 10query 前后逐位一致(零行为变化)。
本地2B(8081)生成草稿落盘。
2026-11-09 00:13:34 +08:00

175 lines
5.3 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// pattern-fullscan-probe — 全表 L2 检测探针(t_d8830204)
// 读:经 sidecar IPC lancedb_scan_all 取全量存活行(detect 端点被硬限 5000,物理序前 N 条
// 是"不幸窗口"→ 检不到;本探针无上限,覆盖全表)。
// 检:真实 pattern.DetectFromRecords(与生产同一份代码,不改判据、不降阈值)。
// 写(仅 -write,且仅 conf≥0.85 的诚实过闸项):
// - L2:pattern.GraphPatternStore.UpsertPatterns → graph_nodes(type='pattern')
// - L3:skillgen.DraftSkill → skills(status=draft) + attachments/<pattern>/skill-draft.md
// 红线:L3 草稿绝不自动 promote;不自动进 skills/ 技能库。
package main
import (
"flag"
"fmt"
"os"
"sort"
"github.com/xiaoxue/memoryweave/internal/models"
"github.com/xiaoxue/memoryweave/internal/pattern"
"github.com/xiaoxue/memoryweave/internal/skillgen"
"github.com/xiaoxue/memoryweave/internal/storage"
)
func joinPlus(tools []string) string {
out := ""
for i, t := range tools {
if i > 0 {
out += "+"
}
out += t
}
return out
}
func main() {
write := flag.Bool("write", false, "write L2 patterns (conf>=0.5) + L3 drafts (conf>=0.85); default dry-run")
attRoot := flag.String("attach", "", "attachments root for L3 drafts (default skillgen.DefaultAttachmentsRoot)")
flag.Parse()
sock := os.Getenv("LANCEDB_SOCKET")
if sock == "" {
sock = "/tmp/zhiyi-ipc.sock"
}
rc := storage.NewRustLanceDBClient(sock, nil)
recs, err := rc.ScanAll()
if err != nil {
fmt.Println("scan_all error:", err)
os.Exit(1)
}
dist := make([]models.MemoryRecord, 0, len(recs))
for _, r := range recs {
if r.Category == "distilled" || r.Category == "" {
dist = append(dist, r)
}
}
fmt.Printf("scan_all live=%d distilled_scanned=%d\n", len(recs), len(dist))
// debug: 抽样看 content 是否非空 + 工具命中
nonEmpty := 0
samples := 0
for _, r := range dist {
if r.Content != "" {
nonEmpty++
}
if samples < 3 && r.Content != "" {
fmt.Printf(" sample id=%s len=%d tools=%v content=%.80s\n",
r.ID, len(r.Content), pattern.ExtractTools(r.Content), r.Content)
samples++
}
}
fmt.Printf("non_empty_content=%d/%d\n", nonEmpty, len(dist))
// debug: ID 是否非空(DetectFromRecords 要求非空 ID 去重计源)
emptyID := 0
for _, r := range dist {
if r.ID == "" {
emptyID++
}
}
fmt.Printf("empty_ID=%d/%d\n", emptyID, len(dist))
// debug: 工具命中统计(复刻 combo 计数)
toolRecs := 0
comboN := map[string]int{}
for _, r := range dist {
tools := pattern.ExtractTools(r.Content)
if len(tools) >= 2 {
toolRecs++
comboN["tool_combo:"+joinPlus(tools)]++
}
}
fmt.Printf("records_with_>=2_tools=%d\n--- top combos (records, not dedup src) ---\n", toolRecs)
type kv struct {
k string
v int
}
var arr []kv
for k, v := range comboN {
arr = append(arr, kv{k, v})
}
sort.Slice(arr, func(i, j int) bool { return arr[i].v > arr[j].v })
for i, e := range arr {
if i >= 15 {
break
}
fmt.Printf(" %-45s %d\n", e.k, e.v)
}
cfg := pattern.DefaultConfig() // 不改任何阈值(MinWriteConfidence=0.5, DraftThreshold=0.85)
cfg.MaxRecords = len(dist) // 覆盖全表(DefaultConfig 仅 500;API 端点又硬限 5000)
pats := pattern.DetectFromRecords(dist, cfg)
fmt.Printf("detected=%d\n", len(pats))
// debug: 打印前 3 个 pattern 的 ID/name/conf/src
for i, p := range pats {
if i >= 3 {
break
}
fmt.Printf(" pat[%d] id=%q name=%q conf=%.2f nsrc=%d\n", i, p.ID, p.Name, p.Confidence, len(p.SourceTasks))
}
sort.SliceStable(pats, func(i, j int) bool { return len(pats[i].SourceTasks) > len(pats[j].SourceTasks) })
for _, p := range pats {
fmt.Printf(" %-45s type=%s conf=%.2f nsrc=%d\n", p.Name, p.Type, p.Confidence, len(p.SourceTasks))
}
writeOK, draftOK := []pattern.Pattern{}, []pattern.Pattern{}
for _, p := range pats {
if p.Confidence >= pattern.MinWriteConfidence {
writeOK = append(writeOK, p)
}
if p.Confidence >= skillgen.DraftThreshold {
draftOK = append(draftOK, p)
}
}
fmt.Printf("pass write-gate(>=%.1f)=%d pass L3(>=%.2f)=%d\n",
pattern.MinWriteConfidence, len(writeOK), skillgen.DraftThreshold, len(draftOK))
if !*write {
fmt.Println("DRY-RUN: 不写库、不落盘(加 -write 才落库)")
return
}
graphPath := os.Getenv("GRAPH_PATH")
if graphPath == "" {
graphPath = "/var/lib/memoryweave/graph.db"
}
// L2 写入(conf>=0.5,生产同一 UpsertPatterns 含 IsValidPatternName 写闸)
pstore := pattern.NewGraphPatternStore(graphPath)
defer pstore.Close()
c, u, e := pstore.UpsertPatterns(writeOK)
if e != nil {
fmt.Println("UpsertPatterns error:", e)
}
fmt.Printf("L2 upsert: created=%d updated=%d\n", c, u)
// L3 草稿(仅 conf>=0.85;本地 2B;绝不 promote)
if len(draftOK) > 0 {
sstore := skillgen.NewStore(graphPath)
defer sstore.Close()
gen := skillgen.NewLocal2BGenerator()
root := *attRoot
if root == "" {
root = skillgen.DefaultAttachmentsRoot()
}
for _, p := range draftOK {
sk, path, derr := skillgen.DraftSkill(sstore, gen, p, root)
if derr != nil {
fmt.Printf(" L3 draft FAIL %s: %v\n", p.Name, derr)
continue
}
fmt.Printf(" L3 draft OK %s -> status=%s file=%s\n", p.Name, sk.Status, path)
}
n, _ := sstore.Count("draft")
fmt.Printf("skills draft count=%d\n", n)
}
}