feat(zhiyi): L2全表检测探针 (t_d8830204) — 打通 L2 pattern 0行→非0 与 L3 草稿生成
背景:POST /patterns/detect 被 lancedb_scan 硬限 5000 且 DetectFromRecords 默认 MaxRecords=500 (物理序前500条是'不幸窗口'检出0)→ 已有 pattern-detect-daily cron(limit=500) 空转,graph_nodes type='pattern' 恒 0。本探针走 sidecar IPC lancedb_scan_all(无5000上限)+ 全表 MaxRecords + 诚实阈值: L2 conf>=0.5 写 graph_nodes(type='pattern',ns=daemon-distill),L3 conf>=0.85 才出草稿到 attachments/<pattern>/skill-draft.md([AUTO-DRAFT]标记,status=draft,绝不自动promote)。 -cwrite 才落库,默认 dry-run。 实测(全表45227行/distilled45096):6个tool_combo,仅 hermes+python (conf=1.0,13源) 过 L2/L3 双闸。 写库后 graph_nodes pattern 0→1, skills 0→1(draft);召回 top5 10query 前后逐位一致(零行为变化)。 本地2B(8081)生成草稿落盘。
This commit is contained in:
@@ -0,0 +1,174 @@
|
||||
// pattern-fullscan-probe — 全表 L2 检测探针(t_d8830204)
|
||||
// 读:经 sidecar IPC lancedb_scan_all 取全量存活行(detect 端点被硬限 5000,物理序前 N 条
|
||||
// 是"不幸窗口"→ 检不到;本探针无上限,覆盖全表)。
|
||||
// 检:真实 pattern.DetectFromRecords(与生产同一份代码,不改判据、不降阈值)。
|
||||
// 写(仅 -write,且仅 conf≥0.85 的诚实过闸项):
|
||||
// - L2:pattern.GraphPatternStore.UpsertPatterns → graph_nodes(type='pattern')
|
||||
// - L3:skillgen.DraftSkill → skills(status=draft) + attachments/<pattern>/skill-draft.md
|
||||
// 红线:L3 草稿绝不自动 promote;不自动进 skills/ 技能库。
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
|
||||
"github.com/xiaoxue/memoryweave/internal/models"
|
||||
"github.com/xiaoxue/memoryweave/internal/pattern"
|
||||
"github.com/xiaoxue/memoryweave/internal/skillgen"
|
||||
"github.com/xiaoxue/memoryweave/internal/storage"
|
||||
)
|
||||
|
||||
func joinPlus(tools []string) string {
|
||||
out := ""
|
||||
for i, t := range tools {
|
||||
if i > 0 {
|
||||
out += "+"
|
||||
}
|
||||
out += t
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func main() {
|
||||
write := flag.Bool("write", false, "write L2 patterns (conf>=0.5) + L3 drafts (conf>=0.85); default dry-run")
|
||||
attRoot := flag.String("attach", "", "attachments root for L3 drafts (default skillgen.DefaultAttachmentsRoot)")
|
||||
flag.Parse()
|
||||
|
||||
sock := os.Getenv("LANCEDB_SOCKET")
|
||||
if sock == "" {
|
||||
sock = "/tmp/zhiyi-ipc.sock"
|
||||
}
|
||||
rc := storage.NewRustLanceDBClient(sock, nil)
|
||||
recs, err := rc.ScanAll()
|
||||
if err != nil {
|
||||
fmt.Println("scan_all error:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
dist := make([]models.MemoryRecord, 0, len(recs))
|
||||
for _, r := range recs {
|
||||
if r.Category == "distilled" || r.Category == "" {
|
||||
dist = append(dist, r)
|
||||
}
|
||||
}
|
||||
fmt.Printf("scan_all live=%d distilled_scanned=%d\n", len(recs), len(dist))
|
||||
|
||||
// debug: 抽样看 content 是否非空 + 工具命中
|
||||
nonEmpty := 0
|
||||
samples := 0
|
||||
for _, r := range dist {
|
||||
if r.Content != "" {
|
||||
nonEmpty++
|
||||
}
|
||||
if samples < 3 && r.Content != "" {
|
||||
fmt.Printf(" sample id=%s len=%d tools=%v content=%.80s\n",
|
||||
r.ID, len(r.Content), pattern.ExtractTools(r.Content), r.Content)
|
||||
samples++
|
||||
}
|
||||
}
|
||||
fmt.Printf("non_empty_content=%d/%d\n", nonEmpty, len(dist))
|
||||
|
||||
// debug: ID 是否非空(DetectFromRecords 要求非空 ID 去重计源)
|
||||
emptyID := 0
|
||||
for _, r := range dist {
|
||||
if r.ID == "" {
|
||||
emptyID++
|
||||
}
|
||||
}
|
||||
fmt.Printf("empty_ID=%d/%d\n", emptyID, len(dist))
|
||||
|
||||
// debug: 工具命中统计(复刻 combo 计数)
|
||||
toolRecs := 0
|
||||
comboN := map[string]int{}
|
||||
for _, r := range dist {
|
||||
tools := pattern.ExtractTools(r.Content)
|
||||
if len(tools) >= 2 {
|
||||
toolRecs++
|
||||
comboN["tool_combo:"+joinPlus(tools)]++
|
||||
}
|
||||
}
|
||||
fmt.Printf("records_with_>=2_tools=%d\n--- top combos (records, not dedup src) ---\n", toolRecs)
|
||||
type kv struct {
|
||||
k string
|
||||
v int
|
||||
}
|
||||
var arr []kv
|
||||
for k, v := range comboN {
|
||||
arr = append(arr, kv{k, v})
|
||||
}
|
||||
sort.Slice(arr, func(i, j int) bool { return arr[i].v > arr[j].v })
|
||||
for i, e := range arr {
|
||||
if i >= 15 {
|
||||
break
|
||||
}
|
||||
fmt.Printf(" %-45s %d\n", e.k, e.v)
|
||||
}
|
||||
|
||||
cfg := pattern.DefaultConfig() // 不改任何阈值(MinWriteConfidence=0.5, DraftThreshold=0.85)
|
||||
cfg.MaxRecords = len(dist) // 覆盖全表(DefaultConfig 仅 500;API 端点又硬限 5000)
|
||||
pats := pattern.DetectFromRecords(dist, cfg)
|
||||
fmt.Printf("detected=%d\n", len(pats))
|
||||
// debug: 打印前 3 个 pattern 的 ID/name/conf/src
|
||||
for i, p := range pats {
|
||||
if i >= 3 {
|
||||
break
|
||||
}
|
||||
fmt.Printf(" pat[%d] id=%q name=%q conf=%.2f nsrc=%d\n", i, p.ID, p.Name, p.Confidence, len(p.SourceTasks))
|
||||
}
|
||||
sort.SliceStable(pats, func(i, j int) bool { return len(pats[i].SourceTasks) > len(pats[j].SourceTasks) })
|
||||
for _, p := range pats {
|
||||
fmt.Printf(" %-45s type=%s conf=%.2f nsrc=%d\n", p.Name, p.Type, p.Confidence, len(p.SourceTasks))
|
||||
}
|
||||
|
||||
writeOK, draftOK := []pattern.Pattern{}, []pattern.Pattern{}
|
||||
for _, p := range pats {
|
||||
if p.Confidence >= pattern.MinWriteConfidence {
|
||||
writeOK = append(writeOK, p)
|
||||
}
|
||||
if p.Confidence >= skillgen.DraftThreshold {
|
||||
draftOK = append(draftOK, p)
|
||||
}
|
||||
}
|
||||
fmt.Printf("pass write-gate(>=%.1f)=%d pass L3(>=%.2f)=%d\n",
|
||||
pattern.MinWriteConfidence, len(writeOK), skillgen.DraftThreshold, len(draftOK))
|
||||
|
||||
if !*write {
|
||||
fmt.Println("DRY-RUN: 不写库、不落盘(加 -write 才落库)")
|
||||
return
|
||||
}
|
||||
|
||||
graphPath := os.Getenv("GRAPH_PATH")
|
||||
if graphPath == "" {
|
||||
graphPath = "/var/lib/memoryweave/graph.db"
|
||||
}
|
||||
// L2 写入(conf>=0.5,生产同一 UpsertPatterns 含 IsValidPatternName 写闸)
|
||||
pstore := pattern.NewGraphPatternStore(graphPath)
|
||||
defer pstore.Close()
|
||||
c, u, e := pstore.UpsertPatterns(writeOK)
|
||||
if e != nil {
|
||||
fmt.Println("UpsertPatterns error:", e)
|
||||
}
|
||||
fmt.Printf("L2 upsert: created=%d updated=%d\n", c, u)
|
||||
|
||||
// L3 草稿(仅 conf>=0.85;本地 2B;绝不 promote)
|
||||
if len(draftOK) > 0 {
|
||||
sstore := skillgen.NewStore(graphPath)
|
||||
defer sstore.Close()
|
||||
gen := skillgen.NewLocal2BGenerator()
|
||||
root := *attRoot
|
||||
if root == "" {
|
||||
root = skillgen.DefaultAttachmentsRoot()
|
||||
}
|
||||
for _, p := range draftOK {
|
||||
sk, path, derr := skillgen.DraftSkill(sstore, gen, p, root)
|
||||
if derr != nil {
|
||||
fmt.Printf(" L3 draft FAIL %s: %v\n", p.Name, derr)
|
||||
continue
|
||||
}
|
||||
fmt.Printf(" L3 draft OK %s -> status=%s file=%s\n", p.Name, sk.Status, path)
|
||||
}
|
||||
n, _ := sstore.Count("draft")
|
||||
fmt.Printf("skills draft count=%d\n", n)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user