背景:POST /patterns/detect 被 lancedb_scan 硬限 5000 且 DetectFromRecords 默认 MaxRecords=500 (物理序前500条是'不幸窗口'检出0)→ 已有 pattern-detect-daily cron(limit=500) 空转,graph_nodes type='pattern' 恒 0。本探针走 sidecar IPC lancedb_scan_all(无5000上限)+ 全表 MaxRecords + 诚实阈值: L2 conf>=0.5 写 graph_nodes(type='pattern',ns=daemon-distill),L3 conf>=0.85 才出草稿到 attachments/<pattern>/skill-draft.md([AUTO-DRAFT]标记,status=draft,绝不自动promote)。 -cwrite 才落库,默认 dry-run。 实测(全表45227行/distilled45096):6个tool_combo,仅 hermes+python (conf=1.0,13源) 过 L2/L3 双闸。 写库后 graph_nodes pattern 0→1, skills 0→1(draft);召回 top5 10query 前后逐位一致(零行为变化)。 本地2B(8081)生成草稿落盘。
175 lines
5.3 KiB
Go
175 lines
5.3 KiB
Go
// pattern-fullscan-probe — 全表 L2 检测探针(t_d8830204)
|
||
// 读:经 sidecar IPC lancedb_scan_all 取全量存活行(detect 端点被硬限 5000,物理序前 N 条
|
||
// 是"不幸窗口"→ 检不到;本探针无上限,覆盖全表)。
|
||
// 检:真实 pattern.DetectFromRecords(与生产同一份代码,不改判据、不降阈值)。
|
||
// 写(仅 -write,且仅 conf≥0.85 的诚实过闸项):
|
||
// - L2:pattern.GraphPatternStore.UpsertPatterns → graph_nodes(type='pattern')
|
||
// - L3:skillgen.DraftSkill → skills(status=draft) + attachments/<pattern>/skill-draft.md
|
||
// 红线:L3 草稿绝不自动 promote;不自动进 skills/ 技能库。
|
||
package main
|
||
|
||
import (
|
||
"flag"
|
||
"fmt"
|
||
"os"
|
||
"sort"
|
||
|
||
"github.com/xiaoxue/memoryweave/internal/models"
|
||
"github.com/xiaoxue/memoryweave/internal/pattern"
|
||
"github.com/xiaoxue/memoryweave/internal/skillgen"
|
||
"github.com/xiaoxue/memoryweave/internal/storage"
|
||
)
|
||
|
||
func joinPlus(tools []string) string {
|
||
out := ""
|
||
for i, t := range tools {
|
||
if i > 0 {
|
||
out += "+"
|
||
}
|
||
out += t
|
||
}
|
||
return out
|
||
}
|
||
|
||
func main() {
|
||
write := flag.Bool("write", false, "write L2 patterns (conf>=0.5) + L3 drafts (conf>=0.85); default dry-run")
|
||
attRoot := flag.String("attach", "", "attachments root for L3 drafts (default skillgen.DefaultAttachmentsRoot)")
|
||
flag.Parse()
|
||
|
||
sock := os.Getenv("LANCEDB_SOCKET")
|
||
if sock == "" {
|
||
sock = "/tmp/zhiyi-ipc.sock"
|
||
}
|
||
rc := storage.NewRustLanceDBClient(sock, nil)
|
||
recs, err := rc.ScanAll()
|
||
if err != nil {
|
||
fmt.Println("scan_all error:", err)
|
||
os.Exit(1)
|
||
}
|
||
dist := make([]models.MemoryRecord, 0, len(recs))
|
||
for _, r := range recs {
|
||
if r.Category == "distilled" || r.Category == "" {
|
||
dist = append(dist, r)
|
||
}
|
||
}
|
||
fmt.Printf("scan_all live=%d distilled_scanned=%d\n", len(recs), len(dist))
|
||
|
||
// debug: 抽样看 content 是否非空 + 工具命中
|
||
nonEmpty := 0
|
||
samples := 0
|
||
for _, r := range dist {
|
||
if r.Content != "" {
|
||
nonEmpty++
|
||
}
|
||
if samples < 3 && r.Content != "" {
|
||
fmt.Printf(" sample id=%s len=%d tools=%v content=%.80s\n",
|
||
r.ID, len(r.Content), pattern.ExtractTools(r.Content), r.Content)
|
||
samples++
|
||
}
|
||
}
|
||
fmt.Printf("non_empty_content=%d/%d\n", nonEmpty, len(dist))
|
||
|
||
// debug: ID 是否非空(DetectFromRecords 要求非空 ID 去重计源)
|
||
emptyID := 0
|
||
for _, r := range dist {
|
||
if r.ID == "" {
|
||
emptyID++
|
||
}
|
||
}
|
||
fmt.Printf("empty_ID=%d/%d\n", emptyID, len(dist))
|
||
|
||
// debug: 工具命中统计(复刻 combo 计数)
|
||
toolRecs := 0
|
||
comboN := map[string]int{}
|
||
for _, r := range dist {
|
||
tools := pattern.ExtractTools(r.Content)
|
||
if len(tools) >= 2 {
|
||
toolRecs++
|
||
comboN["tool_combo:"+joinPlus(tools)]++
|
||
}
|
||
}
|
||
fmt.Printf("records_with_>=2_tools=%d\n--- top combos (records, not dedup src) ---\n", toolRecs)
|
||
type kv struct {
|
||
k string
|
||
v int
|
||
}
|
||
var arr []kv
|
||
for k, v := range comboN {
|
||
arr = append(arr, kv{k, v})
|
||
}
|
||
sort.Slice(arr, func(i, j int) bool { return arr[i].v > arr[j].v })
|
||
for i, e := range arr {
|
||
if i >= 15 {
|
||
break
|
||
}
|
||
fmt.Printf(" %-45s %d\n", e.k, e.v)
|
||
}
|
||
|
||
cfg := pattern.DefaultConfig() // 不改任何阈值(MinWriteConfidence=0.5, DraftThreshold=0.85)
|
||
cfg.MaxRecords = len(dist) // 覆盖全表(DefaultConfig 仅 500;API 端点又硬限 5000)
|
||
pats := pattern.DetectFromRecords(dist, cfg)
|
||
fmt.Printf("detected=%d\n", len(pats))
|
||
// debug: 打印前 3 个 pattern 的 ID/name/conf/src
|
||
for i, p := range pats {
|
||
if i >= 3 {
|
||
break
|
||
}
|
||
fmt.Printf(" pat[%d] id=%q name=%q conf=%.2f nsrc=%d\n", i, p.ID, p.Name, p.Confidence, len(p.SourceTasks))
|
||
}
|
||
sort.SliceStable(pats, func(i, j int) bool { return len(pats[i].SourceTasks) > len(pats[j].SourceTasks) })
|
||
for _, p := range pats {
|
||
fmt.Printf(" %-45s type=%s conf=%.2f nsrc=%d\n", p.Name, p.Type, p.Confidence, len(p.SourceTasks))
|
||
}
|
||
|
||
writeOK, draftOK := []pattern.Pattern{}, []pattern.Pattern{}
|
||
for _, p := range pats {
|
||
if p.Confidence >= pattern.MinWriteConfidence {
|
||
writeOK = append(writeOK, p)
|
||
}
|
||
if p.Confidence >= skillgen.DraftThreshold {
|
||
draftOK = append(draftOK, p)
|
||
}
|
||
}
|
||
fmt.Printf("pass write-gate(>=%.1f)=%d pass L3(>=%.2f)=%d\n",
|
||
pattern.MinWriteConfidence, len(writeOK), skillgen.DraftThreshold, len(draftOK))
|
||
|
||
if !*write {
|
||
fmt.Println("DRY-RUN: 不写库、不落盘(加 -write 才落库)")
|
||
return
|
||
}
|
||
|
||
graphPath := os.Getenv("GRAPH_PATH")
|
||
if graphPath == "" {
|
||
graphPath = "/var/lib/memoryweave/graph.db"
|
||
}
|
||
// L2 写入(conf>=0.5,生产同一 UpsertPatterns 含 IsValidPatternName 写闸)
|
||
pstore := pattern.NewGraphPatternStore(graphPath)
|
||
defer pstore.Close()
|
||
c, u, e := pstore.UpsertPatterns(writeOK)
|
||
if e != nil {
|
||
fmt.Println("UpsertPatterns error:", e)
|
||
}
|
||
fmt.Printf("L2 upsert: created=%d updated=%d\n", c, u)
|
||
|
||
// L3 草稿(仅 conf>=0.85;本地 2B;绝不 promote)
|
||
if len(draftOK) > 0 {
|
||
sstore := skillgen.NewStore(graphPath)
|
||
defer sstore.Close()
|
||
gen := skillgen.NewLocal2BGenerator()
|
||
root := *attRoot
|
||
if root == "" {
|
||
root = skillgen.DefaultAttachmentsRoot()
|
||
}
|
||
for _, p := range draftOK {
|
||
sk, path, derr := skillgen.DraftSkill(sstore, gen, p, root)
|
||
if derr != nil {
|
||
fmt.Printf(" L3 draft FAIL %s: %v\n", p.Name, derr)
|
||
continue
|
||
}
|
||
fmt.Printf(" L3 draft OK %s -> status=%s file=%s\n", p.Name, sk.Status, path)
|
||
}
|
||
n, _ := sstore.Count("draft")
|
||
fmt.Printf("skills draft count=%d\n", n)
|
||
}
|
||
}
|