这批是 10-26~27 已在生产二进制(zhiyid-protectrails-20261027)里运行、 但一直留在工作树未提交的改动。收口原因:三件改造要开分支,工作树脏 = 分支基准与回滚点都不存在。 主要内容: - api/routes/contract.go + trace.go:召回响应契约字段(trace_id/status/scope/ continuation/channel)与 /api/v1/trace 回溯端点 - governance/entity_quality.go:实体质量闸 - storage/trace.go + recall.go/recall_write_buffer.go:召回写入缓冲与轨迹 - 测试:contract/recall_contract_fields/recall_lifecycle/stale_lifecycle/ entity_quality/trace 同时修一个 flaky:recall_lifecycle_test 原本硬编码「今天 11:00 已过」, 凌晨跑就会把「已失效」fixture 当成仍有效。改成相对时间(过期=now-1h), 生命周期判据与运行时刻无关。修完 go test ./internal/api/routes ./internal/storage 全绿。
452 lines
12 KiB
Go
452 lines
12 KiB
Go
// 织忆 MemoryWeave — L2 模式检测引擎(t_d32e5f8a)
|
||
//
|
||
// 背景:MemOS 血统的 L2「跨任务重复模式」检测。原实现活在 Python daemon 的
|
||
// main_loop(30s tick),2026-09-24 为 state-db 写安全整体退役后 L2 pattern 蒸馏
|
||
// 停摆(graph_nodes type='pattern' 停为 0 行;旧 Python daemon 残留的 85 行是
|
||
// type='pattern_stat',id 全 NULL、语义为 L1 命名空间统计快照,非 L2 pattern / 2026-09-18)。本包把 L1→L2 蒸馏
|
||
// 复活成**短命单发**(modernc sqlite 直连 graph.db,开/关不占长连接,不碰 state.db),
|
||
// 并按设计 §4.1 叠加两条检测规则:
|
||
// - 工具组合频率:同一 tool 组合出现 ≥3 次 → tool_combo pattern
|
||
// (confidence = min(freq*0.1, 1.0))
|
||
// - 相似解决路径聚类:cosine ≥0.85 成簇且簇内 ≥3 条 → workflow pattern
|
||
//
|
||
// 存储:复用现有 graph_nodes(type='pattern'),**不另开新表**(避免与现存图谱查询割裂;
|
||
// 表内另有 85 行 type='pattern_stat' 属旧 daemon 统计快照,与 L2 pattern 无关)。
|
||
// namespace='daemon-distill' 沿用旧 daemon 约定。
|
||
package pattern
|
||
|
||
import (
|
||
"fmt"
|
||
"math"
|
||
"sort"
|
||
"strings"
|
||
|
||
"github.com/xiaoxue/memoryweave/internal/models"
|
||
)
|
||
|
||
// Pattern 一个被检测出的 L2 模式。
|
||
type Pattern struct {
|
||
ID string
|
||
Type string // "tool_combo" | "workflow"
|
||
Name string // 稳定命名键(同组合去重)
|
||
Data string // tool_combo=排序后工具串;workflow=共同步骤/代表路径摘要
|
||
Confidence float64
|
||
SourceTasks []string // 支撑该 pattern 的记忆 ID(设计 source_tasks)
|
||
Tools []string // tool_combo 专用
|
||
}
|
||
|
||
// Config 检测参数(设计 §4.1)。
|
||
type Config struct {
|
||
MinComboFreq int // 工具组合出现 ≥ N 次才产 pattern(默认 3)
|
||
ClusterThresh float64 // 相似路径聚类 cosine 阈值(默认 0.85)
|
||
MinClusterSize int // 簇内 ≥ N 条才产 workflow pattern(默认 3)
|
||
MaxRecords int // 单次扫描参与检测的记忆上限(防 O(n²) 爆炸,默认 500)
|
||
TopKPatternList int // 召回/列出取前 N(默认 10)
|
||
}
|
||
|
||
// DefaultConfig 设计参数。
|
||
func DefaultConfig() Config {
|
||
return Config{
|
||
MinComboFreq: 3,
|
||
ClusterThresh: 0.85,
|
||
MinClusterSize: 3,
|
||
MaxRecords: 500,
|
||
TopKPatternList: 10,
|
||
}
|
||
}
|
||
|
||
// knownTools 常见工具/CLI/命令名(小写)。工具组合频率分析从这里识别 + 兜底正则。
|
||
var knownTools = map[string]bool{
|
||
"jq": true, "yq": true, "git": true, "grep": true, "sed": true, "awk": true,
|
||
"curl": true, "wget": true, "tar": true, "zip": true, "unzip": true,
|
||
"docker": true, "kubectl": true, "helm": true, "npm": true, "pip": true,
|
||
"go": true, "python": true, "python3": true, "node": true, "pnpm": true,
|
||
"make": true, "cmake": true, "gcc": true, "gcc-12": true, "clang": true,
|
||
"sqlite3": true, "psql": true, "mysql": true, "redis-cli": true,
|
||
"ssh": true, "scp": true, "rsync": true, "systemctl": true, "journalctl": true,
|
||
"systemd-run": true, "crontab": true, "find": true, "rg": true, "ripgrep": true,
|
||
"fd": true, "fzf": true, "gh": true, "gcloud": true, "aws": true, "jq2": true,
|
||
"hermes": true, "zhiyi": true, "obsidian": true, "esbuild": true, "ts": true,
|
||
"tsc": true, "cargo": true, "rustc": true, "swift": true, "brew": true,
|
||
}
|
||
|
||
// DetectFromRecords 纯检测核心:从 L1 记忆列表发现重复模式。
|
||
// 不读 LanceDB / 不写库,只做「记录 → pattern 候选」,便于确定性单测。
|
||
// 返回合并去重后的 pattern 列表(同 Name 合并 source_tasks、取更高 confidence)。
|
||
func DetectFromRecords(records []models.MemoryRecord, cfg Config) []Pattern {
|
||
if cfg.MinComboFreq <= 0 {
|
||
cfg.MinComboFreq = 3
|
||
}
|
||
if cfg.MinClusterSize <= 0 {
|
||
cfg.MinClusterSize = 3
|
||
}
|
||
if cfg.ClusterThresh <= 0 {
|
||
cfg.ClusterThresh = 0.85
|
||
}
|
||
lim := cfg.MaxRecords
|
||
if lim <= 0 {
|
||
lim = 500
|
||
}
|
||
if len(records) > lim {
|
||
records = records[:lim]
|
||
}
|
||
|
||
combined := make(map[string]*Pattern) // Name → merged pattern
|
||
order := []string{}
|
||
|
||
merge := func(p Pattern) {
|
||
if ex, ok := combined[p.Name]; ok {
|
||
ex.SourceTasks = append(ex.SourceTasks, p.SourceTasks...)
|
||
if p.Confidence > ex.Confidence {
|
||
ex.Confidence = p.Confidence
|
||
}
|
||
return
|
||
}
|
||
combined[p.Name] = &p
|
||
order = append(order, p.Name)
|
||
}
|
||
|
||
// 1. 工具组合频率分析
|
||
comboCount := map[string]*struct {
|
||
tools []string
|
||
tasks []string
|
||
}{}
|
||
for _, rec := range records {
|
||
tools := ExtractTools(rec.Content)
|
||
if len(tools) == 0 {
|
||
continue
|
||
}
|
||
tools = dedupeSorted(tools)
|
||
key := "tool_combo:" + strings.Join(tools, "+")
|
||
c := comboCount[key]
|
||
if c == nil {
|
||
c = &struct {
|
||
tools []string
|
||
tasks []string
|
||
}{tools: tools}
|
||
comboCount[key] = c
|
||
}
|
||
c.tasks = append(c.tasks, rec.ID)
|
||
}
|
||
for name, c := range comboCount {
|
||
if len(c.tasks) < cfg.MinComboFreq {
|
||
continue
|
||
}
|
||
merge(Pattern{
|
||
Type: "tool_combo",
|
||
Name: name,
|
||
Data: strings.Join(c.tools, "+"),
|
||
Tools: c.tools,
|
||
Confidence: confidenceFromFreq(len(c.tasks)),
|
||
SourceTasks: c.tasks,
|
||
})
|
||
}
|
||
|
||
// 2. 相似解决路径聚类(cosine ≥ 阈值 成簇,簇内 ≥ N 条)
|
||
paths := clusterPaths(records, cfg.ClusterThresh, cfg.MinClusterSize)
|
||
for _, cluster := range paths {
|
||
// 代表路径 = 簇内最长 content(共同步骤近似)
|
||
rep := ""
|
||
var ids []string
|
||
for _, r := range cluster {
|
||
ids = append(ids, r.ID)
|
||
if len(r.Content) > len(rep) {
|
||
rep = r.Content
|
||
}
|
||
}
|
||
conf := clusterConfidence(cluster) // 平均两两 cosine,封顶 1.0
|
||
if conf > 1.0 {
|
||
conf = 1.0
|
||
}
|
||
// 簇名带代表内容哈希前 8 位 —— 同 size 不同内容簇不碰撞(否则 cluster-3 撞簇)
|
||
name := fmt.Sprintf("workflow:cluster-%d-%s", len(cluster), contentFingerprint(rep))
|
||
merge(Pattern{
|
||
Type: "workflow",
|
||
Name: name,
|
||
Data: truncateForPattern(rep, 120),
|
||
Confidence: conf,
|
||
SourceTasks: ids,
|
||
})
|
||
}
|
||
|
||
out := make([]Pattern, 0, len(order))
|
||
for _, n := range order {
|
||
p := *combined[n]
|
||
p.SourceTasks = dedupeStrings(p.SourceTasks)
|
||
if p.Type == "tool_combo" {
|
||
p.ID = "pat_" + p.Name
|
||
} else {
|
||
p.ID = "pat_" + p.Name
|
||
}
|
||
out = append(out, p)
|
||
}
|
||
sort.SliceStable(out, func(i, j int) bool {
|
||
if out[i].Type != out[j].Type {
|
||
return out[i].Type < out[j].Type
|
||
}
|
||
return out[i].Name < out[j].Name
|
||
})
|
||
return out
|
||
}
|
||
|
||
// confidenceFromFreq 设计公式:min(freq*0.1, 1.0)。
|
||
func confidenceFromFreq(freq int) float64 {
|
||
c := float64(freq) * 0.1
|
||
if c > 1.0 {
|
||
c = 1.0
|
||
}
|
||
return c
|
||
}
|
||
|
||
// clusterPaths 贪心成簇:为每条未分配记忆开一个簇,把与其 cosine≥阈值 的其余未分配
|
||
// 记忆并入;最终只保留 size≥minSize 的簇。向量缺失时用记录自带 Vector,全缺则退到
|
||
// 文本 bigram Jaccard(确定性,不依赖网络)。
|
||
func clusterPaths(records []models.MemoryRecord, thresh float64, minSize int) [][]models.MemoryRecord {
|
||
n := len(records)
|
||
if n < minSize {
|
||
return nil
|
||
}
|
||
type sig struct {
|
||
vec []float32
|
||
big map[string]bool
|
||
text string
|
||
}
|
||
sigs := make([]sig, n)
|
||
for i, r := range records {
|
||
sigs[i] = sig{vec: r.Vector, text: r.Content}
|
||
}
|
||
sim := func(a, b int) float64 {
|
||
sa, sb := sigs[a], sigs[b]
|
||
if len(sa.vec) > 0 && len(sb.vec) > 0 {
|
||
return cosineF32(sa.vec, sb.vec)
|
||
}
|
||
if len(sa.vec) == 0 && len(sb.vec) == 0 {
|
||
return bigramJaccard(sa.text, sb.text)
|
||
}
|
||
return 0
|
||
}
|
||
assigned := make([]bool, n)
|
||
var clusters [][]models.MemoryRecord
|
||
for i := 0; i < n; i++ {
|
||
if assigned[i] {
|
||
continue
|
||
}
|
||
cluster := []models.MemoryRecord{records[i]}
|
||
assigned[i] = true
|
||
for j := i + 1; j < n; j++ {
|
||
if assigned[j] {
|
||
continue
|
||
}
|
||
if sim(i, j) >= thresh {
|
||
cluster = append(cluster, records[j])
|
||
assigned[j] = true
|
||
}
|
||
}
|
||
if len(cluster) >= minSize {
|
||
clusters = append(clusters, cluster)
|
||
}
|
||
}
|
||
return clusters
|
||
}
|
||
|
||
// clusterConfidence 簇内平均两两相似度(cosine 或 Jaccard)。
|
||
func clusterConfidence(cluster []models.MemoryRecord) float64 {
|
||
n := len(cluster)
|
||
if n < 2 {
|
||
return 1.0
|
||
}
|
||
sum := 0.0
|
||
pairs := 0
|
||
for i := 0; i < n; i++ {
|
||
for j := i + 1; j < n; j++ {
|
||
var s float64
|
||
a, b := cluster[i], cluster[j]
|
||
if len(a.Vector) > 0 && len(b.Vector) > 0 {
|
||
s = cosineF32(a.Vector, b.Vector)
|
||
} else {
|
||
s = bigramJaccard(a.Content, b.Content)
|
||
}
|
||
sum += s
|
||
pairs++
|
||
}
|
||
}
|
||
if pairs == 0 {
|
||
return 1.0
|
||
}
|
||
return sum / float64(pairs)
|
||
}
|
||
|
||
// ExtractTools 从记忆内容提取工具/命令名。识别 knownTools + 兜底 ascii token。
|
||
// 确定性:结果排序去重。
|
||
func ExtractTools(content string) []string {
|
||
found := map[string]bool{}
|
||
lower := strings.ToLower(content)
|
||
for t := range knownTools {
|
||
// 词边界匹配(避免 "go" 命中 "google")
|
||
if tokenPresent(lower, t) {
|
||
found[t] = true
|
||
}
|
||
}
|
||
// 兜底:捕获已知列表外的 ascii 命令式 token(2~14 字符,含 - 的命令风格)
|
||
for _, tok := range regexTokenExtract(content) {
|
||
lt := strings.ToLower(tok)
|
||
if len(lt) < 2 {
|
||
continue
|
||
}
|
||
// 排除明显非工具的常见词(纯小写短词且不在 knownTools 的放行——保持宽松,
|
||
// 让工具组合信号进来;过噪由 MinComboFreq 过滤)
|
||
found[lt] = true
|
||
}
|
||
out := make([]string, 0, len(found))
|
||
for t := range found {
|
||
out = append(out, t)
|
||
}
|
||
sort.Strings(out)
|
||
return out
|
||
}
|
||
|
||
// tokenPresent 检查 lower 文本中 t 是否作为独立 token 出现(词边界,避免 "go" 命中 "google")。
|
||
func tokenPresent(lower, t string) bool {
|
||
if t == "" {
|
||
return false
|
||
}
|
||
// 切词精确匹配
|
||
for _, w := range strings.Fields(lower) {
|
||
// 剥掉包裹的引号/逗号/括号
|
||
w = strings.Trim(w, `"',()[]|;:`)
|
||
if w == t {
|
||
return true
|
||
}
|
||
}
|
||
// 空格/首尾边界
|
||
if strings.Contains(lower, " "+t+" ") || strings.HasPrefix(lower, t+" ") || strings.HasSuffix(lower, " "+t) {
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
// regexTokenExtract 轻量 token 抽取(无正则依赖,按字符切):
|
||
// 返回形如命令/token 的 ascii 串(长度 ≥2,含字母,可能带数字/_.-)。
|
||
func regexTokenExtract(content string) []string {
|
||
var out []string
|
||
var cur []rune
|
||
flush := func() {
|
||
if len(cur) >= 2 {
|
||
out = append(out, string(cur))
|
||
}
|
||
cur = nil
|
||
}
|
||
for _, r := range content {
|
||
isTok := (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') ||
|
||
(r >= '0' && r <= '9') || r == '_' || r == '-' || r == '.'
|
||
if isTok {
|
||
cur = append(cur, r)
|
||
} else {
|
||
flush()
|
||
}
|
||
}
|
||
flush()
|
||
return out
|
||
}
|
||
|
||
func dedupeSorted(in []string) []string {
|
||
seen := map[string]bool{}
|
||
out := make([]string, 0, len(in))
|
||
for _, s := range in {
|
||
if !seen[s] {
|
||
seen[s] = true
|
||
out = append(out, s)
|
||
}
|
||
}
|
||
sort.Strings(out)
|
||
return out
|
||
}
|
||
|
||
func dedupeStrings(in []string) []string {
|
||
seen := map[string]bool{}
|
||
out := make([]string, 0, len(in))
|
||
for _, s := range in {
|
||
if !seen[s] {
|
||
seen[s] = true
|
||
out = append(out, s)
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
func cosineF32(a, b []float32) float64 {
|
||
if len(a) == 0 || len(b) == 0 || len(a) != len(b) {
|
||
return 0
|
||
}
|
||
var dot, na, nb float64
|
||
for i := range a {
|
||
x, y := float64(a[i]), float64(b[i])
|
||
dot += x * y
|
||
na += x * x
|
||
nb += y * y
|
||
}
|
||
if na == 0 || nb == 0 {
|
||
return 0
|
||
}
|
||
c := dot / (float64(math.Sqrt(na)) * float64(math.Sqrt(nb)))
|
||
if c > 1.0 {
|
||
c = 1.0
|
||
}
|
||
if c < -1.0 {
|
||
c = -1.0
|
||
}
|
||
return c
|
||
}
|
||
|
||
// bigramJaccard 字符 bigram Jaccard 相似度(无向量时的确定性回退)。
|
||
func bigramJaccard(a, b string) float64 {
|
||
bg := func(s string) map[string]bool {
|
||
m := make(map[string]bool)
|
||
r := []rune(s)
|
||
for i := 0; i+1 < len(r); i++ {
|
||
m[string(r[i:i+2])] = true
|
||
}
|
||
return m
|
||
}
|
||
ma, mb := bg(a), bg(b)
|
||
if len(ma) == 0 && len(mb) == 0 {
|
||
return 1.0
|
||
}
|
||
if len(ma) == 0 || len(mb) == 0 {
|
||
return 0.0
|
||
}
|
||
inter := 0
|
||
for k := range ma {
|
||
if mb[k] {
|
||
inter++
|
||
}
|
||
}
|
||
uni := len(ma) + len(mb) - inter
|
||
if uni == 0 {
|
||
return 0
|
||
}
|
||
return float64(inter) / float64(uni)
|
||
}
|
||
|
||
// contentFingerprint 取 content 稳定短指纹(FNV 前 8 hex 位)——同 size 不同内容簇去重。
|
||
func contentFingerprint(s string) string {
|
||
if s == "" {
|
||
return "empty"
|
||
}
|
||
// FNV-1a 32
|
||
h := uint32(2166136261)
|
||
for i := 0; i < len(s); i++ {
|
||
h ^= uint32(s[i])
|
||
h *= 16777619
|
||
}
|
||
return fmt.Sprintf("%08x", h)
|
||
}
|
||
|
||
func truncateForPattern(s string, n int) string {
|
||
r := []rune(s)
|
||
if len(r) <= n {
|
||
return s
|
||
}
|
||
return string(r[:n]) + "…"
|
||
}
|