Files
memoryweave/go/internal/selfoptimize/selfoptimize.go
T
小唯 faf71cbe99 fix(tuning): 调参接线消费方 + 阻尼 + 持久化(治「旋钮没接到任何东西」)
实测三真相(2026-10-14):
① 无消费者 —— grep Tuner. 全仓只有 3 处路由注册,没有任何代码读
   Diversity/GapThreshold/DecayRate/DistillInterval ⇒ 调参改了数字、行为零变化。
② 不幂等(棘轮)—— 同一判据每调一次推一步:连跑两次 decay 0.015→0.018→0.0216。
③ 不持久化 —— 包级内存变量,重启即回默认。

修法:
- 接线:storage.SetDefaultMMRDiversity / selfoptimize.SetGapThreshold(消费者持值、调参器推送,
  避免 storage←routes 反向依赖)。初值对齐消费方真实默认(Diversity 0.5→0.3)⇒ 当天行为不变。
- 阻尼:同一参数 6h 冷却。
- 持久化:/var/lib/memoryweave/tuning-state.json(env ZHIYI_TUNING_STATE 可覆盖),启动 load+接线。
- 删规则3(废弃率→提高衰减率):方向危险 —— NewForgetter 注释记载
  「0.015→0.03 曾致误删(36天库龄下 0-recall 长记忆全被清)」。
- 删规则5(蒸馏损失→缩短蒸馏间隔):无正确消费方 —— engine.go 的 batchTimeout 是
  「队列冻结兜底 ticker」(安全网),不是蒸馏节奏。

测试 4 个:哨兵/少样本拦截、冷却、推送、持久化(含「健康数据仍能调」反向验证)。
2026-10-14 09:50:09 +08:00

923 lines
27 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// 织忆 MemoryWeave — 自优化引擎
package selfoptimize
import (
"encoding/json"
"math"
"os"
"strconv"
"sync"
"sync/atomic"
"time"
"github.com/xiaoxue/memoryweave/internal/storage"
)
// Redis persistence key
const dashboardRedisKey = "zhiyi:dashboard"
// dashboardLocalPath 本地 JSON 落盘点 —— Redis 不可用时的兜底(2026-09-27 加)
//
// 背景:本机无 Redis(无 redis-cli/redis-server、6379 无监听、单元无 REDIS_ADDR/URL)
// ⇒ storage.GetRedisClient() 返回 nil ⇒ redisPersistence 恒为 nil
// ⇒ 原 persist() 第一行就 return:7 项指标只活在进程内,**重启即归零**,
//
// 而 tuning / executor / distributed 都在读它做自优化决策。
//
// 可用 ZHIYI_DASHBOARD_PATH 覆盖(测试用)。
func dashboardLocalPath() string {
if p := os.Getenv("ZHIYI_DASHBOARD_PATH"); p != "" {
return p
}
return "/var/lib/memoryweave/dashboard.json"
}
// dashboardState 可序列化快照 —— Dashboard 内含 sync.RWMutex,不能直接 Marshal/整块复制
type dashboardState struct {
UsefulCount int `json:"useful_count"`
NotUsefulCount int `json:"not_useful_count"`
TotalRecalls int `json:"total_recalls"`
HitCount int `json:"hit_count"`
ClosedGaps int `json:"closed_gaps"`
TotalGaps int `json:"total_gaps"`
CascadeFixedTotal int `json:"cascade_fixed_total"`
TotalFixes int `json:"total_fixes"`
DeprecatedToday int `json:"deprecated_today"`
DeprecatedDate string `json:"deprecated_date"` // deprecated_today 的归属日期(跨天归零用)
DistillLossSum float64 `json:"distill_loss_sum"`
DistillLossCount int `json:"distill_loss_count"`
DistillSuccessCount int `json:"distill_success_count"`
AutoResolvedConflicts int `json:"auto_resolved_conflicts"`
TotalConflicts int `json:"total_conflicts"`
}
func todayStr() string { return time.Now().Format("2006-01-02") }
// redisPersistence 是否开启了 Redis 持久化(由 server.go 在启动时设置)
var redisPersistence *storage.RedisClient
// EnableRedisPersistence 启动 Dashboard 持久化(Redis 优先;Redis 不可用则转本地 JSON)
func EnableRedisPersistence() {
rc := storage.GetRedisClient()
if rc != nil {
redisPersistence = rc
_ = loadDashboard()
return
}
_ = loadDashboardLocal()
}
// loadDashboardLocal 从本地 JSON 恢复指标;文件不存在/损坏 → 静默保持零值(= 老行为)
func loadDashboardLocal() error {
b, err := os.ReadFile(dashboardLocalPath())
if err != nil {
return err
}
var st dashboardState
if err := json.Unmarshal(b, &st); err != nil {
return err
}
Dash.mu.Lock()
defer Dash.mu.Unlock()
Dash.UsefulCount = st.UsefulCount
Dash.NotUsefulCount = st.NotUsefulCount
Dash.TotalRecalls = st.TotalRecalls
Dash.HitCount = st.HitCount
Dash.ClosedGaps = st.ClosedGaps
Dash.TotalGaps = st.TotalGaps
Dash.CascadeFixedTotal = st.CascadeFixedTotal
Dash.TotalFixes = st.TotalFixes
Dash.DeprecatedToday = st.DeprecatedToday
if st.DeprecatedDate != todayStr() { // "今日"计数跨天归零(进程跨零点不重置,下次加载时纠正)
Dash.DeprecatedToday = 0
}
Dash.DistillLossSum = st.DistillLossSum
Dash.DistillLossCount = st.DistillLossCount
Dash.DistillSuccessCount = st.DistillSuccessCount
Dash.AutoResolvedConflicts = st.AutoResolvedConflicts
Dash.TotalConflicts = st.TotalConflicts
return nil
}
// persistLocalLocked 原子写本地 JSON(best-effort:任何失败都静默 → 行为退回"仅内存")
// 注意:调用方必须已持有 d.mu 锁
func (d *Dashboard) persistLocalLocked() {
st := dashboardState{
UsefulCount: d.UsefulCount,
NotUsefulCount: d.NotUsefulCount,
TotalRecalls: d.TotalRecalls,
HitCount: d.HitCount,
ClosedGaps: d.ClosedGaps,
TotalGaps: d.TotalGaps,
CascadeFixedTotal: d.CascadeFixedTotal,
TotalFixes: d.TotalFixes,
DeprecatedToday: d.DeprecatedToday,
DeprecatedDate: todayStr(),
DistillLossSum: d.DistillLossSum,
DistillLossCount: d.DistillLossCount,
DistillSuccessCount: d.DistillSuccessCount,
AutoResolvedConflicts: d.AutoResolvedConflicts,
TotalConflicts: d.TotalConflicts,
}
b, err := json.Marshal(st)
if err != nil {
return
}
p := dashboardLocalPath()
tmp := p + ".tmp"
if err := os.WriteFile(tmp, b, 0o600); err != nil {
return
}
_ = os.Rename(tmp, p)
}
// persist 持久化 Dashboard:Redis 可用→写 Redis HASH;否则→写本地 JSON
// 注意:调用方必须已持有 d.mu 锁(RLock 或 Lock)
func (d *Dashboard) persist() {
if redisPersistence == nil {
d.persistLocalLocked()
return
}
_ = redisPersistence.HSet(dashboardRedisKey, "useful_count", strconv.Itoa(d.UsefulCount))
_ = redisPersistence.HSet(dashboardRedisKey, "not_useful_count", strconv.Itoa(d.NotUsefulCount))
_ = redisPersistence.HSet(dashboardRedisKey, "total_recalls", strconv.Itoa(d.TotalRecalls))
_ = redisPersistence.HSet(dashboardRedisKey, "hit_count", strconv.Itoa(d.HitCount))
_ = redisPersistence.HSet(dashboardRedisKey, "closed_gaps", strconv.Itoa(d.ClosedGaps))
_ = redisPersistence.HSet(dashboardRedisKey, "total_gaps", strconv.Itoa(d.TotalGaps))
_ = redisPersistence.HSet(dashboardRedisKey, "cascade_fixed_total", strconv.Itoa(d.CascadeFixedTotal))
_ = redisPersistence.HSet(dashboardRedisKey, "total_fixes", strconv.Itoa(d.TotalFixes))
_ = redisPersistence.HSet(dashboardRedisKey, "deprecated_today", strconv.Itoa(d.DeprecatedToday))
_ = redisPersistence.HSet(dashboardRedisKey, "distill_loss_sum", strconv.FormatFloat(d.DistillLossSum, 'f', 4, 64))
_ = redisPersistence.HSet(dashboardRedisKey, "distill_loss_count", strconv.Itoa(d.DistillLossCount))
_ = redisPersistence.HSet(dashboardRedisKey, "distill_success_count", strconv.Itoa(d.DistillSuccessCount))
_ = redisPersistence.HSet(dashboardRedisKey, "auto_resolved_conflicts", strconv.Itoa(d.AutoResolvedConflicts))
_ = redisPersistence.HSet(dashboardRedisKey, "total_conflicts", strconv.Itoa(d.TotalConflicts))
_ = redisPersistence.Expire(dashboardRedisKey, 90*24*time.Hour) // 90 天 TTL
}
// loadDashboard 从 Redis 加载 Dashboard 数据
func loadDashboard() error {
if redisPersistence == nil {
return nil
}
data, err := redisPersistence.HGetAll(dashboardRedisKey)
if err != nil || len(data) == 0 {
return err
}
Dash.mu.Lock()
defer Dash.mu.Unlock()
if v, err := strconv.Atoi(data["useful_count"]); err == nil {
Dash.UsefulCount = v
}
if v, err := strconv.Atoi(data["not_useful_count"]); err == nil {
Dash.NotUsefulCount = v
}
if v, err := strconv.Atoi(data["total_recalls"]); err == nil {
Dash.TotalRecalls = v
}
if v, err := strconv.Atoi(data["hit_count"]); err == nil {
Dash.HitCount = v
}
if v, err := strconv.Atoi(data["closed_gaps"]); err == nil {
Dash.ClosedGaps = v
}
if v, err := strconv.Atoi(data["total_gaps"]); err == nil {
Dash.TotalGaps = v
}
if v, err := strconv.Atoi(data["cascade_fixed_total"]); err == nil {
Dash.CascadeFixedTotal = v
}
if v, err := strconv.Atoi(data["total_fixes"]); err == nil {
Dash.TotalFixes = v
}
if v, err := strconv.Atoi(data["deprecated_today"]); err == nil {
Dash.DeprecatedToday = v
}
if v, err := strconv.ParseFloat(data["distill_loss_sum"], 64); err == nil {
Dash.DistillLossSum = v
}
if v, err := strconv.Atoi(data["distill_loss_count"]); err == nil {
Dash.DistillLossCount = v
}
if v, err := strconv.Atoi(data["distill_success_count"]); err == nil {
Dash.DistillSuccessCount = v
}
if v, err := strconv.Atoi(data["auto_resolved_conflicts"]); err == nil {
Dash.AutoResolvedConflicts = v
}
if v, err := strconv.Atoi(data["total_conflicts"]); err == nil {
Dash.TotalConflicts = v
}
return nil
}
// ─── 自优化仪表盘 ────────────────────────────────────────
type Dashboard struct {
mu sync.RWMutex
UsefulCount int `json:"useful_count"`
NotUsefulCount int `json:"not_useful_count"`
TotalRecalls int `json:"total_recalls"`
HitCount int `json:"hit_count"`
ClosedGaps int `json:"closed_gaps"`
TotalGaps int `json:"total_gaps"`
CascadeFixedTotal int `json:"cascade_fixed_total"`
TotalFixes int `json:"total_fixes"`
DeprecatedToday int `json:"deprecated_today"`
DistillLossSum float64 `json:"distill_loss_sum"`
DistillLossCount int `json:"distill_loss_count"`
DistillSuccessCount int `json:"distill_success_count"`
AutoResolvedConflicts int `json:"auto_resolved_conflicts"`
TotalConflicts int `json:"total_conflicts"`
}
var Dash = &Dashboard{}
// metricNA 是「无数据」哨兵值。
//
// 2026-10-14(牧尘批准):比率型指标在**分母退化**时(样本数为 0,或负样本通道
// 恒空)此前返回 1 —— 被下游读成「100% 好用/100% 命中」。那不是满分,是失明:
// NotUseful=0 只说明「没人说过不好」,不说明「全都好」;
// HitCount/TotalRecalls 的分子来自 core.go 的 `len(results) > 0`,
// 量的是「召回有没有返回东西」,不是「召回对不对」。
//
// 现在统一返回 -1:真实比率必落在 [0,1],-1 不可能被误读成任何一档成绩。
// 消费方规则(配套键,同 map 返回):
//
// <name>_valid = 1 表示该比率可解读;0 表示值必为 -1、**禁止**当成绩用
// <name>_samples = 该比率的分母样本数(0 时即退化)
//
// 已有消费方均按 key 存在性取值(SyncFromDashboard / 各 JSON 端点),
// 本改动**只增键、不改键名**,故不破坏任何调用方。
const metricNA = -1.0
// rateOrNA 分母 > 0 才算比率(保留两位),否则返回 metricNA。
func rateOrNA(num, den int) float64 {
if den <= 0 {
return metricNA
}
return math.Round(float64(num)/float64(den)*100) / 100
}
// b2f 布尔转 0/1,用于 *_valid 键。
func b2f(b bool) float64 {
if b {
return 1
}
return 0
}
// Metrics 返回核心指标。
//
// 比率项在分母退化时返回 metricNA(-1) 而非 1;每项比率都配 *_valid / *_samples。
// 计数项(deprecated_per_day)与均值项(avg_distill_loss)同规则。
func (d *Dashboard) Metrics() map[string]float64 {
d.mu.RLock()
defer d.mu.RUnlock()
// 负样本通道为空 ⇒ 无从判断好坏(详见 metricNA 注释)。
usefulRate := metricNA
if d.NotUsefulCount > 0 {
usefulRate = rateOrNA(d.UsefulCount, d.UsefulCount+d.NotUsefulCount)
}
hitRate := rateOrNA(d.HitCount, d.TotalRecalls)
gapRate := rateOrNA(d.ClosedGaps, d.TotalGaps)
cascadeRate := rateOrNA(d.CascadeFixedTotal, d.TotalFixes)
autoRate := rateOrNA(d.AutoResolvedConflicts, d.TotalConflicts)
// 均值项:零样本的均值同样是退化,按同一规则返回 metricNA。
avgLoss := metricNA
if d.DistillLossCount > 0 {
avgLoss = math.Round(d.DistillLossSum/float64(d.DistillLossCount)*100) / 100
}
return map[string]float64{
"recall_usefulness_rate": usefulRate,
"recall_usefulness_samples": float64(d.UsefulCount + d.NotUsefulCount),
"recall_usefulness_negative": float64(d.NotUsefulCount),
"recall_usefulness_positive": float64(d.UsefulCount),
"recall_usefulness_valid": b2f(d.NotUsefulCount > 0),
"recall_hit_rate": hitRate,
"recall_hit_samples": float64(d.TotalRecalls),
"recall_hit_valid": b2f(d.TotalRecalls > 0),
"gap_closure_rate": gapRate,
"gap_closure_samples": float64(d.TotalGaps),
"gap_closure_valid": b2f(d.TotalGaps > 0),
"cascade_fix_rate": cascadeRate,
"cascade_fix_samples": float64(d.TotalFixes),
"cascade_fix_valid": b2f(d.TotalFixes > 0),
"auto_resolve_rate": autoRate,
"auto_resolve_samples": float64(d.TotalConflicts),
"auto_resolve_valid": b2f(d.TotalConflicts > 0),
"deprecated_per_day": float64(d.DeprecatedToday),
"distill_success_count": float64(d.DistillSuccessCount),
"avg_distill_loss": avgLoss,
"avg_distill_loss_samples": float64(d.DistillLossCount),
"avg_distill_loss_valid": b2f(d.DistillLossCount > 0),
}
}
func (d *Dashboard) RecordRecall(hit bool) {
d.mu.Lock()
defer d.mu.Unlock()
d.TotalRecalls++
if hit {
d.HitCount++
}
d.persist()
}
func (d *Dashboard) RecordFeedback(useful bool) {
d.mu.Lock()
defer d.mu.Unlock()
if useful {
d.UsefulCount++
} else {
d.NotUsefulCount++
}
d.persist() // 2026-09-27 补:此处漏了 persist() ⇒ useful/not_useful 从不落盘、重启即丢
}
// RecordUseful 记录有用反馈
func (d *Dashboard) RecordUseful() {
d.RecordFeedback(true)
d.persist()
}
// RecordNotUseful 记录无用反馈
func (d *Dashboard) RecordNotUseful() {
d.RecordFeedback(false)
d.persist()
}
// RecordDistillSuccess 记录蒸馏成功次数(独立于用户反馈,2026-10-14 t_c06ad4d0 解污染)。
// 原先蒸馏回调误调 RecordUseful() ⇒ 把「蒸馏成功次数」灌进 recall_usefulness_rate 的分子,
// 与分母(用户 not-useful 反馈)不同源,比率在数学上不成立、恒被蒸馏数抬高到 ~1.0。
// 现拆成独立计数 distill_success_count,recall_usefulness_rate 只由 [useful|not-useful] 反馈驱动。
func (d *Dashboard) RecordDistillSuccess() {
d.mu.Lock()
defer d.mu.Unlock()
d.DistillSuccessCount++
d.persist()
}
// QualityScore 计算当前质量分数
func (d *Dashboard) QualityScore() float64 {
d.mu.RLock()
defer d.mu.RUnlock()
total := d.UsefulCount + d.NotUsefulCount
if total == 0 {
return 0.5
}
return float64(d.UsefulCount) / float64(total)
}
// ─── 知识缺口检测 ────────────────────────────────────────
type GapType string
const (
GapUnknown GapType = "A" // 真不知道
GapSynonym GapType = "B" // 同义词不匹配
GapRecallFailed GapType = "C" // 召回失败
GapFragmented GapType = "D" // 碎片化
)
type Gap struct {
Topic string `json:"topic"`
Type GapType `json:"type"`
MissCount int `json:"miss_count"`
CreatedAt time.Time `json:"created_at"`
Closed bool `json:"closed"`
}
type GapDetector struct {
mu sync.RWMutex
gaps map[string]*Gap
misses map[string]int
// 向量比较引擎(用于缺口分类)
embedder *storage.Embedder
ldb storage.LanceDB
// Redis 持久化
redisClient *storage.RedisClient
}
// gapRedisKey 缺口持久化 Redis key
const gapRedisKey = "zhiyi:gaps"
const gapMissRedisKey = "zhiyi:gap_misses"
// EnableGapRedisPersistence 启动缺口 Redis 持久化
func (gd *GapDetector) EnableGapRedisPersistence() {
rc := storage.GetRedisClient()
if rc == nil {
return
}
gd.redisClient = rc
gd.loadFromRedis()
}
func (gd *GapDetector) loadFromRedis() {
if gd.redisClient == nil {
return
}
// 加载 gaps
data, err := gd.redisClient.HGetAll(gapRedisKey)
if err == nil && len(data) > 0 {
for topic, jsonStr := range data {
var gap Gap
if err := json.Unmarshal([]byte(jsonStr), &gap); err == nil {
gd.gaps[topic] = &gap
}
}
}
// 加载 misses
missData, err := gd.redisClient.HGetAll(gapMissRedisKey)
if err == nil && len(missData) > 0 {
for topic, countStr := range missData {
if count, err := strconv.Atoi(countStr); err == nil {
gd.misses[topic] = count
}
}
}
}
func (gd *GapDetector) persistGaps() {
if gd.redisClient == nil {
return
}
for topic, gap := range gd.gaps {
data, _ := json.Marshal(gap)
gd.redisClient.HSet(gapRedisKey, topic, string(data))
}
gd.redisClient.Expire(gapRedisKey, 90*24*time.Hour)
}
func (gd *GapDetector) persistMisses() {
if gd.redisClient == nil {
return
}
for topic, count := range gd.misses {
gd.redisClient.HSet(gapMissRedisKey, topic, strconv.Itoa(count))
}
gd.redisClient.Expire(gapMissRedisKey, 90*24*time.Hour)
}
func NewGapDetector(emb *storage.Embedder, ldb storage.LanceDB) *GapDetector {
return &GapDetector{
gaps: make(map[string]*Gap),
misses: make(map[string]int),
embedder: emb,
ldb: ldb,
}
}
// RecordMiss 记录一次召回失败
// gapThreshold 是「同一 topic 累计几次 miss 才生成 gap」的阈值。
// 由 routes 的调参器经 SetGapThreshold 推送;安全边界 [2,10]:
// 下界 2 防「一次偶然即成缺口」把缺口表灌爆,上界 10 防阈值大到永不生成(闭环静默死掉)。
var gapThreshold atomic.Int64
func init() { gapThreshold.Store(3) }
// SetGapThreshold 由调参器推送新的缺口阈值(越界值一律忽略)。
func SetGapThreshold(n int) {
if n < 2 || n > 10 {
return
}
gapThreshold.Store(int64(n))
}
// GapThreshold 返回当前缺口阈值。
func GapThreshold() int { return int(gapThreshold.Load()) }
func (gd *GapDetector) RecordMiss(topic string) *Gap {
gd.mu.Lock()
defer gd.mu.Unlock()
gd.misses[topic]++
gd.persistMisses()
// 2026-10-14:阈值改读调参器推送值(默认 3,与原硬编码一致 ⇒ 行为不变)。
// 此前 Tuner.GapThreshold 没有任何读者 ⇒ 调参改了也不生效。
if gd.misses[topic] >= GapThreshold() {
if _, exists := gd.gaps[topic]; !exists {
gap := &Gap{
Topic: topic,
Type: gd.classifyGap(topic),
MissCount: gd.misses[topic],
CreatedAt: time.Now(),
}
gd.gaps[topic] = gap
gd.persistGaps()
Dash.TotalGaps++ // 同步仪表盘
return gap
}
}
return nil
}
// classifyGap 基于向量比较的缺口分类(与设计文档一致)
//
// sim > 0.85 且 category 相近 → Type C(召回失败)
// sim > 0.75 但 entity 名称不同 → Type B(同义词不匹配)
// max sim < 0.3(全聚类 centroid)→ Type A(真未知)
// 多个 L1 记忆各自部分覆盖 → Type D(碎片化)
func (gd *GapDetector) classifyGap(topic string) GapType {
if gd.embedder == nil || gd.ldb == nil {
// 降级:无 embedder 时用简单启发式
// 英文大写开头 → 可能是同义词未收录
for _, r := range topic {
if r >= 'A' && r <= 'Z' {
return GapSynonym
}
}
// 中文:检查是否含常见同义词标记(又称、别名、aka、aka)
if containsAny(topic, []string{"又称", "别名", "也就是", "aka", "AKA", "同义词"}) {
return GapSynonym
}
// 含中文字符但无同义词标记 → 真未知
return GapUnknown
}
vec, err := gd.embedder.EncodeSingle(topic)
if err != nil {
return GapUnknown
}
// ANN 搜索 top-5 最近记忆
results, err := gd.ldb.Search("memories", vec, 5, "")
if err != nil || len(results) == 0 {
return GapUnknown
}
maxSim := results[0].QualityScore // 复用 quality_score 字段存向量相似度
if maxSim > 0.85 {
// 存在高度相似 → 召回失败(应调整 top_k/diversity)
return GapRecallFailed
}
if maxSim > 0.75 {
// 有相似但名称不同 → 同义词/别称问题
return GapSynonym
}
// 检查多个 L1 是否有部分覆盖 → 碎片化
if len(results) >= 3 {
partialCover := 0
for _, r := range results {
if r.QualityScore > 0.5 && r.QualityScore < 0.7 {
partialCover++
}
}
if partialCover >= 2 {
return GapFragmented
}
}
// 最大相似度 < 0.3 → 真未知
if maxSim < 0.3 {
return GapUnknown
}
// 兜底
return GapUnknown
}
func (gd *GapDetector) List() []*Gap {
gd.mu.RLock()
defer gd.mu.RUnlock()
var result []*Gap
for _, g := range gd.gaps {
result = append(result, g)
}
return result
}
func (gd *GapDetector) Close(topic string) {
gd.mu.Lock()
defer gd.mu.Unlock()
if g, ok := gd.gaps[topic]; ok {
g.Closed = true
gd.persistGaps()
Dash.RecordGapClosed() // 同步仪表盘
}
}
// RecordHit 记录一次召回命中,自动关闭该 topic 的 open gap
func (gd *GapDetector) RecordHit(topic string) {
gd.mu.Lock()
defer gd.mu.Unlock()
if g, ok := gd.gaps[topic]; ok && !g.Closed {
g.Closed = true
gd.persistGaps()
Dash.RecordGapClosed()
}
}
// ClearMisses 清除某个 topic 的 miss 计数(recall 命中后调用)
func (gd *GapDetector) ClearMisses(topic string) {
gd.mu.Lock()
defer gd.mu.Unlock()
delete(gd.misses, topic)
gd.persistMisses()
}
// ─── 全局 GapDetector 访问(供 routes 包调用)───────────────
var globalGapDetector *GapDetector
func SetGlobalGapDetector(gd *GapDetector) { globalGapDetector = gd }
func GetGlobalGapDetector() *GapDetector { return globalGapDetector }
// ─── 因果追踪 ────────────────────────────────────────────
type TraceEntry struct {
MemoryID string `json:"memory_id"`
Version int `json:"version"`
Content string `json:"content"`
Source string `json:"source"` // muchen_oral / config_parse / agent_infer / llm_distill
Trigger string `json:"trigger"` // 什么触发了这次修改
UpdatedAt time.Time `json:"updated_at"`
}
type CausalTracker struct {
mu sync.RWMutex
entries map[string][]*TraceEntry // memory_id → version history
deps map[string][]string // memory_id → depends_on[]
// Redis 持久化
redisClient *storage.RedisClient
}
const causalRedisKey = "zhiyi:causal_entries"
// EnableCausalRedisPersistence 启动因果追踪 Redis 持久化
func (ct *CausalTracker) EnableCausalRedisPersistence() {
rc := storage.GetRedisClient()
if rc == nil {
return
}
ct.redisClient = rc
ct.loadCausalFromRedis()
}
func (ct *CausalTracker) loadCausalFromRedis() {
if ct.redisClient == nil {
return
}
data, err := ct.redisClient.HGetAll(causalRedisKey)
if err != nil || len(data) == 0 {
return
}
// entries: memory_id → JSON数组
// deps: memory_id → JSON数组
ct.mu.Lock()
defer ct.mu.Unlock()
for key, jsonStr := range data {
if len(key) > 5 && key[:5] == "dep_:" {
// dep_:memory_id → [dependency_ids]
memID := key[5:]
var deps []string
if json.Unmarshal([]byte(jsonStr), &deps) == nil {
ct.deps[memID] = deps
}
} else {
// memory_id → [TraceEntry]
var entries []*TraceEntry
if json.Unmarshal([]byte(jsonStr), &entries) == nil {
ct.entries[key] = entries
}
}
}
}
func (ct *CausalTracker) persistCausal() {
if ct.redisClient == nil {
return
}
for memID, entries := range ct.entries {
data, _ := json.Marshal(entries)
ct.redisClient.HSet(causalRedisKey, memID, string(data))
}
for memID, deps := range ct.deps {
data, _ := json.Marshal(deps)
ct.redisClient.HSet(causalRedisKey, "dep_:"+memID, string(data))
}
ct.redisClient.Expire(causalRedisKey, 90*24*time.Hour)
}
func NewCausalTracker() *CausalTracker {
return &CausalTracker{
entries: make(map[string][]*TraceEntry),
deps: make(map[string][]string),
}
}
// RecordVersion 记录版本变更
func (ct *CausalTracker) RecordVersion(memoryID, content, source, trigger string) {
ct.mu.Lock()
defer ct.mu.Unlock()
entry := &TraceEntry{
MemoryID: memoryID,
Version: len(ct.entries[memoryID]) + 1,
Content: content,
Source: source,
Trigger: trigger,
UpdatedAt: time.Now(),
}
ct.entries[memoryID] = append(ct.entries[memoryID], entry)
ct.persistCausal()
}
// AddDependency A depends_on B
func (ct *CausalTracker) AddDependency(a, b string) {
ct.mu.Lock()
defer ct.mu.Unlock()
ct.deps[a] = append(ct.deps[a], b)
ct.persistCausal()
}
// GetAffected 当 memoryID 被修正时,返回所有依赖它的记忆
func (ct *CausalTracker) GetAffected(memoryID string, visited map[string]bool) []string {
ct.mu.RLock()
defer ct.mu.RUnlock()
if visited == nil {
visited = make(map[string]bool)
}
if visited[memoryID] {
return nil
}
visited[memoryID] = true
var affected []string
for dependent, deps := range ct.deps {
for _, d := range deps {
if d == memoryID && !visited[dependent] {
affected = append(affected, dependent)
affected = append(affected, ct.GetAffected(dependent, visited)...)
}
}
}
return affected
}
// SourceTrust 来源信任度
func SourceTrust(source string) float64 {
switch source {
case "muchen_oral":
return 1.0
case "muchen_feishu":
return 0.95
case "config_parse":
return 0.7
case "agent_infer":
return 0.5
case "llm_distill":
return 0.4
default:
return 0.3
}
}
// IsVolatile 判断一条记忆是否易变(频繁修正)
func (ct *CausalTracker) IsVolatile(memoryID string) bool {
ct.mu.RLock()
defer ct.mu.RUnlock()
return len(ct.entries[memoryID]) >= 3
}
// Entries 返回全部版本历史(只读)
func (ct *CausalTracker) Entries() map[string][]*TraceEntry {
ct.mu.RLock()
defer ct.mu.RUnlock()
cp := make(map[string][]*TraceEntry, len(ct.entries))
for k, v := range ct.entries {
cp[k] = v
}
return cp
}
// ─── 记忆预取 ────────────────────────────────────────────
type PrefetchGraph struct {
mu sync.RWMutex
coOccurs map[string]map[string]int // A → {B: count}
}
func NewPrefetchGraph() *PrefetchGraph {
return &PrefetchGraph{coOccurs: make(map[string]map[string]int)}
}
func (pg *PrefetchGraph) RecordCoAccess(a, b string) {
pg.mu.Lock()
defer pg.mu.Unlock()
if pg.coOccurs[a] == nil {
pg.coOccurs[a] = make(map[string]int)
}
pg.coOccurs[a][b]++
}
// GetPrefetch 获取某个 query 的预取候选项
// RecordDeprecation 记录废弃操作
func (d *Dashboard) RecordDeprecation() {
d.mu.Lock()
defer d.mu.Unlock()
d.DeprecatedToday++
d.persist()
}
// RecordCorrection 记录用户修正
func (d *Dashboard) RecordCorrection(source string) {
d.mu.Lock()
defer d.mu.Unlock()
d.TotalFixes++
if source == "muchen_correction" {
d.CascadeFixedTotal++
}
d.persist()
}
// RecordGapClosed 记录缺口关闭
func (d *Dashboard) RecordGapClosed() {
d.mu.Lock()
defer d.mu.Unlock()
d.ClosedGaps++
d.persist()
}
// RecordConflictResolved 记录冲突解决
func (d *Dashboard) RecordConflictResolved(auto bool) {
d.mu.Lock()
defer d.mu.Unlock()
d.TotalConflicts++
if auto {
d.AutoResolvedConflicts++
}
d.persist()
}
// RecordDistillLoss 记录蒸馏质量损失
func (d *Dashboard) RecordDistillLoss(loss float64) {
d.mu.Lock()
defer d.mu.Unlock()
d.DistillLossSum += loss
d.DistillLossCount++
d.persist()
}
func (pg *PrefetchGraph) GetPrefetch(query string) []string {
pg.mu.RLock()
defer pg.mu.RUnlock()
related := pg.coOccurs[query]
if related == nil {
return nil
}
var result []string
for topic, count := range related {
total := 0
for _, c := range pg.coOccurs[query] {
total += c
}
prob := float64(count) / float64(total)
if prob > 0.6 {
result = append(result, topic)
}
}
return result
}
// containsAny 检查字符串是否包含任意一个子串(无 strings 依赖)
func containsAny(s string, substrs []string) bool {
for _, sub := range substrs {
if len(sub) == 0 {
continue
}
for i := 0; i <= len(s)-len(sub); i++ {
match := true
for j := 0; j < len(sub); j++ {
if s[i+j] != sub[j] {
match = false
break
}
}
if match {
return true
}
}
}
return false
}