实测三真相(2026-10-14): ① 无消费者 —— grep Tuner. 全仓只有 3 处路由注册,没有任何代码读 Diversity/GapThreshold/DecayRate/DistillInterval ⇒ 调参改了数字、行为零变化。 ② 不幂等(棘轮)—— 同一判据每调一次推一步:连跑两次 decay 0.015→0.018→0.0216。 ③ 不持久化 —— 包级内存变量,重启即回默认。 修法: - 接线:storage.SetDefaultMMRDiversity / selfoptimize.SetGapThreshold(消费者持值、调参器推送, 避免 storage←routes 反向依赖)。初值对齐消费方真实默认(Diversity 0.5→0.3)⇒ 当天行为不变。 - 阻尼:同一参数 6h 冷却。 - 持久化:/var/lib/memoryweave/tuning-state.json(env ZHIYI_TUNING_STATE 可覆盖),启动 load+接线。 - 删规则3(废弃率→提高衰减率):方向危险 —— NewForgetter 注释记载 「0.015→0.03 曾致误删(36天库龄下 0-recall 长记忆全被清)」。 - 删规则5(蒸馏损失→缩短蒸馏间隔):无正确消费方 —— engine.go 的 batchTimeout 是 「队列冻结兜底 ticker」(安全网),不是蒸馏节奏。 测试 4 个:哨兵/少样本拦截、冷却、推送、持久化(含「健康数据仍能调」反向验证)。
923 lines
27 KiB
Go
923 lines
27 KiB
Go
// 织忆 MemoryWeave — 自优化引擎
|
||
package selfoptimize
|
||
|
||
import (
|
||
"encoding/json"
|
||
"math"
|
||
"os"
|
||
"strconv"
|
||
"sync"
|
||
"sync/atomic"
|
||
"time"
|
||
|
||
"github.com/xiaoxue/memoryweave/internal/storage"
|
||
)
|
||
|
||
// Redis persistence key
|
||
const dashboardRedisKey = "zhiyi:dashboard"
|
||
|
||
// dashboardLocalPath 本地 JSON 落盘点 —— Redis 不可用时的兜底(2026-09-27 加)
|
||
//
|
||
// 背景:本机无 Redis(无 redis-cli/redis-server、6379 无监听、单元无 REDIS_ADDR/URL)
|
||
// ⇒ storage.GetRedisClient() 返回 nil ⇒ redisPersistence 恒为 nil
|
||
// ⇒ 原 persist() 第一行就 return:7 项指标只活在进程内,**重启即归零**,
|
||
//
|
||
// 而 tuning / executor / distributed 都在读它做自优化决策。
|
||
//
|
||
// 可用 ZHIYI_DASHBOARD_PATH 覆盖(测试用)。
|
||
func dashboardLocalPath() string {
|
||
if p := os.Getenv("ZHIYI_DASHBOARD_PATH"); p != "" {
|
||
return p
|
||
}
|
||
return "/var/lib/memoryweave/dashboard.json"
|
||
}
|
||
|
||
// dashboardState 可序列化快照 —— Dashboard 内含 sync.RWMutex,不能直接 Marshal/整块复制
|
||
type dashboardState struct {
|
||
UsefulCount int `json:"useful_count"`
|
||
NotUsefulCount int `json:"not_useful_count"`
|
||
TotalRecalls int `json:"total_recalls"`
|
||
HitCount int `json:"hit_count"`
|
||
ClosedGaps int `json:"closed_gaps"`
|
||
TotalGaps int `json:"total_gaps"`
|
||
CascadeFixedTotal int `json:"cascade_fixed_total"`
|
||
TotalFixes int `json:"total_fixes"`
|
||
DeprecatedToday int `json:"deprecated_today"`
|
||
DeprecatedDate string `json:"deprecated_date"` // deprecated_today 的归属日期(跨天归零用)
|
||
DistillLossSum float64 `json:"distill_loss_sum"`
|
||
DistillLossCount int `json:"distill_loss_count"`
|
||
DistillSuccessCount int `json:"distill_success_count"`
|
||
AutoResolvedConflicts int `json:"auto_resolved_conflicts"`
|
||
TotalConflicts int `json:"total_conflicts"`
|
||
}
|
||
|
||
func todayStr() string { return time.Now().Format("2006-01-02") }
|
||
|
||
// redisPersistence 是否开启了 Redis 持久化(由 server.go 在启动时设置)
|
||
var redisPersistence *storage.RedisClient
|
||
|
||
// EnableRedisPersistence 启动 Dashboard 持久化(Redis 优先;Redis 不可用则转本地 JSON)
|
||
func EnableRedisPersistence() {
|
||
rc := storage.GetRedisClient()
|
||
if rc != nil {
|
||
redisPersistence = rc
|
||
_ = loadDashboard()
|
||
return
|
||
}
|
||
_ = loadDashboardLocal()
|
||
}
|
||
|
||
// loadDashboardLocal 从本地 JSON 恢复指标;文件不存在/损坏 → 静默保持零值(= 老行为)
|
||
func loadDashboardLocal() error {
|
||
b, err := os.ReadFile(dashboardLocalPath())
|
||
if err != nil {
|
||
return err
|
||
}
|
||
var st dashboardState
|
||
if err := json.Unmarshal(b, &st); err != nil {
|
||
return err
|
||
}
|
||
Dash.mu.Lock()
|
||
defer Dash.mu.Unlock()
|
||
Dash.UsefulCount = st.UsefulCount
|
||
Dash.NotUsefulCount = st.NotUsefulCount
|
||
Dash.TotalRecalls = st.TotalRecalls
|
||
Dash.HitCount = st.HitCount
|
||
Dash.ClosedGaps = st.ClosedGaps
|
||
Dash.TotalGaps = st.TotalGaps
|
||
Dash.CascadeFixedTotal = st.CascadeFixedTotal
|
||
Dash.TotalFixes = st.TotalFixes
|
||
Dash.DeprecatedToday = st.DeprecatedToday
|
||
if st.DeprecatedDate != todayStr() { // "今日"计数跨天归零(进程跨零点不重置,下次加载时纠正)
|
||
Dash.DeprecatedToday = 0
|
||
}
|
||
Dash.DistillLossSum = st.DistillLossSum
|
||
Dash.DistillLossCount = st.DistillLossCount
|
||
Dash.DistillSuccessCount = st.DistillSuccessCount
|
||
Dash.AutoResolvedConflicts = st.AutoResolvedConflicts
|
||
Dash.TotalConflicts = st.TotalConflicts
|
||
return nil
|
||
}
|
||
|
||
// persistLocalLocked 原子写本地 JSON(best-effort:任何失败都静默 → 行为退回"仅内存")
|
||
// 注意:调用方必须已持有 d.mu 锁
|
||
func (d *Dashboard) persistLocalLocked() {
|
||
st := dashboardState{
|
||
UsefulCount: d.UsefulCount,
|
||
NotUsefulCount: d.NotUsefulCount,
|
||
TotalRecalls: d.TotalRecalls,
|
||
HitCount: d.HitCount,
|
||
ClosedGaps: d.ClosedGaps,
|
||
TotalGaps: d.TotalGaps,
|
||
CascadeFixedTotal: d.CascadeFixedTotal,
|
||
TotalFixes: d.TotalFixes,
|
||
DeprecatedToday: d.DeprecatedToday,
|
||
DeprecatedDate: todayStr(),
|
||
DistillLossSum: d.DistillLossSum,
|
||
DistillLossCount: d.DistillLossCount,
|
||
DistillSuccessCount: d.DistillSuccessCount,
|
||
AutoResolvedConflicts: d.AutoResolvedConflicts,
|
||
TotalConflicts: d.TotalConflicts,
|
||
}
|
||
b, err := json.Marshal(st)
|
||
if err != nil {
|
||
return
|
||
}
|
||
p := dashboardLocalPath()
|
||
tmp := p + ".tmp"
|
||
if err := os.WriteFile(tmp, b, 0o600); err != nil {
|
||
return
|
||
}
|
||
_ = os.Rename(tmp, p)
|
||
}
|
||
|
||
// persist 持久化 Dashboard:Redis 可用→写 Redis HASH;否则→写本地 JSON
|
||
// 注意:调用方必须已持有 d.mu 锁(RLock 或 Lock)
|
||
func (d *Dashboard) persist() {
|
||
if redisPersistence == nil {
|
||
d.persistLocalLocked()
|
||
return
|
||
}
|
||
_ = redisPersistence.HSet(dashboardRedisKey, "useful_count", strconv.Itoa(d.UsefulCount))
|
||
_ = redisPersistence.HSet(dashboardRedisKey, "not_useful_count", strconv.Itoa(d.NotUsefulCount))
|
||
_ = redisPersistence.HSet(dashboardRedisKey, "total_recalls", strconv.Itoa(d.TotalRecalls))
|
||
_ = redisPersistence.HSet(dashboardRedisKey, "hit_count", strconv.Itoa(d.HitCount))
|
||
_ = redisPersistence.HSet(dashboardRedisKey, "closed_gaps", strconv.Itoa(d.ClosedGaps))
|
||
_ = redisPersistence.HSet(dashboardRedisKey, "total_gaps", strconv.Itoa(d.TotalGaps))
|
||
_ = redisPersistence.HSet(dashboardRedisKey, "cascade_fixed_total", strconv.Itoa(d.CascadeFixedTotal))
|
||
_ = redisPersistence.HSet(dashboardRedisKey, "total_fixes", strconv.Itoa(d.TotalFixes))
|
||
_ = redisPersistence.HSet(dashboardRedisKey, "deprecated_today", strconv.Itoa(d.DeprecatedToday))
|
||
_ = redisPersistence.HSet(dashboardRedisKey, "distill_loss_sum", strconv.FormatFloat(d.DistillLossSum, 'f', 4, 64))
|
||
_ = redisPersistence.HSet(dashboardRedisKey, "distill_loss_count", strconv.Itoa(d.DistillLossCount))
|
||
_ = redisPersistence.HSet(dashboardRedisKey, "distill_success_count", strconv.Itoa(d.DistillSuccessCount))
|
||
_ = redisPersistence.HSet(dashboardRedisKey, "auto_resolved_conflicts", strconv.Itoa(d.AutoResolvedConflicts))
|
||
_ = redisPersistence.HSet(dashboardRedisKey, "total_conflicts", strconv.Itoa(d.TotalConflicts))
|
||
_ = redisPersistence.Expire(dashboardRedisKey, 90*24*time.Hour) // 90 天 TTL
|
||
}
|
||
|
||
// loadDashboard 从 Redis 加载 Dashboard 数据
|
||
func loadDashboard() error {
|
||
if redisPersistence == nil {
|
||
return nil
|
||
}
|
||
data, err := redisPersistence.HGetAll(dashboardRedisKey)
|
||
if err != nil || len(data) == 0 {
|
||
return err
|
||
}
|
||
Dash.mu.Lock()
|
||
defer Dash.mu.Unlock()
|
||
|
||
if v, err := strconv.Atoi(data["useful_count"]); err == nil {
|
||
Dash.UsefulCount = v
|
||
}
|
||
if v, err := strconv.Atoi(data["not_useful_count"]); err == nil {
|
||
Dash.NotUsefulCount = v
|
||
}
|
||
if v, err := strconv.Atoi(data["total_recalls"]); err == nil {
|
||
Dash.TotalRecalls = v
|
||
}
|
||
if v, err := strconv.Atoi(data["hit_count"]); err == nil {
|
||
Dash.HitCount = v
|
||
}
|
||
if v, err := strconv.Atoi(data["closed_gaps"]); err == nil {
|
||
Dash.ClosedGaps = v
|
||
}
|
||
if v, err := strconv.Atoi(data["total_gaps"]); err == nil {
|
||
Dash.TotalGaps = v
|
||
}
|
||
if v, err := strconv.Atoi(data["cascade_fixed_total"]); err == nil {
|
||
Dash.CascadeFixedTotal = v
|
||
}
|
||
if v, err := strconv.Atoi(data["total_fixes"]); err == nil {
|
||
Dash.TotalFixes = v
|
||
}
|
||
if v, err := strconv.Atoi(data["deprecated_today"]); err == nil {
|
||
Dash.DeprecatedToday = v
|
||
}
|
||
if v, err := strconv.ParseFloat(data["distill_loss_sum"], 64); err == nil {
|
||
Dash.DistillLossSum = v
|
||
}
|
||
if v, err := strconv.Atoi(data["distill_loss_count"]); err == nil {
|
||
Dash.DistillLossCount = v
|
||
}
|
||
if v, err := strconv.Atoi(data["distill_success_count"]); err == nil {
|
||
Dash.DistillSuccessCount = v
|
||
}
|
||
if v, err := strconv.Atoi(data["auto_resolved_conflicts"]); err == nil {
|
||
Dash.AutoResolvedConflicts = v
|
||
}
|
||
if v, err := strconv.Atoi(data["total_conflicts"]); err == nil {
|
||
Dash.TotalConflicts = v
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// ─── 自优化仪表盘 ────────────────────────────────────────
|
||
|
||
type Dashboard struct {
|
||
mu sync.RWMutex
|
||
UsefulCount int `json:"useful_count"`
|
||
NotUsefulCount int `json:"not_useful_count"`
|
||
TotalRecalls int `json:"total_recalls"`
|
||
HitCount int `json:"hit_count"`
|
||
ClosedGaps int `json:"closed_gaps"`
|
||
TotalGaps int `json:"total_gaps"`
|
||
CascadeFixedTotal int `json:"cascade_fixed_total"`
|
||
TotalFixes int `json:"total_fixes"`
|
||
DeprecatedToday int `json:"deprecated_today"`
|
||
DistillLossSum float64 `json:"distill_loss_sum"`
|
||
DistillLossCount int `json:"distill_loss_count"`
|
||
DistillSuccessCount int `json:"distill_success_count"`
|
||
AutoResolvedConflicts int `json:"auto_resolved_conflicts"`
|
||
TotalConflicts int `json:"total_conflicts"`
|
||
}
|
||
|
||
var Dash = &Dashboard{}
|
||
|
||
// metricNA 是「无数据」哨兵值。
|
||
//
|
||
// 2026-10-14(牧尘批准):比率型指标在**分母退化**时(样本数为 0,或负样本通道
|
||
// 恒空)此前返回 1 —— 被下游读成「100% 好用/100% 命中」。那不是满分,是失明:
|
||
// NotUseful=0 只说明「没人说过不好」,不说明「全都好」;
|
||
// HitCount/TotalRecalls 的分子来自 core.go 的 `len(results) > 0`,
|
||
// 量的是「召回有没有返回东西」,不是「召回对不对」。
|
||
//
|
||
// 现在统一返回 -1:真实比率必落在 [0,1],-1 不可能被误读成任何一档成绩。
|
||
// 消费方规则(配套键,同 map 返回):
|
||
//
|
||
// <name>_valid = 1 表示该比率可解读;0 表示值必为 -1、**禁止**当成绩用
|
||
// <name>_samples = 该比率的分母样本数(0 时即退化)
|
||
//
|
||
// 已有消费方均按 key 存在性取值(SyncFromDashboard / 各 JSON 端点),
|
||
// 本改动**只增键、不改键名**,故不破坏任何调用方。
|
||
const metricNA = -1.0
|
||
|
||
// rateOrNA 分母 > 0 才算比率(保留两位),否则返回 metricNA。
|
||
func rateOrNA(num, den int) float64 {
|
||
if den <= 0 {
|
||
return metricNA
|
||
}
|
||
return math.Round(float64(num)/float64(den)*100) / 100
|
||
}
|
||
|
||
// b2f 布尔转 0/1,用于 *_valid 键。
|
||
func b2f(b bool) float64 {
|
||
if b {
|
||
return 1
|
||
}
|
||
return 0
|
||
}
|
||
|
||
// Metrics 返回核心指标。
|
||
//
|
||
// 比率项在分母退化时返回 metricNA(-1) 而非 1;每项比率都配 *_valid / *_samples。
|
||
// 计数项(deprecated_per_day)与均值项(avg_distill_loss)同规则。
|
||
func (d *Dashboard) Metrics() map[string]float64 {
|
||
d.mu.RLock()
|
||
defer d.mu.RUnlock()
|
||
|
||
// 负样本通道为空 ⇒ 无从判断好坏(详见 metricNA 注释)。
|
||
usefulRate := metricNA
|
||
if d.NotUsefulCount > 0 {
|
||
usefulRate = rateOrNA(d.UsefulCount, d.UsefulCount+d.NotUsefulCount)
|
||
}
|
||
|
||
hitRate := rateOrNA(d.HitCount, d.TotalRecalls)
|
||
gapRate := rateOrNA(d.ClosedGaps, d.TotalGaps)
|
||
cascadeRate := rateOrNA(d.CascadeFixedTotal, d.TotalFixes)
|
||
autoRate := rateOrNA(d.AutoResolvedConflicts, d.TotalConflicts)
|
||
|
||
// 均值项:零样本的均值同样是退化,按同一规则返回 metricNA。
|
||
avgLoss := metricNA
|
||
if d.DistillLossCount > 0 {
|
||
avgLoss = math.Round(d.DistillLossSum/float64(d.DistillLossCount)*100) / 100
|
||
}
|
||
|
||
return map[string]float64{
|
||
"recall_usefulness_rate": usefulRate,
|
||
"recall_usefulness_samples": float64(d.UsefulCount + d.NotUsefulCount),
|
||
"recall_usefulness_negative": float64(d.NotUsefulCount),
|
||
"recall_usefulness_positive": float64(d.UsefulCount),
|
||
"recall_usefulness_valid": b2f(d.NotUsefulCount > 0),
|
||
|
||
"recall_hit_rate": hitRate,
|
||
"recall_hit_samples": float64(d.TotalRecalls),
|
||
"recall_hit_valid": b2f(d.TotalRecalls > 0),
|
||
|
||
"gap_closure_rate": gapRate,
|
||
"gap_closure_samples": float64(d.TotalGaps),
|
||
"gap_closure_valid": b2f(d.TotalGaps > 0),
|
||
|
||
"cascade_fix_rate": cascadeRate,
|
||
"cascade_fix_samples": float64(d.TotalFixes),
|
||
"cascade_fix_valid": b2f(d.TotalFixes > 0),
|
||
|
||
"auto_resolve_rate": autoRate,
|
||
"auto_resolve_samples": float64(d.TotalConflicts),
|
||
"auto_resolve_valid": b2f(d.TotalConflicts > 0),
|
||
|
||
"deprecated_per_day": float64(d.DeprecatedToday),
|
||
|
||
"distill_success_count": float64(d.DistillSuccessCount),
|
||
"avg_distill_loss": avgLoss,
|
||
"avg_distill_loss_samples": float64(d.DistillLossCount),
|
||
"avg_distill_loss_valid": b2f(d.DistillLossCount > 0),
|
||
}
|
||
}
|
||
|
||
func (d *Dashboard) RecordRecall(hit bool) {
|
||
d.mu.Lock()
|
||
defer d.mu.Unlock()
|
||
d.TotalRecalls++
|
||
if hit {
|
||
d.HitCount++
|
||
}
|
||
d.persist()
|
||
}
|
||
|
||
func (d *Dashboard) RecordFeedback(useful bool) {
|
||
d.mu.Lock()
|
||
defer d.mu.Unlock()
|
||
if useful {
|
||
d.UsefulCount++
|
||
} else {
|
||
d.NotUsefulCount++
|
||
}
|
||
d.persist() // 2026-09-27 补:此处漏了 persist() ⇒ useful/not_useful 从不落盘、重启即丢
|
||
}
|
||
|
||
// RecordUseful 记录有用反馈
|
||
func (d *Dashboard) RecordUseful() {
|
||
d.RecordFeedback(true)
|
||
d.persist()
|
||
}
|
||
|
||
// RecordNotUseful 记录无用反馈
|
||
func (d *Dashboard) RecordNotUseful() {
|
||
d.RecordFeedback(false)
|
||
d.persist()
|
||
}
|
||
|
||
// RecordDistillSuccess 记录蒸馏成功次数(独立于用户反馈,2026-10-14 t_c06ad4d0 解污染)。
|
||
// 原先蒸馏回调误调 RecordUseful() ⇒ 把「蒸馏成功次数」灌进 recall_usefulness_rate 的分子,
|
||
// 与分母(用户 not-useful 反馈)不同源,比率在数学上不成立、恒被蒸馏数抬高到 ~1.0。
|
||
// 现拆成独立计数 distill_success_count,recall_usefulness_rate 只由 [useful|not-useful] 反馈驱动。
|
||
func (d *Dashboard) RecordDistillSuccess() {
|
||
d.mu.Lock()
|
||
defer d.mu.Unlock()
|
||
d.DistillSuccessCount++
|
||
d.persist()
|
||
}
|
||
|
||
// QualityScore 计算当前质量分数
|
||
func (d *Dashboard) QualityScore() float64 {
|
||
d.mu.RLock()
|
||
defer d.mu.RUnlock()
|
||
total := d.UsefulCount + d.NotUsefulCount
|
||
if total == 0 {
|
||
return 0.5
|
||
}
|
||
return float64(d.UsefulCount) / float64(total)
|
||
}
|
||
|
||
// ─── 知识缺口检测 ────────────────────────────────────────
|
||
|
||
type GapType string
|
||
|
||
const (
|
||
GapUnknown GapType = "A" // 真不知道
|
||
GapSynonym GapType = "B" // 同义词不匹配
|
||
GapRecallFailed GapType = "C" // 召回失败
|
||
GapFragmented GapType = "D" // 碎片化
|
||
)
|
||
|
||
type Gap struct {
|
||
Topic string `json:"topic"`
|
||
Type GapType `json:"type"`
|
||
MissCount int `json:"miss_count"`
|
||
CreatedAt time.Time `json:"created_at"`
|
||
Closed bool `json:"closed"`
|
||
}
|
||
|
||
type GapDetector struct {
|
||
mu sync.RWMutex
|
||
gaps map[string]*Gap
|
||
misses map[string]int
|
||
|
||
// 向量比较引擎(用于缺口分类)
|
||
embedder *storage.Embedder
|
||
ldb storage.LanceDB
|
||
|
||
// Redis 持久化
|
||
redisClient *storage.RedisClient
|
||
}
|
||
|
||
// gapRedisKey 缺口持久化 Redis key
|
||
const gapRedisKey = "zhiyi:gaps"
|
||
const gapMissRedisKey = "zhiyi:gap_misses"
|
||
|
||
// EnableGapRedisPersistence 启动缺口 Redis 持久化
|
||
func (gd *GapDetector) EnableGapRedisPersistence() {
|
||
rc := storage.GetRedisClient()
|
||
if rc == nil {
|
||
return
|
||
}
|
||
gd.redisClient = rc
|
||
gd.loadFromRedis()
|
||
}
|
||
|
||
func (gd *GapDetector) loadFromRedis() {
|
||
if gd.redisClient == nil {
|
||
return
|
||
}
|
||
// 加载 gaps
|
||
data, err := gd.redisClient.HGetAll(gapRedisKey)
|
||
if err == nil && len(data) > 0 {
|
||
for topic, jsonStr := range data {
|
||
var gap Gap
|
||
if err := json.Unmarshal([]byte(jsonStr), &gap); err == nil {
|
||
gd.gaps[topic] = &gap
|
||
}
|
||
}
|
||
}
|
||
// 加载 misses
|
||
missData, err := gd.redisClient.HGetAll(gapMissRedisKey)
|
||
if err == nil && len(missData) > 0 {
|
||
for topic, countStr := range missData {
|
||
if count, err := strconv.Atoi(countStr); err == nil {
|
||
gd.misses[topic] = count
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
func (gd *GapDetector) persistGaps() {
|
||
if gd.redisClient == nil {
|
||
return
|
||
}
|
||
for topic, gap := range gd.gaps {
|
||
data, _ := json.Marshal(gap)
|
||
gd.redisClient.HSet(gapRedisKey, topic, string(data))
|
||
}
|
||
gd.redisClient.Expire(gapRedisKey, 90*24*time.Hour)
|
||
}
|
||
|
||
func (gd *GapDetector) persistMisses() {
|
||
if gd.redisClient == nil {
|
||
return
|
||
}
|
||
for topic, count := range gd.misses {
|
||
gd.redisClient.HSet(gapMissRedisKey, topic, strconv.Itoa(count))
|
||
}
|
||
gd.redisClient.Expire(gapMissRedisKey, 90*24*time.Hour)
|
||
}
|
||
|
||
func NewGapDetector(emb *storage.Embedder, ldb storage.LanceDB) *GapDetector {
|
||
return &GapDetector{
|
||
gaps: make(map[string]*Gap),
|
||
misses: make(map[string]int),
|
||
embedder: emb,
|
||
ldb: ldb,
|
||
}
|
||
}
|
||
|
||
// RecordMiss 记录一次召回失败
|
||
// gapThreshold 是「同一 topic 累计几次 miss 才生成 gap」的阈值。
|
||
// 由 routes 的调参器经 SetGapThreshold 推送;安全边界 [2,10]:
|
||
// 下界 2 防「一次偶然即成缺口」把缺口表灌爆,上界 10 防阈值大到永不生成(闭环静默死掉)。
|
||
var gapThreshold atomic.Int64
|
||
|
||
func init() { gapThreshold.Store(3) }
|
||
|
||
// SetGapThreshold 由调参器推送新的缺口阈值(越界值一律忽略)。
|
||
func SetGapThreshold(n int) {
|
||
if n < 2 || n > 10 {
|
||
return
|
||
}
|
||
gapThreshold.Store(int64(n))
|
||
}
|
||
|
||
// GapThreshold 返回当前缺口阈值。
|
||
func GapThreshold() int { return int(gapThreshold.Load()) }
|
||
|
||
func (gd *GapDetector) RecordMiss(topic string) *Gap {
|
||
gd.mu.Lock()
|
||
defer gd.mu.Unlock()
|
||
|
||
gd.misses[topic]++
|
||
gd.persistMisses()
|
||
// 2026-10-14:阈值改读调参器推送值(默认 3,与原硬编码一致 ⇒ 行为不变)。
|
||
// 此前 Tuner.GapThreshold 没有任何读者 ⇒ 调参改了也不生效。
|
||
if gd.misses[topic] >= GapThreshold() {
|
||
if _, exists := gd.gaps[topic]; !exists {
|
||
gap := &Gap{
|
||
Topic: topic,
|
||
Type: gd.classifyGap(topic),
|
||
MissCount: gd.misses[topic],
|
||
CreatedAt: time.Now(),
|
||
}
|
||
gd.gaps[topic] = gap
|
||
gd.persistGaps()
|
||
Dash.TotalGaps++ // 同步仪表盘
|
||
return gap
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// classifyGap 基于向量比较的缺口分类(与设计文档一致)
|
||
//
|
||
// sim > 0.85 且 category 相近 → Type C(召回失败)
|
||
// sim > 0.75 但 entity 名称不同 → Type B(同义词不匹配)
|
||
// max sim < 0.3(全聚类 centroid)→ Type A(真未知)
|
||
// 多个 L1 记忆各自部分覆盖 → Type D(碎片化)
|
||
func (gd *GapDetector) classifyGap(topic string) GapType {
|
||
if gd.embedder == nil || gd.ldb == nil {
|
||
// 降级:无 embedder 时用简单启发式
|
||
// 英文大写开头 → 可能是同义词未收录
|
||
for _, r := range topic {
|
||
if r >= 'A' && r <= 'Z' {
|
||
return GapSynonym
|
||
}
|
||
}
|
||
// 中文:检查是否含常见同义词标记(又称、别名、aka、aka)
|
||
if containsAny(topic, []string{"又称", "别名", "也就是", "aka", "AKA", "同义词"}) {
|
||
return GapSynonym
|
||
}
|
||
// 含中文字符但无同义词标记 → 真未知
|
||
return GapUnknown
|
||
}
|
||
|
||
vec, err := gd.embedder.EncodeSingle(topic)
|
||
if err != nil {
|
||
return GapUnknown
|
||
}
|
||
|
||
// ANN 搜索 top-5 最近记忆
|
||
results, err := gd.ldb.Search("memories", vec, 5, "")
|
||
if err != nil || len(results) == 0 {
|
||
return GapUnknown
|
||
}
|
||
|
||
maxSim := results[0].QualityScore // 复用 quality_score 字段存向量相似度
|
||
if maxSim > 0.85 {
|
||
// 存在高度相似 → 召回失败(应调整 top_k/diversity)
|
||
return GapRecallFailed
|
||
}
|
||
if maxSim > 0.75 {
|
||
// 有相似但名称不同 → 同义词/别称问题
|
||
return GapSynonym
|
||
}
|
||
|
||
// 检查多个 L1 是否有部分覆盖 → 碎片化
|
||
if len(results) >= 3 {
|
||
partialCover := 0
|
||
for _, r := range results {
|
||
if r.QualityScore > 0.5 && r.QualityScore < 0.7 {
|
||
partialCover++
|
||
}
|
||
}
|
||
if partialCover >= 2 {
|
||
return GapFragmented
|
||
}
|
||
}
|
||
|
||
// 最大相似度 < 0.3 → 真未知
|
||
if maxSim < 0.3 {
|
||
return GapUnknown
|
||
}
|
||
|
||
// 兜底
|
||
return GapUnknown
|
||
}
|
||
|
||
func (gd *GapDetector) List() []*Gap {
|
||
gd.mu.RLock()
|
||
defer gd.mu.RUnlock()
|
||
var result []*Gap
|
||
for _, g := range gd.gaps {
|
||
result = append(result, g)
|
||
}
|
||
return result
|
||
}
|
||
|
||
func (gd *GapDetector) Close(topic string) {
|
||
gd.mu.Lock()
|
||
defer gd.mu.Unlock()
|
||
if g, ok := gd.gaps[topic]; ok {
|
||
g.Closed = true
|
||
gd.persistGaps()
|
||
Dash.RecordGapClosed() // 同步仪表盘
|
||
}
|
||
}
|
||
|
||
// RecordHit 记录一次召回命中,自动关闭该 topic 的 open gap
|
||
func (gd *GapDetector) RecordHit(topic string) {
|
||
gd.mu.Lock()
|
||
defer gd.mu.Unlock()
|
||
if g, ok := gd.gaps[topic]; ok && !g.Closed {
|
||
g.Closed = true
|
||
gd.persistGaps()
|
||
Dash.RecordGapClosed()
|
||
}
|
||
}
|
||
|
||
// ClearMisses 清除某个 topic 的 miss 计数(recall 命中后调用)
|
||
func (gd *GapDetector) ClearMisses(topic string) {
|
||
gd.mu.Lock()
|
||
defer gd.mu.Unlock()
|
||
delete(gd.misses, topic)
|
||
gd.persistMisses()
|
||
}
|
||
|
||
// ─── 全局 GapDetector 访问(供 routes 包调用)───────────────
|
||
|
||
var globalGapDetector *GapDetector
|
||
|
||
func SetGlobalGapDetector(gd *GapDetector) { globalGapDetector = gd }
|
||
|
||
func GetGlobalGapDetector() *GapDetector { return globalGapDetector }
|
||
|
||
// ─── 因果追踪 ────────────────────────────────────────────
|
||
|
||
type TraceEntry struct {
|
||
MemoryID string `json:"memory_id"`
|
||
Version int `json:"version"`
|
||
Content string `json:"content"`
|
||
Source string `json:"source"` // muchen_oral / config_parse / agent_infer / llm_distill
|
||
Trigger string `json:"trigger"` // 什么触发了这次修改
|
||
UpdatedAt time.Time `json:"updated_at"`
|
||
}
|
||
|
||
type CausalTracker struct {
|
||
mu sync.RWMutex
|
||
entries map[string][]*TraceEntry // memory_id → version history
|
||
deps map[string][]string // memory_id → depends_on[]
|
||
|
||
// Redis 持久化
|
||
redisClient *storage.RedisClient
|
||
}
|
||
|
||
const causalRedisKey = "zhiyi:causal_entries"
|
||
|
||
// EnableCausalRedisPersistence 启动因果追踪 Redis 持久化
|
||
func (ct *CausalTracker) EnableCausalRedisPersistence() {
|
||
rc := storage.GetRedisClient()
|
||
if rc == nil {
|
||
return
|
||
}
|
||
ct.redisClient = rc
|
||
ct.loadCausalFromRedis()
|
||
}
|
||
|
||
func (ct *CausalTracker) loadCausalFromRedis() {
|
||
if ct.redisClient == nil {
|
||
return
|
||
}
|
||
data, err := ct.redisClient.HGetAll(causalRedisKey)
|
||
if err != nil || len(data) == 0 {
|
||
return
|
||
}
|
||
// entries: memory_id → JSON数组
|
||
// deps: memory_id → JSON数组
|
||
ct.mu.Lock()
|
||
defer ct.mu.Unlock()
|
||
for key, jsonStr := range data {
|
||
if len(key) > 5 && key[:5] == "dep_:" {
|
||
// dep_:memory_id → [dependency_ids]
|
||
memID := key[5:]
|
||
var deps []string
|
||
if json.Unmarshal([]byte(jsonStr), &deps) == nil {
|
||
ct.deps[memID] = deps
|
||
}
|
||
} else {
|
||
// memory_id → [TraceEntry]
|
||
var entries []*TraceEntry
|
||
if json.Unmarshal([]byte(jsonStr), &entries) == nil {
|
||
ct.entries[key] = entries
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
func (ct *CausalTracker) persistCausal() {
|
||
if ct.redisClient == nil {
|
||
return
|
||
}
|
||
for memID, entries := range ct.entries {
|
||
data, _ := json.Marshal(entries)
|
||
ct.redisClient.HSet(causalRedisKey, memID, string(data))
|
||
}
|
||
for memID, deps := range ct.deps {
|
||
data, _ := json.Marshal(deps)
|
||
ct.redisClient.HSet(causalRedisKey, "dep_:"+memID, string(data))
|
||
}
|
||
ct.redisClient.Expire(causalRedisKey, 90*24*time.Hour)
|
||
}
|
||
|
||
func NewCausalTracker() *CausalTracker {
|
||
return &CausalTracker{
|
||
entries: make(map[string][]*TraceEntry),
|
||
deps: make(map[string][]string),
|
||
}
|
||
}
|
||
|
||
// RecordVersion 记录版本变更
|
||
func (ct *CausalTracker) RecordVersion(memoryID, content, source, trigger string) {
|
||
ct.mu.Lock()
|
||
defer ct.mu.Unlock()
|
||
|
||
entry := &TraceEntry{
|
||
MemoryID: memoryID,
|
||
Version: len(ct.entries[memoryID]) + 1,
|
||
Content: content,
|
||
Source: source,
|
||
Trigger: trigger,
|
||
UpdatedAt: time.Now(),
|
||
}
|
||
ct.entries[memoryID] = append(ct.entries[memoryID], entry)
|
||
ct.persistCausal()
|
||
}
|
||
|
||
// AddDependency A depends_on B
|
||
func (ct *CausalTracker) AddDependency(a, b string) {
|
||
ct.mu.Lock()
|
||
defer ct.mu.Unlock()
|
||
ct.deps[a] = append(ct.deps[a], b)
|
||
ct.persistCausal()
|
||
}
|
||
|
||
// GetAffected 当 memoryID 被修正时,返回所有依赖它的记忆
|
||
func (ct *CausalTracker) GetAffected(memoryID string, visited map[string]bool) []string {
|
||
ct.mu.RLock()
|
||
defer ct.mu.RUnlock()
|
||
|
||
if visited == nil {
|
||
visited = make(map[string]bool)
|
||
}
|
||
if visited[memoryID] {
|
||
return nil
|
||
}
|
||
visited[memoryID] = true
|
||
|
||
var affected []string
|
||
for dependent, deps := range ct.deps {
|
||
for _, d := range deps {
|
||
if d == memoryID && !visited[dependent] {
|
||
affected = append(affected, dependent)
|
||
affected = append(affected, ct.GetAffected(dependent, visited)...)
|
||
}
|
||
}
|
||
}
|
||
return affected
|
||
}
|
||
|
||
// SourceTrust 来源信任度
|
||
func SourceTrust(source string) float64 {
|
||
switch source {
|
||
case "muchen_oral":
|
||
return 1.0
|
||
case "muchen_feishu":
|
||
return 0.95
|
||
case "config_parse":
|
||
return 0.7
|
||
case "agent_infer":
|
||
return 0.5
|
||
case "llm_distill":
|
||
return 0.4
|
||
default:
|
||
return 0.3
|
||
}
|
||
}
|
||
|
||
// IsVolatile 判断一条记忆是否易变(频繁修正)
|
||
func (ct *CausalTracker) IsVolatile(memoryID string) bool {
|
||
ct.mu.RLock()
|
||
defer ct.mu.RUnlock()
|
||
return len(ct.entries[memoryID]) >= 3
|
||
}
|
||
|
||
// Entries 返回全部版本历史(只读)
|
||
func (ct *CausalTracker) Entries() map[string][]*TraceEntry {
|
||
ct.mu.RLock()
|
||
defer ct.mu.RUnlock()
|
||
cp := make(map[string][]*TraceEntry, len(ct.entries))
|
||
for k, v := range ct.entries {
|
||
cp[k] = v
|
||
}
|
||
return cp
|
||
}
|
||
|
||
// ─── 记忆预取 ────────────────────────────────────────────
|
||
|
||
type PrefetchGraph struct {
|
||
mu sync.RWMutex
|
||
coOccurs map[string]map[string]int // A → {B: count}
|
||
}
|
||
|
||
func NewPrefetchGraph() *PrefetchGraph {
|
||
return &PrefetchGraph{coOccurs: make(map[string]map[string]int)}
|
||
}
|
||
|
||
func (pg *PrefetchGraph) RecordCoAccess(a, b string) {
|
||
pg.mu.Lock()
|
||
defer pg.mu.Unlock()
|
||
if pg.coOccurs[a] == nil {
|
||
pg.coOccurs[a] = make(map[string]int)
|
||
}
|
||
pg.coOccurs[a][b]++
|
||
}
|
||
|
||
// GetPrefetch 获取某个 query 的预取候选项
|
||
// RecordDeprecation 记录废弃操作
|
||
func (d *Dashboard) RecordDeprecation() {
|
||
d.mu.Lock()
|
||
defer d.mu.Unlock()
|
||
d.DeprecatedToday++
|
||
d.persist()
|
||
}
|
||
|
||
// RecordCorrection 记录用户修正
|
||
func (d *Dashboard) RecordCorrection(source string) {
|
||
d.mu.Lock()
|
||
defer d.mu.Unlock()
|
||
d.TotalFixes++
|
||
if source == "muchen_correction" {
|
||
d.CascadeFixedTotal++
|
||
}
|
||
d.persist()
|
||
}
|
||
|
||
// RecordGapClosed 记录缺口关闭
|
||
func (d *Dashboard) RecordGapClosed() {
|
||
d.mu.Lock()
|
||
defer d.mu.Unlock()
|
||
d.ClosedGaps++
|
||
d.persist()
|
||
}
|
||
|
||
// RecordConflictResolved 记录冲突解决
|
||
func (d *Dashboard) RecordConflictResolved(auto bool) {
|
||
d.mu.Lock()
|
||
defer d.mu.Unlock()
|
||
d.TotalConflicts++
|
||
if auto {
|
||
d.AutoResolvedConflicts++
|
||
}
|
||
d.persist()
|
||
}
|
||
|
||
// RecordDistillLoss 记录蒸馏质量损失
|
||
func (d *Dashboard) RecordDistillLoss(loss float64) {
|
||
d.mu.Lock()
|
||
defer d.mu.Unlock()
|
||
d.DistillLossSum += loss
|
||
d.DistillLossCount++
|
||
d.persist()
|
||
}
|
||
|
||
func (pg *PrefetchGraph) GetPrefetch(query string) []string {
|
||
pg.mu.RLock()
|
||
defer pg.mu.RUnlock()
|
||
|
||
related := pg.coOccurs[query]
|
||
if related == nil {
|
||
return nil
|
||
}
|
||
|
||
var result []string
|
||
for topic, count := range related {
|
||
total := 0
|
||
for _, c := range pg.coOccurs[query] {
|
||
total += c
|
||
}
|
||
prob := float64(count) / float64(total)
|
||
if prob > 0.6 {
|
||
result = append(result, topic)
|
||
}
|
||
}
|
||
return result
|
||
}
|
||
|
||
// containsAny 检查字符串是否包含任意一个子串(无 strings 依赖)
|
||
func containsAny(s string, substrs []string) bool {
|
||
for _, sub := range substrs {
|
||
if len(sub) == 0 {
|
||
continue
|
||
}
|
||
for i := 0; i <= len(s)-len(sub); i++ {
|
||
match := true
|
||
for j := 0; j < len(sub); j++ {
|
||
if s[i+j] != sub[j] {
|
||
match = false
|
||
break
|
||
}
|
||
}
|
||
if match {
|
||
return true
|
||
}
|
||
}
|
||
}
|
||
return false
|
||
}
|