Files
memoryweave/go/internal/selfoptimize/selfoptimize.go
T
小唯 b2a016e90f fix(metrics): t_c70f385c 指标口径修复 — recall_hit_rate 退役 + gap_closure_rate 断言 + 最小样本门 N=20
- recall_hit_rate 退役:分子原为 core.go 的 len(results)>0(恒=1、无区分度),
  「返回且被使用」语义已由 recall_usefulness_rate 承担(207 样本 0.98)。
  Metrics()/distributed/metrics-gauge/executor/tuning/CLI/daily-check 全链移除该键。
- gap_closure_rate 统一为「缺口个数」口径:Close() 加 !g.Closed 守卫(与 RecordHit 一致),
  根治同一缺口重复关闭导致 ClosedGaps 虚增、越界 4.5;Metrics() 加越界断言 gapRate>1 → -1。
- 所有率类指标加最小样本门 minRateSamples=20:样本<20 ⇒ -1 + valid=0,
  杜绝 auto_resolve_rate 2 样本判满分。
- 测试:metrics_retire_regression_test.go 用生产同款计数复现改前/改后 JSON;
  selfoptimize 全套通过。
2026-10-17 20:32:34 +08:00

962 lines
29 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// 织忆 MemoryWeave — 自优化引擎
package selfoptimize
import (
"encoding/json"
"math"
"os"
"strconv"
"sync"
"sync/atomic"
"time"
"github.com/xiaoxue/memoryweave/internal/storage"
)
// Redis persistence key
const dashboardRedisKey = "zhiyi:dashboard"
// dashboardLocalPath 本地 JSON 落盘点 —— Redis 不可用时的兜底(2026-09-27 加)
//
// 背景:本机无 Redis(无 redis-cli/redis-server、6379 无监听、单元无 REDIS_ADDR/URL)
// ⇒ storage.GetRedisClient() 返回 nil ⇒ redisPersistence 恒为 nil
// ⇒ 原 persist() 第一行就 return:7 项指标只活在进程内,**重启即归零**,
//
// 而 tuning / executor / distributed 都在读它做自优化决策。
//
// 可用 ZHIYI_DASHBOARD_PATH 覆盖(测试用)。
func dashboardLocalPath() string {
if p := os.Getenv("ZHIYI_DASHBOARD_PATH"); p != "" {
return p
}
return "/var/lib/memoryweave/dashboard.json"
}
// dashboardState 可序列化快照 —— Dashboard 内含 sync.RWMutex,不能直接 Marshal/整块复制
type dashboardState struct {
UsefulCount int `json:"useful_count"`
NotUsefulCount int `json:"not_useful_count"`
TotalRecalls int `json:"total_recalls"`
HitCount int `json:"hit_count"`
ClosedGaps int `json:"closed_gaps"`
TotalGaps int `json:"total_gaps"`
CascadeFixedTotal int `json:"cascade_fixed_total"`
TotalFixes int `json:"total_fixes"`
DeprecatedToday int `json:"deprecated_today"`
DeprecatedDate string `json:"deprecated_date"` // deprecated_today 的归属日期(跨天归零用)
DistillLossSum float64 `json:"distill_loss_sum"`
DistillLossCount int `json:"distill_loss_count"`
DistillSuccessCount int `json:"distill_success_count"`
AutoResolvedConflicts int `json:"auto_resolved_conflicts"`
TotalConflicts int `json:"total_conflicts"`
}
func todayStr() string { return time.Now().Format("2006-01-02") }
// redisPersistence 是否开启了 Redis 持久化(由 server.go 在启动时设置)
var redisPersistence *storage.RedisClient
// EnableRedisPersistence 启动 Dashboard 持久化(Redis 优先;Redis 不可用则转本地 JSON)
func EnableRedisPersistence() {
rc := storage.GetRedisClient()
if rc != nil {
redisPersistence = rc
_ = loadDashboard()
return
}
_ = loadDashboardLocal()
}
// loadDashboardLocal 从本地 JSON 恢复指标;文件不存在/损坏 → 静默保持零值(= 老行为)
func loadDashboardLocal() error {
b, err := os.ReadFile(dashboardLocalPath())
if err != nil {
return err
}
var st dashboardState
if err := json.Unmarshal(b, &st); err != nil {
return err
}
Dash.mu.Lock()
defer Dash.mu.Unlock()
Dash.UsefulCount = st.UsefulCount
Dash.NotUsefulCount = st.NotUsefulCount
Dash.TotalRecalls = st.TotalRecalls
Dash.HitCount = st.HitCount
Dash.ClosedGaps = st.ClosedGaps
Dash.TotalGaps = st.TotalGaps
Dash.CascadeFixedTotal = st.CascadeFixedTotal
Dash.TotalFixes = st.TotalFixes
Dash.DeprecatedToday = st.DeprecatedToday
if st.DeprecatedDate != todayStr() { // "今日"计数跨天归零(进程跨零点不重置,下次加载时纠正)
Dash.DeprecatedToday = 0
}
Dash.DistillLossSum = st.DistillLossSum
Dash.DistillLossCount = st.DistillLossCount
Dash.DistillSuccessCount = st.DistillSuccessCount
Dash.AutoResolvedConflicts = st.AutoResolvedConflicts
Dash.TotalConflicts = st.TotalConflicts
return nil
}
// persistLocalLocked 原子写本地 JSON(best-effort:任何失败都静默 → 行为退回"仅内存")
// 注意:调用方必须已持有 d.mu 锁
func (d *Dashboard) persistLocalLocked() {
st := dashboardState{
UsefulCount: d.UsefulCount,
NotUsefulCount: d.NotUsefulCount,
TotalRecalls: d.TotalRecalls,
HitCount: d.HitCount,
ClosedGaps: d.ClosedGaps,
TotalGaps: d.TotalGaps,
CascadeFixedTotal: d.CascadeFixedTotal,
TotalFixes: d.TotalFixes,
DeprecatedToday: d.DeprecatedToday,
DeprecatedDate: todayStr(),
DistillLossSum: d.DistillLossSum,
DistillLossCount: d.DistillLossCount,
DistillSuccessCount: d.DistillSuccessCount,
AutoResolvedConflicts: d.AutoResolvedConflicts,
TotalConflicts: d.TotalConflicts,
}
b, err := json.Marshal(st)
if err != nil {
return
}
p := dashboardLocalPath()
tmp := p + ".tmp"
if err := os.WriteFile(tmp, b, 0o600); err != nil {
return
}
_ = os.Rename(tmp, p)
}
// persist 持久化 Dashboard:Redis 可用→写 Redis HASH;否则→写本地 JSON
// 注意:调用方必须已持有 d.mu 锁(RLock 或 Lock)
func (d *Dashboard) persist() {
if redisPersistence == nil {
d.persistLocalLocked()
return
}
_ = redisPersistence.HSet(dashboardRedisKey, "useful_count", strconv.Itoa(d.UsefulCount))
_ = redisPersistence.HSet(dashboardRedisKey, "not_useful_count", strconv.Itoa(d.NotUsefulCount))
_ = redisPersistence.HSet(dashboardRedisKey, "total_recalls", strconv.Itoa(d.TotalRecalls))
_ = redisPersistence.HSet(dashboardRedisKey, "hit_count", strconv.Itoa(d.HitCount))
_ = redisPersistence.HSet(dashboardRedisKey, "closed_gaps", strconv.Itoa(d.ClosedGaps))
_ = redisPersistence.HSet(dashboardRedisKey, "total_gaps", strconv.Itoa(d.TotalGaps))
_ = redisPersistence.HSet(dashboardRedisKey, "cascade_fixed_total", strconv.Itoa(d.CascadeFixedTotal))
_ = redisPersistence.HSet(dashboardRedisKey, "total_fixes", strconv.Itoa(d.TotalFixes))
_ = redisPersistence.HSet(dashboardRedisKey, "deprecated_today", strconv.Itoa(d.DeprecatedToday))
_ = redisPersistence.HSet(dashboardRedisKey, "distill_loss_sum", strconv.FormatFloat(d.DistillLossSum, 'f', 4, 64))
_ = redisPersistence.HSet(dashboardRedisKey, "distill_loss_count", strconv.Itoa(d.DistillLossCount))
_ = redisPersistence.HSet(dashboardRedisKey, "distill_success_count", strconv.Itoa(d.DistillSuccessCount))
_ = redisPersistence.HSet(dashboardRedisKey, "auto_resolved_conflicts", strconv.Itoa(d.AutoResolvedConflicts))
_ = redisPersistence.HSet(dashboardRedisKey, "total_conflicts", strconv.Itoa(d.TotalConflicts))
_ = redisPersistence.Expire(dashboardRedisKey, 90*24*time.Hour) // 90 天 TTL
}
// loadDashboard 从 Redis 加载 Dashboard 数据
func loadDashboard() error {
if redisPersistence == nil {
return nil
}
data, err := redisPersistence.HGetAll(dashboardRedisKey)
if err != nil || len(data) == 0 {
return err
}
Dash.mu.Lock()
defer Dash.mu.Unlock()
if v, err := strconv.Atoi(data["useful_count"]); err == nil {
Dash.UsefulCount = v
}
if v, err := strconv.Atoi(data["not_useful_count"]); err == nil {
Dash.NotUsefulCount = v
}
if v, err := strconv.Atoi(data["total_recalls"]); err == nil {
Dash.TotalRecalls = v
}
if v, err := strconv.Atoi(data["hit_count"]); err == nil {
Dash.HitCount = v
}
if v, err := strconv.Atoi(data["closed_gaps"]); err == nil {
Dash.ClosedGaps = v
}
if v, err := strconv.Atoi(data["total_gaps"]); err == nil {
Dash.TotalGaps = v
}
if v, err := strconv.Atoi(data["cascade_fixed_total"]); err == nil {
Dash.CascadeFixedTotal = v
}
if v, err := strconv.Atoi(data["total_fixes"]); err == nil {
Dash.TotalFixes = v
}
if v, err := strconv.Atoi(data["deprecated_today"]); err == nil {
Dash.DeprecatedToday = v
}
if v, err := strconv.ParseFloat(data["distill_loss_sum"], 64); err == nil {
Dash.DistillLossSum = v
}
if v, err := strconv.Atoi(data["distill_loss_count"]); err == nil {
Dash.DistillLossCount = v
}
if v, err := strconv.Atoi(data["distill_success_count"]); err == nil {
Dash.DistillSuccessCount = v
}
if v, err := strconv.Atoi(data["auto_resolved_conflicts"]); err == nil {
Dash.AutoResolvedConflicts = v
}
if v, err := strconv.Atoi(data["total_conflicts"]); err == nil {
Dash.TotalConflicts = v
}
return nil
}
// ─── 自优化仪表盘 ────────────────────────────────────────
type Dashboard struct {
mu sync.RWMutex
UsefulCount int `json:"useful_count"`
NotUsefulCount int `json:"not_useful_count"`
TotalRecalls int `json:"total_recalls"`
HitCount int `json:"hit_count"`
ClosedGaps int `json:"closed_gaps"`
TotalGaps int `json:"total_gaps"`
CascadeFixedTotal int `json:"cascade_fixed_total"`
TotalFixes int `json:"total_fixes"`
DeprecatedToday int `json:"deprecated_today"`
DistillLossSum float64 `json:"distill_loss_sum"`
DistillLossCount int `json:"distill_loss_count"`
DistillSuccessCount int `json:"distill_success_count"`
AutoResolvedConflicts int `json:"auto_resolved_conflicts"`
TotalConflicts int `json:"total_conflicts"`
}
var Dash = &Dashboard{}
// metricNA 是「无数据」哨兵值。
//
// 2026-10-14(牧尘批准):比率型指标在**分母退化**时(样本数为 0,或负样本通道
// 恒空)此前返回 1 —— 被下游读成「100% 好用/100% 命中」。那不是满分,是失明:
// NotUseful=0 只说明「没人说过不好」,不说明「全都好」;
// HitCount/TotalRecalls 的分子来自 core.go 的 `len(results) > 0`,
// 量的是「召回有没有返回东西」,不是「召回对不对」。
//
// 现在统一返回 -1:真实比率必落在 [0,1],-1 不可能被误读成任何一档成绩。
// 消费方规则(配套键,同 map 返回):
//
// <name>_valid = 1 表示该比率可解读;0 表示值必为 -1、**禁止**当成绩用
// <name>_samples = 该比率的分母样本数(0 时即退化)
//
// 已有消费方均按 key 存在性取值(SyncFromDashboard / 各 JSON 端点),
// 本改动**只增键、不改键名**,故不破坏任何调用方。
const metricNA = -1.0
// rateOrNA 分母 > 0 才算比率(保留两位),否则返回 metricNA。
func rateOrNA(num, den int) float64 {
if den <= 0 {
return metricNA
}
return math.Round(float64(num)/float64(den)*100) / 100
}
// minRateSamples 是率类指标的最小样本门。
//
// 样本太少(< 20)时,即使比率为真值也**统计上不可解读** —— 例如 auto_resolve_rate
// 曾仅 2 个样本即判 valid=1 满分。现在样本不足一律按退化处理(metricNA + valid=0),
// 不参与面板解读,避免「小样本满分」误导。
const minRateSamples = 20
// b2f 布尔转 0/1,用于 *_valid 键。
func b2f(b bool) float64 {
if b {
return 1
}
return 0
}
// Metrics 返回核心指标。
//
// 比率项在分母退化时返回 metricNA(-1) 而非 1;每项比率都配 *_valid / *_samples。
// 计数项(deprecated_per_day)与均值项(avg_distill_loss)同规则。
func (d *Dashboard) Metrics() map[string]float64 {
d.mu.RLock()
defer d.mu.RUnlock()
// 负样本通道为空 ⇒ 无从判断好坏(详见 metricNA 注释)。
usefulRate := metricNA
usefulSamples := d.UsefulCount + d.NotUsefulCount
if d.NotUsefulCount > 0 {
usefulRate = rateOrNA(d.UsefulCount, usefulSamples)
}
// recall_hit_rate 已于 2026-10-17 退役:其分子原本是 core.go 的 `len(results) > 0`,
// 量的是「召回有没有返回东西」而非「返回的对不对」,恒为 1、无区分度。
// 「返回且被使用」的语义已由 recall_usefulness_rate 承担(198 样本 0.98)。
// 禁止保留恒等于 1 的指标挂在面板上误导人 ⇒ 从 Metrics() 移除该键。
gapRate := rateOrNA(d.ClosedGaps, d.TotalGaps)
cascadeRate := rateOrNA(d.CascadeFixedTotal, d.TotalFixes)
autoRate := rateOrNA(d.AutoResolvedConflicts, d.TotalConflicts)
// gap_closure_rate 断言:ClosedGaps/TotalGaps 统一为「缺口个数」口径后,比率必落 [0,1]。
// 若写入侧再次把「关闭事件数」混进来导致越界,断言拦截 → -1 + valid=0(禁止越界值上板)。
if gapRate > 1 {
gapRate = metricNA
}
// 均值项:零样本的均值同样是退化,按同一规则返回 metricNA。
avgLoss := metricNA
if d.DistillLossCount > 0 {
avgLoss = math.Round(d.DistillLossSum/float64(d.DistillLossCount)*100) / 100
}
// 最小样本门:samples < minRateSamples ⇒ 值 = -1 且 valid = 0,不参与面板解读。
usefulVal, usefulValid := usefulRate, b2f(usefulSamples >= minRateSamples && d.NotUsefulCount > 0)
gapVal, gapValid := gapRate, b2f(d.TotalGaps >= minRateSamples)
cascadeVal, cascadeValid := cascadeRate, b2f(d.TotalFixes >= minRateSamples)
autoVal, autoValid := autoRate, b2f(d.TotalConflicts >= minRateSamples)
lossVal, lossValid := avgLoss, b2f(d.DistillLossCount >= minRateSamples)
if usefulSamples < minRateSamples || d.NotUsefulCount == 0 {
usefulVal = metricNA
}
if d.TotalGaps < minRateSamples || gapVal == metricNA {
// 样本不足 或 越界断言触发 ⇒ 值 -1 且 valid=0(值与 flag 保持同步)。
gapVal = metricNA
gapValid = 0
}
if d.TotalFixes < minRateSamples {
cascadeVal = metricNA
}
if d.TotalConflicts < minRateSamples {
autoVal = metricNA
}
if d.DistillLossCount < minRateSamples {
lossVal = metricNA
}
return map[string]float64{
"recall_usefulness_rate": usefulVal,
"recall_usefulness_samples": float64(usefulSamples),
"recall_usefulness_negative": float64(d.NotUsefulCount),
"recall_usefulness_positive": float64(d.UsefulCount),
"recall_usefulness_valid": usefulValid,
"gap_closure_rate": gapVal,
"gap_closure_samples": float64(d.TotalGaps),
"gap_closure_valid": gapValid,
"cascade_fix_rate": cascadeVal,
"cascade_fix_samples": float64(d.TotalFixes),
"cascade_fix_valid": cascadeValid,
"auto_resolve_rate": autoVal,
"auto_resolve_samples": float64(d.TotalConflicts),
"auto_resolve_valid": autoValid,
"deprecated_per_day": float64(d.DeprecatedToday),
"distill_success_count": float64(d.DistillSuccessCount),
"avg_distill_loss": lossVal,
"avg_distill_loss_samples": float64(d.DistillLossCount),
"avg_distill_loss_valid": lossValid,
}
}
func (d *Dashboard) RecordRecall(hit bool) {
d.mu.Lock()
defer d.mu.Unlock()
d.TotalRecalls++
if hit {
d.HitCount++
}
d.persist()
}
func (d *Dashboard) RecordFeedback(useful bool) {
d.mu.Lock()
defer d.mu.Unlock()
if useful {
d.UsefulCount++
} else {
d.NotUsefulCount++
}
d.persist() // 2026-09-27 补:此处漏了 persist() ⇒ useful/not_useful 从不落盘、重启即丢
}
// RecordUseful 记录有用反馈
func (d *Dashboard) RecordUseful() {
d.RecordFeedback(true)
d.persist()
}
// RecordNotUseful 记录无用反馈
func (d *Dashboard) RecordNotUseful() {
d.RecordFeedback(false)
d.persist()
}
// RecordDistillSuccess 记录蒸馏成功次数(独立于用户反馈,2026-10-14 t_c06ad4d0 解污染)。
// 原先蒸馏回调误调 RecordUseful() ⇒ 把「蒸馏成功次数」灌进 recall_usefulness_rate 的分子,
// 与分母(用户 not-useful 反馈)不同源,比率在数学上不成立、恒被蒸馏数抬高到 ~1.0。
// 现拆成独立计数 distill_success_count,recall_usefulness_rate 只由 [useful|not-useful] 反馈驱动。
func (d *Dashboard) RecordDistillSuccess() {
d.mu.Lock()
defer d.mu.Unlock()
d.DistillSuccessCount++
d.persist()
}
// QualityScore 计算当前质量分数
func (d *Dashboard) QualityScore() float64 {
d.mu.RLock()
defer d.mu.RUnlock()
total := d.UsefulCount + d.NotUsefulCount
if total == 0 {
return 0.5
}
return float64(d.UsefulCount) / float64(total)
}
// ─── 知识缺口检测 ────────────────────────────────────────
type GapType string
const (
GapUnknown GapType = "A" // 真不知道
GapSynonym GapType = "B" // 同义词不匹配
GapRecallFailed GapType = "C" // 召回失败
GapFragmented GapType = "D" // 碎片化
)
type Gap struct {
Topic string `json:"topic"`
Type GapType `json:"type"`
MissCount int `json:"miss_count"`
CreatedAt time.Time `json:"created_at"`
Closed bool `json:"closed"`
}
type GapDetector struct {
mu sync.RWMutex
gaps map[string]*Gap
misses map[string]int
// 向量比较引擎(用于缺口分类)
embedder *storage.Embedder
ldb storage.LanceDB
// Redis 持久化
redisClient *storage.RedisClient
}
// gapRedisKey 缺口持久化 Redis key
const gapRedisKey = "zhiyi:gaps"
const gapMissRedisKey = "zhiyi:gap_misses"
// EnableGapRedisPersistence 启动缺口 Redis 持久化
func (gd *GapDetector) EnableGapRedisPersistence() {
rc := storage.GetRedisClient()
if rc == nil {
return
}
gd.redisClient = rc
gd.loadFromRedis()
}
func (gd *GapDetector) loadFromRedis() {
if gd.redisClient == nil {
return
}
// 加载 gaps
data, err := gd.redisClient.HGetAll(gapRedisKey)
if err == nil && len(data) > 0 {
for topic, jsonStr := range data {
var gap Gap
if err := json.Unmarshal([]byte(jsonStr), &gap); err == nil {
gd.gaps[topic] = &gap
}
}
}
// 加载 misses
missData, err := gd.redisClient.HGetAll(gapMissRedisKey)
if err == nil && len(missData) > 0 {
for topic, countStr := range missData {
if count, err := strconv.Atoi(countStr); err == nil {
gd.misses[topic] = count
}
}
}
}
func (gd *GapDetector) persistGaps() {
if gd.redisClient == nil {
return
}
for topic, gap := range gd.gaps {
data, _ := json.Marshal(gap)
gd.redisClient.HSet(gapRedisKey, topic, string(data))
}
gd.redisClient.Expire(gapRedisKey, 90*24*time.Hour)
}
func (gd *GapDetector) persistMisses() {
if gd.redisClient == nil {
return
}
for topic, count := range gd.misses {
gd.redisClient.HSet(gapMissRedisKey, topic, strconv.Itoa(count))
}
gd.redisClient.Expire(gapMissRedisKey, 90*24*time.Hour)
}
func NewGapDetector(emb *storage.Embedder, ldb storage.LanceDB) *GapDetector {
return &GapDetector{
gaps: make(map[string]*Gap),
misses: make(map[string]int),
embedder: emb,
ldb: ldb,
}
}
// RecordMiss 记录一次召回失败
// gapThreshold 是「同一 topic 累计几次 miss 才生成 gap」的阈值。
// 由 routes 的调参器经 SetGapThreshold 推送;安全边界 [2,10]:
// 下界 2 防「一次偶然即成缺口」把缺口表灌爆,上界 10 防阈值大到永不生成(闭环静默死掉)。
var gapThreshold atomic.Int64
func init() { gapThreshold.Store(3) }
// SetGapThreshold 由调参器推送新的缺口阈值(越界值一律忽略)。
func SetGapThreshold(n int) {
if n < 2 || n > 10 {
return
}
gapThreshold.Store(int64(n))
}
// GapThreshold 返回当前缺口阈值。
func GapThreshold() int { return int(gapThreshold.Load()) }
func (gd *GapDetector) RecordMiss(topic string) *Gap {
gd.mu.Lock()
defer gd.mu.Unlock()
gd.misses[topic]++
gd.persistMisses()
// 2026-10-14:阈值改读调参器推送值(默认 3,与原硬编码一致 ⇒ 行为不变)。
// 此前 Tuner.GapThreshold 没有任何读者 ⇒ 调参改了也不生效。
if gd.misses[topic] >= GapThreshold() {
if _, exists := gd.gaps[topic]; !exists {
gap := &Gap{
Topic: topic,
Type: gd.classifyGap(topic),
MissCount: gd.misses[topic],
CreatedAt: time.Now(),
}
gd.gaps[topic] = gap
gd.persistGaps()
Dash.TotalGaps++ // 同步仪表盘
return gap
}
}
return nil
}
// classifyGap 基于向量比较的缺口分类(与设计文档一致)
//
// sim > 0.85 且 category 相近 → Type C(召回失败)
// sim > 0.75 但 entity 名称不同 → Type B(同义词不匹配)
// max sim < 0.3(全聚类 centroid)→ Type A(真未知)
// 多个 L1 记忆各自部分覆盖 → Type D(碎片化)
func (gd *GapDetector) classifyGap(topic string) GapType {
if gd.embedder == nil || gd.ldb == nil {
// 降级:无 embedder 时用简单启发式
// 英文大写开头 → 可能是同义词未收录
for _, r := range topic {
if r >= 'A' && r <= 'Z' {
return GapSynonym
}
}
// 中文:检查是否含常见同义词标记(又称、别名、aka、aka)
if containsAny(topic, []string{"又称", "别名", "也就是", "aka", "AKA", "同义词"}) {
return GapSynonym
}
// 含中文字符但无同义词标记 → 真未知
return GapUnknown
}
vec, err := gd.embedder.EncodeSingle(topic)
if err != nil {
return GapUnknown
}
// ANN 搜索 top-5 最近记忆
results, err := gd.ldb.Search("memories", vec, 5, "")
if err != nil || len(results) == 0 {
return GapUnknown
}
maxSim := results[0].QualityScore // 复用 quality_score 字段存向量相似度
if maxSim > 0.85 {
// 存在高度相似 → 召回失败(应调整 top_k/diversity)
return GapRecallFailed
}
if maxSim > 0.75 {
// 有相似但名称不同 → 同义词/别称问题
return GapSynonym
}
// 检查多个 L1 是否有部分覆盖 → 碎片化
if len(results) >= 3 {
partialCover := 0
for _, r := range results {
if r.QualityScore > 0.5 && r.QualityScore < 0.7 {
partialCover++
}
}
if partialCover >= 2 {
return GapFragmented
}
}
// 最大相似度 < 0.3 → 真未知
if maxSim < 0.3 {
return GapUnknown
}
// 兜底
return GapUnknown
}
func (gd *GapDetector) List() []*Gap {
gd.mu.RLock()
defer gd.mu.RUnlock()
var result []*Gap
for _, g := range gd.gaps {
result = append(result, g)
}
return result
}
func (gd *GapDetector) Close(topic string) {
gd.mu.Lock()
defer gd.mu.Unlock()
// 统一「缺口个数」口径:仅当缺口从未关闭时才算一次闭环(与 RecordHit 一致),
// 否则同一缺口被 Close 重复调用会让 ClosedGaps 虚增、gap_closure_rate 越界 > 1。
if g, ok := gd.gaps[topic]; ok && !g.Closed {
g.Closed = true
gd.persistGaps()
Dash.RecordGapClosed() // 同步仪表盘
}
}
// RecordHit 记录一次召回命中,自动关闭该 topic 的 open gap
func (gd *GapDetector) RecordHit(topic string) {
gd.mu.Lock()
defer gd.mu.Unlock()
if g, ok := gd.gaps[topic]; ok && !g.Closed {
g.Closed = true
gd.persistGaps()
Dash.RecordGapClosed()
}
}
// ClearMisses 清除某个 topic 的 miss 计数(recall 命中后调用)
func (gd *GapDetector) ClearMisses(topic string) {
gd.mu.Lock()
defer gd.mu.Unlock()
delete(gd.misses, topic)
gd.persistMisses()
}
// ─── 全局 GapDetector 访问(供 routes 包调用)───────────────
var globalGapDetector *GapDetector
func SetGlobalGapDetector(gd *GapDetector) { globalGapDetector = gd }
func GetGlobalGapDetector() *GapDetector { return globalGapDetector }
// ─── 因果追踪 ────────────────────────────────────────────
type TraceEntry struct {
MemoryID string `json:"memory_id"`
Version int `json:"version"`
Content string `json:"content"`
Source string `json:"source"` // muchen_oral / config_parse / agent_infer / llm_distill
Trigger string `json:"trigger"` // 什么触发了这次修改
UpdatedAt time.Time `json:"updated_at"`
}
type CausalTracker struct {
mu sync.RWMutex
entries map[string][]*TraceEntry // memory_id → version history
deps map[string][]string // memory_id → depends_on[]
// Redis 持久化
redisClient *storage.RedisClient
}
const causalRedisKey = "zhiyi:causal_entries"
// EnableCausalRedisPersistence 启动因果追踪 Redis 持久化
func (ct *CausalTracker) EnableCausalRedisPersistence() {
rc := storage.GetRedisClient()
if rc == nil {
return
}
ct.redisClient = rc
ct.loadCausalFromRedis()
}
func (ct *CausalTracker) loadCausalFromRedis() {
if ct.redisClient == nil {
return
}
data, err := ct.redisClient.HGetAll(causalRedisKey)
if err != nil || len(data) == 0 {
return
}
// entries: memory_id → JSON数组
// deps: memory_id → JSON数组
ct.mu.Lock()
defer ct.mu.Unlock()
for key, jsonStr := range data {
if len(key) > 5 && key[:5] == "dep_:" {
// dep_:memory_id → [dependency_ids]
memID := key[5:]
var deps []string
if json.Unmarshal([]byte(jsonStr), &deps) == nil {
ct.deps[memID] = deps
}
} else {
// memory_id → [TraceEntry]
var entries []*TraceEntry
if json.Unmarshal([]byte(jsonStr), &entries) == nil {
ct.entries[key] = entries
}
}
}
}
func (ct *CausalTracker) persistCausal() {
if ct.redisClient == nil {
return
}
for memID, entries := range ct.entries {
data, _ := json.Marshal(entries)
ct.redisClient.HSet(causalRedisKey, memID, string(data))
}
for memID, deps := range ct.deps {
data, _ := json.Marshal(deps)
ct.redisClient.HSet(causalRedisKey, "dep_:"+memID, string(data))
}
ct.redisClient.Expire(causalRedisKey, 90*24*time.Hour)
}
func NewCausalTracker() *CausalTracker {
return &CausalTracker{
entries: make(map[string][]*TraceEntry),
deps: make(map[string][]string),
}
}
// RecordVersion 记录版本变更
func (ct *CausalTracker) RecordVersion(memoryID, content, source, trigger string) {
ct.mu.Lock()
defer ct.mu.Unlock()
entry := &TraceEntry{
MemoryID: memoryID,
Version: len(ct.entries[memoryID]) + 1,
Content: content,
Source: source,
Trigger: trigger,
UpdatedAt: time.Now(),
}
ct.entries[memoryID] = append(ct.entries[memoryID], entry)
ct.persistCausal()
}
// AddDependency A depends_on B
func (ct *CausalTracker) AddDependency(a, b string) {
ct.mu.Lock()
defer ct.mu.Unlock()
ct.deps[a] = append(ct.deps[a], b)
ct.persistCausal()
}
// GetAffected 当 memoryID 被修正时,返回所有依赖它的记忆
func (ct *CausalTracker) GetAffected(memoryID string, visited map[string]bool) []string {
ct.mu.RLock()
defer ct.mu.RUnlock()
if visited == nil {
visited = make(map[string]bool)
}
if visited[memoryID] {
return nil
}
visited[memoryID] = true
var affected []string
for dependent, deps := range ct.deps {
for _, d := range deps {
if d == memoryID && !visited[dependent] {
affected = append(affected, dependent)
affected = append(affected, ct.GetAffected(dependent, visited)...)
}
}
}
return affected
}
// SourceTrust 来源信任度
func SourceTrust(source string) float64 {
switch source {
case "muchen_oral":
return 1.0
case "muchen_feishu":
return 0.95
case "config_parse":
return 0.7
case "agent_infer":
return 0.5
case "llm_distill":
return 0.4
default:
return 0.3
}
}
// IsVolatile 判断一条记忆是否易变(频繁修正)
func (ct *CausalTracker) IsVolatile(memoryID string) bool {
ct.mu.RLock()
defer ct.mu.RUnlock()
return len(ct.entries[memoryID]) >= 3
}
// Entries 返回全部版本历史(只读)
func (ct *CausalTracker) Entries() map[string][]*TraceEntry {
ct.mu.RLock()
defer ct.mu.RUnlock()
cp := make(map[string][]*TraceEntry, len(ct.entries))
for k, v := range ct.entries {
cp[k] = v
}
return cp
}
// ─── 记忆预取 ────────────────────────────────────────────
type PrefetchGraph struct {
mu sync.RWMutex
coOccurs map[string]map[string]int // A → {B: count}
}
func NewPrefetchGraph() *PrefetchGraph {
return &PrefetchGraph{coOccurs: make(map[string]map[string]int)}
}
func (pg *PrefetchGraph) RecordCoAccess(a, b string) {
pg.mu.Lock()
defer pg.mu.Unlock()
if pg.coOccurs[a] == nil {
pg.coOccurs[a] = make(map[string]int)
}
pg.coOccurs[a][b]++
}
// GetPrefetch 获取某个 query 的预取候选项
// RecordDeprecation 记录废弃操作
func (d *Dashboard) RecordDeprecation() {
d.mu.Lock()
defer d.mu.Unlock()
d.DeprecatedToday++
d.persist()
}
// RecordCorrection 记录用户修正
func (d *Dashboard) RecordCorrection(source string) {
d.mu.Lock()
defer d.mu.Unlock()
d.TotalFixes++
if source == "muchen_correction" {
d.CascadeFixedTotal++
}
d.persist()
}
// RecordGapClosed 记录缺口关闭
func (d *Dashboard) RecordGapClosed() {
d.mu.Lock()
defer d.mu.Unlock()
d.ClosedGaps++
d.persist()
}
// RecordConflictResolved 记录冲突解决
func (d *Dashboard) RecordConflictResolved(auto bool) {
d.mu.Lock()
defer d.mu.Unlock()
d.TotalConflicts++
if auto {
d.AutoResolvedConflicts++
}
d.persist()
}
// RecordDistillLoss 记录蒸馏质量损失
func (d *Dashboard) RecordDistillLoss(loss float64) {
d.mu.Lock()
defer d.mu.Unlock()
d.DistillLossSum += loss
d.DistillLossCount++
d.persist()
}
func (pg *PrefetchGraph) GetPrefetch(query string) []string {
pg.mu.RLock()
defer pg.mu.RUnlock()
related := pg.coOccurs[query]
if related == nil {
return nil
}
var result []string
for topic, count := range related {
total := 0
for _, c := range pg.coOccurs[query] {
total += c
}
prob := float64(count) / float64(total)
if prob > 0.6 {
result = append(result, topic)
}
}
return result
}
// containsAny 检查字符串是否包含任意一个子串(无 strings 依赖)
func containsAny(s string, substrs []string) bool {
for _, sub := range substrs {
if len(sub) == 0 {
continue
}
for i := 0; i <= len(s)-len(sub); i++ {
match := true
for j := 0; j < len(sub); j++ {
if s[i+j] != sub[j] {
match = false
break
}
}
if match {
return true
}
}
}
return false
}