memoryweave/go/internal/selfoptimize/selfoptimize.go

// 织忆 MemoryWeave — 自优化引擎
package selfoptimize

import (
	"encoding/json"
	"math"
	"strconv"
	"sync"
	"time"

	"github.com/xiaoxue/memoryweave/internal/storage"
)

// Redis persistence key
const dashboardRedisKey = "zhiyi:dashboard"

// redisPersistence 是否开启了 Redis 持久化（由 server.go 在启动时设置）
var redisPersistence *storage.RedisClient

// EnableRedisPersistence 启动 Dashboard 持久化（从 Redis 加载 + 每次变更同步）
func EnableRedisPersistence() {
	rc := storage.GetRedisClient()
	if rc == nil {
		return
	}
	redisPersistence = rc
	_ = loadDashboard()
}

// persist 写当前 Dashboard 到 Redis HASH
// 注意：调用方必须已持有 d.mu 锁（RLock 或 Lock）
func (d *Dashboard) persist() {
	if redisPersistence == nil {
		return
	}
	_ = redisPersistence.HSet(dashboardRedisKey, "useful_count", strconv.Itoa(d.UsefulCount))
	_ = redisPersistence.HSet(dashboardRedisKey, "not_useful_count", strconv.Itoa(d.NotUsefulCount))
	_ = redisPersistence.HSet(dashboardRedisKey, "total_recalls", strconv.Itoa(d.TotalRecalls))
	_ = redisPersistence.HSet(dashboardRedisKey, "hit_count", strconv.Itoa(d.HitCount))
	_ = redisPersistence.HSet(dashboardRedisKey, "closed_gaps", strconv.Itoa(d.ClosedGaps))
	_ = redisPersistence.HSet(dashboardRedisKey, "total_gaps", strconv.Itoa(d.TotalGaps))
	_ = redisPersistence.HSet(dashboardRedisKey, "cascade_fixed_total", strconv.Itoa(d.CascadeFixedTotal))
	_ = redisPersistence.HSet(dashboardRedisKey, "total_fixes", strconv.Itoa(d.TotalFixes))
	_ = redisPersistence.HSet(dashboardRedisKey, "deprecated_today", strconv.Itoa(d.DeprecatedToday))
	_ = redisPersistence.HSet(dashboardRedisKey, "distill_loss_sum", strconv.FormatFloat(d.DistillLossSum, 'f', 4, 64))
	_ = redisPersistence.HSet(dashboardRedisKey, "distill_loss_count", strconv.Itoa(d.DistillLossCount))
	_ = redisPersistence.HSet(dashboardRedisKey, "auto_resolved_conflicts", strconv.Itoa(d.AutoResolvedConflicts))
	_ = redisPersistence.HSet(dashboardRedisKey, "total_conflicts", strconv.Itoa(d.TotalConflicts))
	_ = redisPersistence.Expire(dashboardRedisKey, 90*24*time.Hour) // 90 天 TTL
}

// loadDashboard 从 Redis 加载 Dashboard 数据
func loadDashboard() error {
	if redisPersistence == nil {
		return nil
	}
	data, err := redisPersistence.HGetAll(dashboardRedisKey)
	if err != nil || len(data) == 0 {
		return err
	}
	Dash.mu.Lock()
	defer Dash.mu.Unlock()

	if v, err := strconv.Atoi(data["useful_count"]); err == nil { Dash.UsefulCount = v }
	if v, err := strconv.Atoi(data["not_useful_count"]); err == nil { Dash.NotUsefulCount = v }
	if v, err := strconv.Atoi(data["total_recalls"]); err == nil { Dash.TotalRecalls = v }
	if v, err := strconv.Atoi(data["hit_count"]); err == nil { Dash.HitCount = v }
	if v, err := strconv.Atoi(data["closed_gaps"]); err == nil { Dash.ClosedGaps = v }
	if v, err := strconv.Atoi(data["total_gaps"]); err == nil { Dash.TotalGaps = v }
	if v, err := strconv.Atoi(data["cascade_fixed_total"]); err == nil { Dash.CascadeFixedTotal = v }
	if v, err := strconv.Atoi(data["total_fixes"]); err == nil { Dash.TotalFixes = v }
	if v, err := strconv.Atoi(data["deprecated_today"]); err == nil { Dash.DeprecatedToday = v }
	if v, err := strconv.ParseFloat(data["distill_loss_sum"], 64); err == nil { Dash.DistillLossSum = v }
	if v, err := strconv.Atoi(data["distill_loss_count"]); err == nil { Dash.DistillLossCount = v }
	if v, err := strconv.Atoi(data["auto_resolved_conflicts"]); err == nil { Dash.AutoResolvedConflicts = v }
	if v, err := strconv.Atoi(data["total_conflicts"]); err == nil { Dash.TotalConflicts = v }
	return nil
}

// ─── 自优化仪表盘 ────────────────────────────────────────

type Dashboard struct {
	mu                   sync.RWMutex
	UsefulCount          int     `json:"useful_count"`
	NotUsefulCount       int     `json:"not_useful_count"`
	TotalRecalls         int     `json:"total_recalls"`
	HitCount             int     `json:"hit_count"`
	ClosedGaps           int     `json:"closed_gaps"`
	TotalGaps            int     `json:"total_gaps"`
	CascadeFixedTotal    int     `json:"cascade_fixed_total"`
	TotalFixes           int     `json:"total_fixes"`
	DeprecatedToday       int     `json:"deprecated_today"`
	DistillLossSum       float64 `json:"distill_loss_sum"`
	DistillLossCount     int     `json:"distill_loss_count"`
	AutoResolvedConflicts int    `json:"auto_resolved_conflicts"`
	TotalConflicts       int     `json:"total_conflicts"`
}

var Dash = &Dashboard{}

// Metrics 返回 7 项核心指标
func (d *Dashboard) Metrics() map[string]float64 {
	d.mu.RLock()
	defer d.mu.RUnlock()

	usefulRate := 0.0
	if d.UsefulCount+d.NotUsefulCount > 0 {
		usefulRate = float64(d.UsefulCount) / float64(d.UsefulCount+d.NotUsefulCount)
	}

	hitRate := 0.0
	if d.TotalRecalls > 0 {
		hitRate = float64(d.HitCount) / float64(d.TotalRecalls)
	}

	gapRate := 0.0
	if d.TotalGaps > 0 {
		gapRate = float64(d.ClosedGaps) / float64(d.TotalGaps)
	}

	cascadeRate := 0.0
	if d.TotalFixes > 0 {
		cascadeRate = float64(d.CascadeFixedTotal) / float64(d.TotalFixes)
	}

	avgLoss := 0.0
	if d.DistillLossCount > 0 {
		avgLoss = d.DistillLossSum / float64(d.DistillLossCount)
	}

	autoRate := 0.0
	if d.TotalConflicts > 0 {
		autoRate = float64(d.AutoResolvedConflicts) / float64(d.TotalConflicts)
	}

	return map[string]float64{
		"recall_usefulness_rate": math.Round(usefulRate*100) / 100,
		"recall_hit_rate":        math.Round(hitRate*100) / 100,
		"gap_closure_rate":       math.Round(gapRate*100) / 100,
		"cascade_fix_rate":       math.Round(cascadeRate*100) / 100,
		"deprecated_per_day":     float64(d.DeprecatedToday),
		"avg_distill_loss":       math.Round(avgLoss*100) / 100,
		"auto_resolve_rate":       math.Round(autoRate*100) / 100,
	}
}

func (d *Dashboard) RecordRecall(hit bool) {
	d.mu.Lock()
	defer d.mu.Unlock()
	d.TotalRecalls++
	if hit {
		d.HitCount++
	}
	d.persist()
}

func (d *Dashboard) RecordFeedback(useful bool) {
	d.mu.Lock()
	defer d.mu.Unlock()
	if useful {
		d.UsefulCount++
	} else {
		d.NotUsefulCount++
	}
}

// RecordUseful 记录有用反馈
func (d *Dashboard) RecordUseful() {
	d.RecordFeedback(true)
	d.persist()
}

// RecordNotUseful 记录无用反馈
func (d *Dashboard) RecordNotUseful() {
	d.RecordFeedback(false)
	d.persist()
}

// QualityScore 计算当前质量分数
func (d *Dashboard) QualityScore() float64 {
	d.mu.RLock()
	defer d.mu.RUnlock()
	total := d.UsefulCount + d.NotUsefulCount
	if total == 0 { return 0.5 }
	return float64(d.UsefulCount) / float64(total)
}

// ─── 知识缺口检测 ────────────────────────────────────────

type GapType string

const (
	GapUnknown      GapType = "A" // 真不知道
	GapSynonym      GapType = "B" // 同义词不匹配
	GapRecallFailed GapType = "C" // 召回失败
	GapFragmented   GapType = "D" // 碎片化
)

type Gap struct {
	Topic     string    `json:"topic"`
	Type      GapType   `json:"type"`
	MissCount int       `json:"miss_count"`
	CreatedAt time.Time `json:"created_at"`
	Closed    bool      `json:"closed"`
}

type GapDetector struct {
	mu     sync.RWMutex
	gaps   map[string]*Gap
	misses map[string]int

	// 向量比较引擎（用于缺口分类）
	embedder *storage.Embedder
	ldb      storage.LanceDB

	// Redis 持久化
	redisClient *storage.RedisClient
}

// gapRedisKey 缺口持久化 Redis key
const gapRedisKey = "zhiyi:gaps"
const gapMissRedisKey = "zhiyi:gap_misses"

// EnableGapRedisPersistence 启动缺口 Redis 持久化
func (gd *GapDetector) EnableGapRedisPersistence() {
	rc := storage.GetRedisClient()
	if rc == nil {
		return
	}
	gd.redisClient = rc
	gd.loadFromRedis()
}

func (gd *GapDetector) loadFromRedis() {
	if gd.redisClient == nil {
		return
	}
	// 加载 gaps
	data, err := gd.redisClient.HGetAll(gapRedisKey)
	if err == nil && len(data) > 0 {
		for topic, jsonStr := range data {
			var gap Gap
			if err := json.Unmarshal([]byte(jsonStr), &gap); err == nil {
				gd.gaps[topic] = &gap
			}
		}
	}
	// 加载 misses
	missData, err := gd.redisClient.HGetAll(gapMissRedisKey)
	if err == nil && len(missData) > 0 {
		for topic, countStr := range missData {
			if count, err := strconv.Atoi(countStr); err == nil {
				gd.misses[topic] = count
			}
		}
	}
}

func (gd *GapDetector) persistGaps() {
	if gd.redisClient == nil {
		return
	}
	for topic, gap := range gd.gaps {
		data, _ := json.Marshal(gap)
		gd.redisClient.HSet(gapRedisKey, topic, string(data))
	}
	gd.redisClient.Expire(gapRedisKey, 90*24*time.Hour)
}

func (gd *GapDetector) persistMisses() {
	if gd.redisClient == nil {
		return
	}
	for topic, count := range gd.misses {
		gd.redisClient.HSet(gapMissRedisKey, topic, strconv.Itoa(count))
	}
	gd.redisClient.Expire(gapMissRedisKey, 90*24*time.Hour)
}

func NewGapDetector(emb *storage.Embedder, ldb storage.LanceDB) *GapDetector {
	return &GapDetector{
		gaps:     make(map[string]*Gap),
		misses:   make(map[string]int),
		embedder: emb,
		ldb:      ldb,
	}
}

// RecordMiss 记录一次召回失败
func (gd *GapDetector) RecordMiss(topic string) *Gap {
	gd.mu.Lock()
	defer gd.mu.Unlock()

	gd.misses[topic]++
	gd.persistMisses()
	if gd.misses[topic] >= 3 {
		if _, exists := gd.gaps[topic]; !exists {
			gap := &Gap{
				Topic:     topic,
				Type:      gd.classifyGap(topic),
				MissCount: gd.misses[topic],
				CreatedAt: time.Now(),
			}
			gd.gaps[topic] = gap
			gd.persistGaps()
			Dash.TotalGaps++ // 同步仪表盘
			return gap
		}
	}
	return nil
}

// classifyGap 基于向量比较的缺口分类（与设计文档一致）
//  sim > 0.85 且 category 相近 → Type C（召回失败）
//  sim > 0.75 但 entity 名称不同 → Type B（同义词不匹配）
//  max sim < 0.3（全聚类 centroid）→ Type A（真未知）
//  多个 L1 记忆各自部分覆盖 → Type D（碎片化）
func (gd *GapDetector) classifyGap(topic string) GapType {
	if gd.embedder == nil || gd.ldb == nil {
		// 降级：无 embedder 时用简单启发式
		// 英文大写开头 → 可能是同义词未收录
		for _, r := range topic {
			if r >= 'A' && r <= 'Z' {
				return GapSynonym
			}
		}
		// 中文：检查是否含常见同义词标记（又称、别名、aka、aka）
		if containsAny(topic, []string{"又称", "别名", "也就是", "aka", "AKA", "同义词"}) {
			return GapSynonym
		}
		// 含中文字符但无同义词标记 → 真未知
		return GapUnknown
	}

	vec, err := gd.embedder.EncodeSingle(topic)
	if err != nil {
		return GapUnknown
	}

	// ANN 搜索 top-5 最近记忆
	results, err := gd.ldb.Search("memories", vec, 5, "")
	if err != nil || len(results) == 0 {
		return GapUnknown
	}

	maxSim := results[0].QualityScore // 复用 quality_score 字段存向量相似度
	if maxSim > 0.85 {
		// 存在高度相似 → 召回失败（应调整 top_k/diversity）
		return GapRecallFailed
	}
	if maxSim > 0.75 {
		// 有相似但名称不同 → 同义词/别称问题
		return GapSynonym
	}

	// 检查多个 L1 是否有部分覆盖 → 碎片化
	if len(results) >= 3 {
		partialCover := 0
		for _, r := range results {
			if r.QualityScore > 0.5 && r.QualityScore < 0.7 {
				partialCover++
			}
		}
		if partialCover >= 2 {
			return GapFragmented
		}
	}

	// 最大相似度 < 0.3 → 真未知
	if maxSim < 0.3 {
		return GapUnknown
	}

	// 兜底
	return GapUnknown
}

func (gd *GapDetector) List() []*Gap {
	gd.mu.RLock()
	defer gd.mu.RUnlock()
	var result []*Gap
	for _, g := range gd.gaps {
		result = append(result, g)
	}
	return result
}

func (gd *GapDetector) Close(topic string) {
	gd.mu.Lock()
	defer gd.mu.Unlock()
	if g, ok := gd.gaps[topic]; ok {
		g.Closed = true
		gd.persistGaps()
		Dash.RecordGapClosed() // 同步仪表盘
	}
}

// RecordHit 记录一次召回命中，自动关闭该 topic 的 open gap
func (gd *GapDetector) RecordHit(topic string) {
	gd.mu.Lock()
	defer gd.mu.Unlock()
	if g, ok := gd.gaps[topic]; ok && !g.Closed {
		g.Closed = true
		gd.persistGaps()
		Dash.RecordGapClosed()
	}
}

// ClearMisses 清除某个 topic 的 miss 计数（recall 命中后调用）
func (gd *GapDetector) ClearMisses(topic string) {
	gd.mu.Lock()
	defer gd.mu.Unlock()
	delete(gd.misses, topic)
	gd.persistMisses()
}

// ─── 全局 GapDetector 访问（供 routes 包调用）───────────────

var globalGapDetector *GapDetector

func SetGlobalGapDetector(gd *GapDetector) { globalGapDetector = gd }

func GetGlobalGapDetector() *GapDetector { return globalGapDetector }

// ─── 因果追踪 ────────────────────────────────────────────

type TraceEntry struct {
	MemoryID  string    `json:"memory_id"`
	Version   int       `json:"version"`
	Content   string    `json:"content"`
	Source    string    `json:"source"`    // muchen_oral / config_parse / agent_infer / llm_distill
	Trigger   string    `json:"trigger"`   // 什么触发了这次修改
	UpdatedAt time.Time `json:"updated_at"`
}

type CausalTracker struct {
	mu      sync.RWMutex
	entries map[string][]*TraceEntry // memory_id → version history
	deps    map[string][]string      // memory_id → depends_on[]

	// Redis 持久化
	redisClient *storage.RedisClient
}

const causalRedisKey = "zhiyi:causal_entries"

// EnableCausalRedisPersistence 启动因果追踪 Redis 持久化
func (ct *CausalTracker) EnableCausalRedisPersistence() {
	rc := storage.GetRedisClient()
	if rc == nil {
		return
	}
	ct.redisClient = rc
	ct.loadCausalFromRedis()
}

func (ct *CausalTracker) loadCausalFromRedis() {
	if ct.redisClient == nil {
		return
	}
	data, err := ct.redisClient.HGetAll(causalRedisKey)
	if err != nil || len(data) == 0 {
		return
	}
	// entries: memory_id → JSON数组
	// deps: memory_id → JSON数组
	ct.mu.Lock()
	defer ct.mu.Unlock()
	for key, jsonStr := range data {
		if len(key) > 5 && key[:5] == "dep_:" {
			// dep_:memory_id → [dependency_ids]
			memID := key[5:]
			var deps []string
			if json.Unmarshal([]byte(jsonStr), &deps) == nil {
				ct.deps[memID] = deps
			}
		} else {
			// memory_id → [TraceEntry]
			var entries []*TraceEntry
			if json.Unmarshal([]byte(jsonStr), &entries) == nil {
				ct.entries[key] = entries
			}
		}
	}
}

func (ct *CausalTracker) persistCausal() {
	if ct.redisClient == nil {
		return
	}
	for memID, entries := range ct.entries {
		data, _ := json.Marshal(entries)
		ct.redisClient.HSet(causalRedisKey, memID, string(data))
	}
	for memID, deps := range ct.deps {
		data, _ := json.Marshal(deps)
		ct.redisClient.HSet(causalRedisKey, "dep_:"+memID, string(data))
	}
	ct.redisClient.Expire(causalRedisKey, 90*24*time.Hour)
}

func NewCausalTracker() *CausalTracker {
	return &CausalTracker{
		entries: make(map[string][]*TraceEntry),
		deps:    make(map[string][]string),
	}
}

// RecordVersion 记录版本变更
func (ct *CausalTracker) RecordVersion(memoryID, content, source, trigger string) {
	ct.mu.Lock()
	defer ct.mu.Unlock()

	entry := &TraceEntry{
		MemoryID:  memoryID,
		Version:   len(ct.entries[memoryID]) + 1,
		Content:   content,
		Source:    source,
		Trigger:   trigger,
		UpdatedAt: time.Now(),
	}
	ct.entries[memoryID] = append(ct.entries[memoryID], entry)
	ct.persistCausal()
}

// AddDependency A depends_on B
func (ct *CausalTracker) AddDependency(a, b string) {
	ct.mu.Lock()
	defer ct.mu.Unlock()
	ct.deps[a] = append(ct.deps[a], b)
	ct.persistCausal()
}

// GetAffected 当 memoryID 被修正时，返回所有依赖它的记忆
func (ct *CausalTracker) GetAffected(memoryID string, visited map[string]bool) []string {
	ct.mu.RLock()
	defer ct.mu.RUnlock()

	if visited == nil {
		visited = make(map[string]bool)
	}
	if visited[memoryID] {
		return nil
	}
	visited[memoryID] = true

	var affected []string
	for dependent, deps := range ct.deps {
		for _, d := range deps {
			if d == memoryID && !visited[dependent] {
				affected = append(affected, dependent)
				affected = append(affected, ct.GetAffected(dependent, visited)...)
			}
		}
	}
	return affected
}

// SourceTrust 来源信任度
func SourceTrust(source string) float64 {
	switch source {
	case "muchen_oral":
		return 1.0
	case "muchen_feishu":
		return 0.95
	case "config_parse":
		return 0.7
	case "agent_infer":
		return 0.5
	case "llm_distill":
		return 0.4
	default:
		return 0.3
	}
}

// IsVolatile 判断一条记忆是否易变（频繁修正）
func (ct *CausalTracker) IsVolatile(memoryID string) bool {
	ct.mu.RLock()
	defer ct.mu.RUnlock()
	return len(ct.entries[memoryID]) >= 3
}

// Entries 返回全部版本历史（只读）
func (ct *CausalTracker) Entries() map[string][]*TraceEntry {
	ct.mu.RLock()
	defer ct.mu.RUnlock()
	cp := make(map[string][]*TraceEntry, len(ct.entries))
	for k, v := range ct.entries {
		cp[k] = v
	}
	return cp
}

// ─── 记忆预取 ────────────────────────────────────────────

type PrefetchGraph struct {
	mu       sync.RWMutex
	coOccurs map[string]map[string]int // A → {B: count}
}

func NewPrefetchGraph() *PrefetchGraph {
	return &PrefetchGraph{coOccurs: make(map[string]map[string]int)}
}

func (pg *PrefetchGraph) RecordCoAccess(a, b string) {
	pg.mu.Lock()
	defer pg.mu.Unlock()
	if pg.coOccurs[a] == nil {
		pg.coOccurs[a] = make(map[string]int)
	}
	pg.coOccurs[a][b]++
}

// GetPrefetch 获取某个 query 的预取候选项
// RecordDeprecation 记录废弃操作
func (d *Dashboard) RecordDeprecation() {
	d.mu.Lock()
	defer d.mu.Unlock()
	d.DeprecatedToday++
	d.persist()
}

// RecordCorrection 记录用户修正
func (d *Dashboard) RecordCorrection(source string) {
	d.mu.Lock()
	defer d.mu.Unlock()
	d.TotalFixes++
	if source == "muchen_correction" {
		d.CascadeFixedTotal++
	}
	d.persist()
}

// RecordGapClosed 记录缺口关闭
func (d *Dashboard) RecordGapClosed() {
	d.mu.Lock()
	defer d.mu.Unlock()
	d.ClosedGaps++
	d.persist()
}

// RecordConflictResolved 记录冲突解决
func (d *Dashboard) RecordConflictResolved(auto bool) {
	d.mu.Lock()
	defer d.mu.Unlock()
	d.TotalConflicts++
	if auto {
		d.AutoResolvedConflicts++
	}
	d.persist()
}

// RecordDistillLoss 记录蒸馏质量损失
func (d *Dashboard) RecordDistillLoss(loss float64) {
	d.mu.Lock()
	defer d.mu.Unlock()
	d.DistillLossSum += loss
	d.DistillLossCount++
	d.persist()
}

func (pg *PrefetchGraph) GetPrefetch(query string) []string {
	pg.mu.RLock()
	defer pg.mu.RUnlock()

	related := pg.coOccurs[query]
	if related == nil {
		return nil
	}

	var result []string
	for topic, count := range related {
		total := 0
		for _, c := range pg.coOccurs[query] {
			total += c
		}
		prob := float64(count) / float64(total)
		if prob > 0.6 {
			result = append(result, topic)
		}
	}
	return result
}

// containsAny 检查字符串是否包含任意一个子串（无 strings 依赖）
func containsAny(s string, substrs []string) bool {
	for _, sub := range substrs {
		if len(sub) == 0 {
			continue
		}
		for i := 0; i <= len(s)-len(sub); i++ {
			match := true
			for j := 0; j < len(sub); j++ {
				if s[i+j] != sub[j] {
					match = false
					break
				}
			}
			if match {
				return true
			}
		}
	}
	return false
}