Files
memoryweave/go/internal/selfoptimize/selfoptimize_test.go
T
小唯 b2a016e90f fix(metrics): t_c70f385c 指标口径修复 — recall_hit_rate 退役 + gap_closure_rate 断言 + 最小样本门 N=20
- recall_hit_rate 退役:分子原为 core.go 的 len(results)>0(恒=1、无区分度),
  「返回且被使用」语义已由 recall_usefulness_rate 承担(207 样本 0.98)。
  Metrics()/distributed/metrics-gauge/executor/tuning/CLI/daily-check 全链移除该键。
- gap_closure_rate 统一为「缺口个数」口径:Close() 加 !g.Closed 守卫(与 RecordHit 一致),
  根治同一缺口重复关闭导致 ClosedGaps 虚增、越界 4.5;Metrics() 加越界断言 gapRate>1 → -1。
- 所有率类指标加最小样本门 minRateSamples=20:样本<20 ⇒ -1 + valid=0,
  杜绝 auto_resolve_rate 2 样本判满分。
- 测试:metrics_retire_regression_test.go 用生产同款计数复现改前/改后 JSON;
  selfoptimize 全套通过。
2026-10-17 20:32:34 +08:00

528 lines
16 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// 织忆 MemoryWeave — 自优化层单元测试
package selfoptimize
import (
"os"
"path/filepath"
"testing"
"time"
)
// zeroDash 复位全局 Dash(测试用;逐字段置零以避免复制含 mutex 的结构体)
func zeroDash() {
Dash.mu.Lock()
defer Dash.mu.Unlock()
Dash.UsefulCount = 0
Dash.NotUsefulCount = 0
Dash.TotalRecalls = 0
Dash.HitCount = 0
Dash.ClosedGaps = 0
Dash.TotalGaps = 0
Dash.CascadeFixedTotal = 0
Dash.TotalFixes = 0
Dash.DeprecatedToday = 0
Dash.DistillLossSum = 0
Dash.DistillLossCount = 0
Dash.AutoResolvedConflicts = 0
Dash.TotalConflicts = 0
}
// TestMain 单元测试不得写到生产落盘点(曾实测:整套测试跑完把 /var/lib/memoryweave/dashboard.json 写成了测试数字)
func TestMain(m *testing.M) {
dir, err := os.MkdirTemp("", "selfopt-test-*")
if err != nil {
panic(err)
}
_ = os.Setenv("ZHIYI_DASHBOARD_PATH", filepath.Join(dir, "dashboard.json"))
code := m.Run()
_ = os.RemoveAll(dir)
os.Exit(code)
}
// ─── 仪表盘 ──────────────────────────────────────────────
// TestDashboard_InitialMetricsNoData —— 空盘(分母为 0)的语义于 2026-10-14 变更:
// 旧实现返回 0,0 意为「0% 有用」,同样是一句它没资格下的结论;
// 新语义返回 metricNA(-1) = 「无数据」。断言随之更新(判定:无样本 ⇒ 无结论)。
func TestDashboard_InitialMetricsNoData(t *testing.T) {
d := &Dashboard{}
m := d.Metrics()
if m["recall_usefulness_rate"] != metricNA {
t.Errorf("expected metricNA(%v), got %f", metricNA, m["recall_usefulness_rate"])
}
// 2026-10-17:recall_hit_rate 已退役(恒=1、无区分度),其「返回且被使用」语义由
// recall_usefulness_rate 承担。Metrics() 不再输出该键 ⇒ 断言键不存在(读它应得 0,会误导)。
if _, ok := m["recall_hit_rate"]; ok {
t.Errorf("recall_hit_rate 已退役,Metrics() 不应再输出该键")
}
if m["recall_usefulness_valid"] != 0 {
t.Errorf("expected valid=0, got %f", m["recall_usefulness_valid"])
}
}
func TestDashboard_UsefulnessRate(t *testing.T) {
d := &Dashboard{}
// 满足最小样本门 N=20:15 有用 + 5 无用 = 20 样本 ⇒ 15/20 = 0.75
for i := 0; i < 15; i++ {
d.RecordFeedback(true)
}
for i := 0; i < 5; i++ {
d.RecordFeedback(false)
}
m := d.Metrics()
if m["recall_usefulness_rate"] != 0.75 {
t.Errorf("expected 0.75, got %f", m["recall_usefulness_rate"])
}
}
// TestDashboard_UsefulnessRateBelowMinSamples —— 最小样本门(N=20):反馈样本 < 20 ⇒ -1 + valid=0。
func TestDashboard_UsefulnessRateBelowMinSamples(t *testing.T) {
d := &Dashboard{}
for i := 0; i < 2; i++ { // 仅 2 样本(旧 auto_resolve_rate 同类误判满分)
d.RecordFeedback(true)
}
m := d.Metrics()
if m["recall_usefulness_rate"] != metricNA {
t.Errorf("样本 < 20 应为 metricNA(-1), got %f", m["recall_usefulness_rate"])
}
if m["recall_usefulness_valid"] != 0 {
t.Errorf("样本 < 20 应为 valid=0, got %f", m["recall_usefulness_valid"])
}
}
// TestDashboard_DistillDoesNotPolluteUsefulnessRate —— 2026-10-14 t_c06ad4d0 解污染回归。
// 蒸馏回调改走 RecordDistillSuccess()(独立计数 distill_success_count),
// 不得再污染 recall_usefulness_rate 的分子/分母。10 次蒸馏成功 ⇒ UsefulCount 仍为 0。
func TestDashboard_DistillDoesNotPolluteUsefulnessRate(t *testing.T) {
d := &Dashboard{}
for i := 0; i < 10; i++ {
d.RecordDistillSuccess()
}
if d.DistillSuccessCount != 10 {
t.Fatalf("distill_success_count want 10 got %d", d.DistillSuccessCount)
}
if d.UsefulCount != 0 {
t.Errorf("distill 不得污染 UsefulCount: got %d, want 0", d.UsefulCount)
}
if d.NotUsefulCount != 0 {
t.Errorf("distill 不得污染 NotUsefulCount: got %d, want 0", d.NotUsefulCount)
}
m := d.Metrics()
if m["distill_success_count"] != 10 {
t.Errorf("metric distill_success_count want 10 got %f", m["distill_success_count"])
}
// 无任何用户反馈 ⇒ 比率无样本,仍应返回 metricNA(不可读),而不是被蒸馏数抬高到 1。
if m["recall_usefulness_rate"] != metricNA {
t.Errorf("无反馈样本时 recall_usefulness_rate 应为 metricNA(-1),蒸馏不得抬高它: got %f", m["recall_usefulness_rate"])
}
}
// TestDashboard_HitRateRetired —— 2026-10-17:recall_hit_rate 已退役。
// 其分子原是 core.go 的 `len(results)>0`(恒=1、无区分度),「返回且被使用」语义由
// recall_usefulness_rate 承担。RecordRecall 仍累积 TotalRecalls/HitCount(供持久化/审计),
// 但 Metrics() 不再输出 recall_hit_rate 键。
func TestDashboard_HitRateRetired(t *testing.T) {
d := &Dashboard{}
d.RecordRecall(true)
d.RecordRecall(true)
d.RecordRecall(false)
m := d.Metrics()
if _, ok := m["recall_hit_rate"]; ok {
t.Errorf("recall_hit_rate 已退役,Metrics() 不应再输出该键")
}
if m["recall_hit_samples"] != 0 {
t.Errorf("recall_hit_samples 已退役,不应输出(got %f)", m["recall_hit_samples"])
}
}
func TestDashboard_GapClosureRate(t *testing.T) {
d := &Dashboard{}
d.TotalGaps = 20 // ≥ 最小样本门 N=20
d.ClosedGaps = 10
m := d.Metrics()
if m["gap_closure_rate"] != 0.5 {
t.Errorf("expected 0.5, got %f", m["gap_closure_rate"])
}
}
// TestDashboard_GapClosureBelowMinSamples —— 最小样本门(N=20)+ 越界断言:
// TotalGaps < 20 ⇒ -1 + valid=0;ClosedGaps > TotalGaps(写入侧口径错)⇒ 断言拦截为 -1 + valid=0。
func TestDashboard_GapClosureBelowMinSamples(t *testing.T) {
d := &Dashboard{}
d.TotalGaps = 2
d.ClosedGaps = 1
m := d.Metrics()
if m["gap_closure_rate"] != metricNA {
t.Errorf("TotalGaps < 20 应为 metricNA(-1), got %f", m["gap_closure_rate"])
}
if m["gap_closure_valid"] != 0 {
t.Errorf("TotalGaps < 20 应为 valid=0, got %f", m["gap_closure_valid"])
}
// 越界断言:ClosedGaps > TotalGaps(同类 4.5 越界值)⇒ 拦截为 -1,禁止上板
d2 := &Dashboard{}
d2.TotalGaps = 20
d2.ClosedGaps = 30 // 关闭事件数混入 ⇒ 越界
m2 := d2.Metrics()
if m2["gap_closure_rate"] != metricNA {
t.Errorf("ClosedGaps > TotalGaps 越界应被断言拦截为 metricNA(-1), got %f", m2["gap_closure_rate"])
}
if m2["gap_closure_valid"] != 0 {
t.Errorf("越界时 valid 应为 0, got %f", m2["gap_closure_valid"])
}
}
func TestDashboard_RecordDeprecation(t *testing.T) {
d := &Dashboard{}
d.RecordDeprecation()
d.RecordDeprecation()
if d.DeprecatedToday != 2 {
t.Errorf("expected 2, got %d", d.DeprecatedToday)
}
}
func TestDashboard_RecordCorrection(t *testing.T) {
d := &Dashboard{}
d.RecordCorrection("muchen_correction")
d.RecordCorrection("config_parse")
if d.TotalFixes != 2 {
t.Errorf("expected 2 fixes, got %d", d.TotalFixes)
}
if d.CascadeFixedTotal != 1 {
t.Errorf("expected 1 cascade, got %d", d.CascadeFixedTotal)
}
}
func TestDashboard_RecordConflictResolved(t *testing.T) {
d := &Dashboard{}
// 满足最小样本门 N=20:10 自动 + 10 手动 = 20 冲突 ⇒ 10/20 = 0.5
for i := 0; i < 10; i++ {
d.RecordConflictResolved(true)
}
for i := 0; i < 10; i++ {
d.RecordConflictResolved(false)
}
if d.TotalConflicts != 20 {
t.Errorf("expected 20 conflicts, got %d", d.TotalConflicts)
}
if d.AutoResolvedConflicts != 10 {
t.Errorf("expected 10 auto, got %d", d.AutoResolvedConflicts)
}
m := d.Metrics()
if m["auto_resolve_rate"] != 0.5 {
t.Errorf("expected 0.5, got %f", m["auto_resolve_rate"])
}
}
// ─── 知识缺口检测 ────────────────────────────────────────
func TestGapDetector_NotTriggeredBeforeThreshold(t *testing.T) {
gd := NewGapDetector(nil, nil)
gap := gd.RecordMiss("kubernetes")
if gap != nil {
t.Error("should not trigger gap after 1 miss")
}
gd.RecordMiss("kubernetes")
if gap := gd.RecordMiss("kubernetes"); gap == nil {
t.Error("should trigger gap after 3 misses")
}
}
func TestGapDetector_DetectsGapAtThreshold(t *testing.T) {
gd := NewGapDetector(nil, nil)
gd.RecordMiss("topicX")
gd.RecordMiss("topicX")
gap := gd.RecordMiss("topicX")
if gap == nil {
t.Fatal("expected gap after 3 misses")
}
if gap.Topic != "topicX" {
t.Errorf("wrong topic: %s", gap.Topic)
}
if gap.MissCount != 3 {
t.Errorf("expected 3 misses, got %d", gap.MissCount)
}
}
func TestGapDetector_CloseGap(t *testing.T) {
gd := NewGapDetector(nil, nil)
gd.RecordMiss("topic")
gd.RecordMiss("topic")
gd.RecordMiss("topic")
gd.Close("topic")
gaps := gd.List()
if len(gaps) == 0 || !gaps[0].Closed {
t.Error("gap should be closed")
}
}
func TestGapDetector_ClassifySynonym(t *testing.T) {
gd := NewGapDetector(nil, nil)
gd.RecordMiss("API")
gd.RecordMiss("API")
gd.RecordMiss("API")
gaps := gd.List()
if len(gaps) > 0 && gaps[0].Type != GapSynonym {
t.Errorf("expected synonym gap, got %s", gaps[0].Type)
}
}
// ─── 因果追踪 ────────────────────────────────────────────
func TestCausalTracker_RecordVersions(t *testing.T) {
ct := NewCausalTracker()
ct.RecordVersion("mem1", "Docker is used by Hermes", "muchen_oral", "init")
ct.RecordVersion("mem1", "Docker is used by Hermes AND OpenClaw", "config_parse", "update")
if !ct.IsVolatile("mem1") {
// 2 versions → not volatile (volatile = 3+)
}
ct.RecordVersion("mem1", "Docker usage updated", "muchen_correction", "correction")
if !ct.IsVolatile("mem1") {
t.Error("mem1 should be volatile after 3 versions")
}
}
func TestCausalTracker_DependencyChain(t *testing.T) {
ct := NewCausalTracker()
ct.AddDependency("memory_A", "memory_B")
ct.AddDependency("memory_B", "memory_C")
affected := ct.GetAffected("memory_C", nil)
// memory_B depends on C, memory_A depends on B
foundB := false
foundA := false
for _, id := range affected {
if id == "memory_B" {
foundB = true
}
if id == "memory_A" {
foundA = true
}
}
if !foundB || !foundA {
t.Errorf("expected both A and B affected, got %v", affected)
}
}
func TestCausalTracker_CircularDependency(t *testing.T) {
ct := NewCausalTracker()
ct.AddDependency("A", "B")
ct.AddDependency("B", "A")
affected := ct.GetAffected("A", nil)
// Should not loop infinitely
if len(affected) >= 10 {
t.Errorf("circular dependency not handled: %d affected", len(affected))
}
}
// ─── 来源信任度 ──────────────────────────────────────────
func TestSourceTrust(t *testing.T) {
tests := []struct {
source string
expect float64
}{
{"muchen_oral", 1.0},
{"muchen_feishu", 0.95},
{"config_parse", 0.7},
{"agent_infer", 0.5},
{"llm_distill", 0.4},
{"unknown", 0.3},
}
for _, tc := range tests {
if got := SourceTrust(tc.source); got != tc.expect {
t.Errorf("SourceTrust(%s) = %f, want %f", tc.source, got, tc.expect)
}
}
}
// ─── 记忆预取 ────────────────────────────────────────────
func TestPrefetchGraph_CoOccurrence(t *testing.T) {
pg := NewPrefetchGraph()
pg.RecordCoAccess("docker", "nginx")
pg.RecordCoAccess("docker", "nginx")
pg.RecordCoAccess("docker", "kubernetes")
prefetch := pg.GetPrefetch("docker")
if len(prefetch) < 1 {
t.Errorf("expected at least 1 prefetch item, got %d", len(prefetch))
}
}
func TestPrefetchGraph_NothingToPrefetch(t *testing.T) {
pg := NewPrefetchGraph()
prefetch := pg.GetPrefetch("nonexistent")
if len(prefetch) != 0 {
t.Errorf("expected empty prefetch, got %d", len(prefetch))
}
}
// ─── 并发安全 ────────────────────────────────────────────
func TestDashboard_Concurrent(t *testing.T) {
d := &Dashboard{}
done := make(chan bool)
for i := 0; i < 50; i++ {
go func(v bool) {
for j := 0; j < 100; j++ {
d.RecordFeedback(v)
d.RecordRecall(v)
}
done <- true
}(i%2 == 0)
}
for i := 0; i < 50; i++ {
<-done
}
m := d.Metrics()
if m["recall_usefulness_rate"] < 0 || m["recall_usefulness_rate"] > 1 {
t.Errorf("usefulness rate out of bounds: %f", m["recall_usefulness_rate"])
}
}
// ─── 基准测试 ────────────────────────────────────────────
func BenchmarkDashboardMetrics(b *testing.B) {
d := &Dashboard{}
d.UsefulCount = 1000
d.NotUsefulCount = 200
d.TotalRecalls = 1200
d.HitCount = 900
b.ResetTimer()
for i := 0; i < b.N; i++ {
d.Metrics()
}
}
func BenchmarkGapDetection(b *testing.B) {
gd := NewGapDetector(nil, nil)
b.ResetTimer()
for i := 0; i < b.N; i++ {
gd.RecordMiss("benchmark_topic")
}
}
func BenchmarkCausalTracking(b *testing.B) {
ct := NewCausalTracker()
for i := 0; i < 100; i++ {
ct.AddDependency("dep"+string(rune('a'+i%26)), "dep"+string(rune('a'+(i+1)%26)))
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
ct.GetAffected("depa", nil)
}
}
func BenchmarkPrefetch(b *testing.B) {
pg := NewPrefetchGraph()
for i := 0; i < 100; i++ {
pg.RecordCoAccess("main", "item"+string(rune('a'+i%26)))
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
pg.GetPrefetch("main")
}
}
func init() {
// Ensure consistent timestamps for snapshot tests
time.Local = time.UTC
}
// ─── 本地 JSON 持久化(2026-09-27 加:Redis 不可用时的兜底)──────────────
func TestDashboardLocalPersistence_RoundTrip(t *testing.T) {
p := filepath.Join(t.TempDir(), "dash.json")
t.Setenv("ZHIYI_DASHBOARD_PATH", p)
src := &Dashboard{
TotalRecalls: 7, HitCount: 6, UsefulCount: 15, NotUsefulCount: 5, // 20 样本 ≥ N
DistillLossSum: 4.2, DistillLossCount: 20,
ClosedGaps: 10, TotalGaps: 20, TotalFixes: 30, CascadeFixedTotal: 10,
AutoResolvedConflicts: 20, TotalConflicts: 30, DeprecatedToday: 4,
}
src.mu.Lock()
src.persist() // 测试环境无 Redis → 走本地 JSON 分支
src.mu.Unlock()
if _, err := os.Stat(p); err != nil {
t.Fatalf("本地落盘文件未生成: %v", err)
}
zeroDash() // 模拟进程重启
if err := loadDashboardLocal(); err != nil {
t.Fatalf("loadDashboardLocal: %v", err)
}
m := Dash.Metrics()
want := map[string]float64{
"recall_usefulness_rate": 0.75, // 15/20
"gap_closure_rate": 0.5, // 10/20
"cascade_fix_rate": 0.33, // 10/30
"auto_resolve_rate": 0.67, // 20/30
"deprecated_per_day": 4,
"avg_distill_loss": 0.21, // 4.2/20
}
for k, v := range want {
if m[k] != v {
t.Errorf("%s: want %v got %v", k, v, m[k])
}
}
}
func TestDashboardLocalPersistence_StaleDeprecatedResetsToZero(t *testing.T) {
p := filepath.Join(t.TempDir(), "dash.json")
t.Setenv("ZHIYI_DASHBOARD_PATH", p)
// 手写一份"很久以前"的快照:deprecated_today 应归零,其余计数应保留
blob := `{"deprecated_today":9,"deprecated_date":"2000-01-01","total_recalls":3,"hit_count":2}`
if err := os.WriteFile(p, []byte(blob), 0o600); err != nil {
t.Fatal(err)
}
zeroDash()
if err := loadDashboardLocal(); err != nil {
t.Fatal(err)
}
m := Dash.Metrics()
if m["deprecated_per_day"] != 0 {
t.Errorf("跨天未归零: %v", m["deprecated_per_day"])
}
// 2026-10-17:recall_hit_rate 已退役。blob 无反馈计数 ⇒ 用 recall_usefulness_rate
// 确认「其余计数状态被保留」(无反馈 = metricNA,未被 load 改坏)。
if m["recall_usefulness_rate"] != metricNA {
t.Errorf("recall_usefulness_rate 应保留为 metricNA(blob 无反馈计数), got %v", m["recall_usefulness_rate"])
}
}
func TestDashboardLocalPersistence_MissingOrBrokenFileIsSafe(t *testing.T) {
p := filepath.Join(t.TempDir(), "nope.json")
t.Setenv("ZHIYI_DASHBOARD_PATH", p)
zeroDash()
// 本测试的本意是「load 失败不得改动 Dash」——所以基准值取「零状态的 Metrics 值」,
// 而不是硬编码 0。2026-10-17 起 recall_hit_rate 已退役,改用 recall_usefulness_rate
// 作基准(零状态为 metricNA(-1),无样本 ⇒ 无结论)。
want := Dash.Metrics()["recall_usefulness_rate"]
if err := loadDashboardLocal(); err == nil {
t.Errorf("缺文件应返回 err(调用方忽略,行为退回仅内存)")
}
if got := Dash.Metrics()["recall_usefulness_rate"]; got != want {
t.Errorf("缺文件不应改动 Dash:期望 %v,实际 %v", want, got)
}
if err := os.WriteFile(p, []byte("{ not json"), 0o600); err != nil {
t.Fatal(err)
}
if err := loadDashboardLocal(); err == nil {
t.Errorf("坏文件应返回 err")
}
if got := Dash.Metrics()["recall_usefulness_rate"]; got != want {
t.Errorf("坏文件不应改动 Dash:期望 %v,实际 %v", want, got)
}
}