diff --git a/cli-anything/cli_anything/zhiyi/zhiyi_cli.py b/cli-anything/cli_anything/zhiyi/zhiyi_cli.py index d0293a3..5955460 100644 --- a/cli-anything/cli_anything/zhiyi/zhiyi_cli.py +++ b/cli-anything/cli_anything/zhiyi/zhiyi_cli.py @@ -197,7 +197,6 @@ def stats(stat_type, json_flag): if "metrics" in data: m = data["metrics"] click.echo(f"\n📊 自优化指标:") - click.echo(f" 召回命中率: {m.get('recall_hit_rate', '?'):.1%}") click.echo(f" 召回有用率: {m.get('recall_usefulness_rate', '?'):.1%}") click.echo(f" 记忆总数: {m.get('total_memories', '?')}") @@ -429,7 +428,7 @@ def _repl_with_skin(c: ZhiYiClient, skin): ["记忆数", str(s.get("total_memories", "?"))], ["图谱节点", str(g.get("node_count", "?"))], ["图谱边", str(g.get("edge_count", "?"))], - ["召回命中率", f"{m.get('recall_hit_rate', '?'):.1%}"], + ["召回有用率", f"{m.get('recall_usefulness_rate', '?'):.1%}"], ], ) elif cmd == "graph" and args and args[0] == "navigate": diff --git a/go/internal/api/routes/tuning.go b/go/internal/api/routes/tuning.go index 8663782..c87a680 100644 --- a/go/internal/api/routes/tuning.go +++ b/go/internal/api/routes/tuning.go @@ -204,20 +204,22 @@ func (tc *TuningConfig) AutoTune(metrics map[string]float64) []TuningEvent { skipped = append(skipped, rule+": "+why) } - // 1. 召回命中率 < 0.5 → 提高 diversity(扩大搜索范围) - hitRate, hitOK := ratioFor(metrics, "recall_hit_rate") + // 1. 召回质量低 → 提高 diversity(扩大搜索范围)。 + // 2026-10-17:原信号 recall_hit_rate 已退役(恒=1、从不及 0.5 ⇒ 本安全阀长期死寂)。 + // 改用 recall_usefulness_rate(返回且被使用)作为召回质量的实时信号:< 0.5 ⇒ 召回太窄、扩大范围。 + hitRate, hitOK := ratioFor(metrics, "recall_usefulness_rate") if !hitOK { - note("rule1_hit_rate", "指标不可用(无数据/退化/样本<30)") + note("rule1_usefulness", "指标不可用(无数据/退化/样本<20)") } if hitOK && hitRate < 0.5 && !tc.canMove("diversity") { - note("rule1_hit_rate", "冷却中(距上次调整 <"+tuneCooldown.String()+")") + note("rule1_usefulness", "冷却中(距上次调整 <"+tuneCooldown.String()+")") } if hitOK && hitRate < 0.5 && tc.canMove("diversity") { newDiversity := tc.Diversity + 0.1 if newDiversity > 1.0 { newDiversity = 1.0 } - events = append(events, recordTuning(tc, "diversity", tc.Diversity, newDiversity, "recall_hit_rate < 0.5")) + events = append(events, recordTuning(tc, "diversity", tc.Diversity, newDiversity, "recall_usefulness_rate < 0.5")) tc.Diversity = newDiversity tc.markMove("diversity") } @@ -358,7 +360,7 @@ func (tc *TuningConfig) AnalyticsHandler(w http.ResponseWriter, r *http.Request) m := selfoptimize.Dash.Metrics() suggestions := []string{} - if m["recall_hit_rate"] < 0.5 && m["avg_distill_loss"] > 0.3 { + if m["recall_usefulness_rate"] < 0.5 && m["avg_distill_loss"] > 0.3 { suggestions = append(suggestions, "蒸馏质量低导致召回率低 — 建议检查 Embedding 模型") } if m["deprecated_per_day"] > 10 { diff --git a/go/internal/api/routes/tuning_guard_test.go b/go/internal/api/routes/tuning_guard_test.go index 12d17b8..fa43324 100644 --- a/go/internal/api/routes/tuning_guard_test.go +++ b/go/internal/api/routes/tuning_guard_test.go @@ -23,10 +23,8 @@ func TestAutoTuneGuard_SentinelAndSamples(t *testing.T) { "gap_closure_samples": 0, "gap_closure_valid": 0, "recall_usefulness_rate": -1, - "recall_hit_rate": -1, "avg_distill_loss": -1, "recall_usefulness_valid": 0, - "recall_hit_valid": 0, "avg_distill_loss_valid": 0, } if ev := tc.AutoTune(na); len(ev) != 0 { diff --git a/go/internal/distributed/distributed.go b/go/internal/distributed/distributed.go index 7b1856d..717d06d 100644 --- a/go/internal/distributed/distributed.go +++ b/go/internal/distributed/distributed.go @@ -233,15 +233,7 @@ func (mh *MetricsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "# TYPE zhiyi_recall_usefulness_samples gauge\n") fmt.Fprintf(w, "zhiyi_recall_usefulness_samples %f\n", metrics["recall_usefulness_samples"]) - fmt.Fprintf(w, "# HELP zhiyi_recall_hit_rate Recall命中率(分子=召回返回非空);-1=无数据\n") - fmt.Fprintf(w, "# TYPE zhiyi_recall_hit_rate gauge\n") - fmt.Fprintf(w, "zhiyi_recall_hit_rate %f\n", metrics["recall_hit_rate"]) - fmt.Fprintf(w, "# HELP zhiyi_recall_hit_valid 该率是否可解读;0 ⇒ 值为 -1\n") - fmt.Fprintf(w, "# TYPE zhiyi_recall_hit_valid gauge\n") - fmt.Fprintf(w, "zhiyi_recall_hit_valid %f\n", metrics["recall_hit_valid"]) - fmt.Fprintf(w, "# HELP zhiyi_recall_hit_samples 该率的分母样本数\n") - fmt.Fprintf(w, "# TYPE zhiyi_recall_hit_samples gauge\n") - fmt.Fprintf(w, "zhiyi_recall_hit_samples %f\n", metrics["recall_hit_samples"]) + // 2026-10-17:zhiyi_recall_hit_rate 已退役(恒=1、无区分度),由 recall_usefulness_rate 承担「返回且被使用」语义。 fmt.Fprintf(w, "# HELP zhiyi_gap_closure_rate 缺口闭环率;-1=无数据\n") fmt.Fprintf(w, "# TYPE zhiyi_gap_closure_rate gauge\n") diff --git a/go/internal/metrics/metrics.go b/go/internal/metrics/metrics.go index 86b1941..213bfa6 100644 --- a/go/internal/metrics/metrics.go +++ b/go/internal/metrics/metrics.go @@ -84,12 +84,6 @@ var ( Help: "recall 有用率 (0-1)", }, ) - RecallHitRate = prometheus.NewGauge( - prometheus.GaugeOpts{ - Name: "zhiyi_recall_hit_rate", - Help: "recall 命中率 (0-1)", - }, - ) GapClosureRate = prometheus.NewGauge( prometheus.GaugeOpts{ Name: "zhiyi_gap_closure_rate", @@ -136,7 +130,6 @@ func init() { MemoryRSSBytes, SQLiteDBSizeBytes, RecallUsefulnessRate, - RecallHitRate, GapClosureRate, DeprecatedPerDay, AutoResolveRate, @@ -155,9 +148,6 @@ func SyncFromDashboard(m map[string]float64) { if v, ok := m["recall_usefulness_rate"]; ok { RecallUsefulnessRate.Set(v) } - if v, ok := m["recall_hit_rate"]; ok { - RecallHitRate.Set(v) - } if v, ok := m["gap_closure_rate"]; ok { GapClosureRate.Set(v) } diff --git a/go/internal/selfoptimize/executor.go b/go/internal/selfoptimize/executor.go index c06aaa5..a9a2628 100644 --- a/go/internal/selfoptimize/executor.go +++ b/go/internal/selfoptimize/executor.go @@ -119,8 +119,10 @@ func (te *TriggerExecutor) evaluate(t *TriggerDef, metrics map[string]float64) b threshold, _ := t.Condition.(float64) return hours >= threshold case TRecallMiss: - // 检查最近的 miss 计数 - rate, ok := metrics["recall_hit_rate"] + // 检查最近的 miss 计数。 + // 2026-10-17:recall_hit_rate 已退役(恒=1、无区分度),改用 recall_usefulness_rate + // (返回且被使用)作为「召回已产生可用结果、应清除 miss」的信号。 + rate, ok := metrics["recall_usefulness_rate"] return ok && rate > 0 && t.LastFired.IsZero() case TQualityDrop: threshold, _ := t.Condition.(float64) diff --git a/go/internal/selfoptimize/metrics_na_test.go b/go/internal/selfoptimize/metrics_na_test.go index f452cb5..cd3742e 100644 --- a/go/internal/selfoptimize/metrics_na_test.go +++ b/go/internal/selfoptimize/metrics_na_test.go @@ -19,7 +19,6 @@ func TestMetricsDegenerateReturnsNA(t *testing.T) { {"无反馈样本", &Dashboard{}, "recall_usefulness_rate"}, {"有正反馈但无负反馈(旧实现返回 1 的情形)", &Dashboard{UsefulCount: 46, NotUsefulCount: 0}, "recall_usefulness_rate"}, - {"从未召回", &Dashboard{}, "recall_hit_rate"}, {"无缺口记录", &Dashboard{}, "gap_closure_rate"}, {"无修复记录", &Dashboard{}, "cascade_fix_rate"}, {"无冲突记录", &Dashboard{}, "auto_resolve_rate"}, @@ -50,25 +49,24 @@ func TestMetricsHealthyPathStillComputes(t *testing.T) { NotUsefulCount: 30, TotalRecalls: 200, HitCount: 150, - TotalGaps: 10, - ClosedGaps: 4, + TotalGaps: 20, // ≥ 最小样本门 N=20 + ClosedGaps: 8, TotalFixes: 25, CascadeFixedTotal: 10, - TotalConflicts: 8, - AutoResolvedConflicts: 2, - DistillLossSum: 1.0, - DistillLossCount: 2, + TotalConflicts: 20, // ≥ 最小样本门 N=20 + AutoResolvedConflicts: 5, + DistillLossSum: 10.0, + DistillLossCount: 20, // ≥ 最小样本门 N=20 DeprecatedToday: 31, } m := d.Metrics() want := map[string]float64{ "recall_usefulness_rate": 0.70, // 70/(70+30) - "recall_hit_rate": 0.75, // 150/200 - "gap_closure_rate": 0.40, // 4/10 + "gap_closure_rate": 0.40, // 8/20 "cascade_fix_rate": 0.40, // 10/25 - "auto_resolve_rate": 0.25, // 2/8 - "avg_distill_loss": 0.50, // 1.0/2 + "auto_resolve_rate": 0.25, // 5/20 + "avg_distill_loss": 0.50, // 10.0/20 "deprecated_per_day": 31, } for k, w := range want { @@ -77,9 +75,9 @@ func TestMetricsHealthyPathStillComputes(t *testing.T) { } } - // 健康路径所有 *_valid 必须为 1 + // 健康路径所有 *_valid 必须为 1(recall_hit_valid 已于 2026-10-17 随 recall_hit_rate 退役) for _, k := range []string{ - "recall_usefulness_valid", "recall_hit_valid", "gap_closure_valid", + "recall_usefulness_valid", "gap_closure_valid", "cascade_fix_valid", "auto_resolve_valid", "avg_distill_loss_valid", } { if m[k] != 1 { @@ -93,9 +91,8 @@ func TestMetricsSamplesReported(t *testing.T) { d := Dashboard{UsefulCount: 46, TotalRecalls: 1000, TotalGaps: 3} m := d.Metrics() want := map[string]float64{ - "recall_usefulness_samples": 46, // 46+0 - "recall_hit_samples": 1000, // 1000 - "gap_closure_samples": 3, // 3 + "recall_usefulness_samples": 46, // 46+0 + "gap_closure_samples": 3, // 3 } for k, w := range want { if got := m[k]; got != w { @@ -106,9 +103,10 @@ func TestMetricsSamplesReported(t *testing.T) { // TestMetricsKeyNamesAreBackwardCompatible —— 只增键、不改键名: // 所有旧键必须仍然存在(否则 SyncFromDashboard / 各 JSON 端点会静默丢指标)。 +// 2026-10-17 例外:recall_hit_rate 是**有意退役**(恒=1、无区分度),不在兼容契约内。 func TestMetricsKeyNamesAreBackwardCompatible(t *testing.T) { legacy := []string{ - "recall_usefulness_rate", "recall_hit_rate", "gap_closure_rate", + "recall_usefulness_rate", "gap_closure_rate", "cascade_fix_rate", "deprecated_per_day", "avg_distill_loss", "auto_resolve_rate", } @@ -118,6 +116,10 @@ func TestMetricsKeyNamesAreBackwardCompatible(t *testing.T) { t.Errorf("旧键 %q 丢失——本改动必须只增不改", k) } } + // 退役键不得复出 + if _, ok := m["recall_hit_rate"]; ok { + t.Errorf("recall_hit_rate 应已退役,不得复出") + } } // TestRateOrNA —— 辅助函数边界:den<=0 一律 -1;否则两位四舍五入。 diff --git a/go/internal/selfoptimize/metrics_retire_regression_test.go b/go/internal/selfoptimize/metrics_retire_regression_test.go new file mode 100644 index 0000000..c3c5573 --- /dev/null +++ b/go/internal/selfoptimize/metrics_retire_regression_test.go @@ -0,0 +1,106 @@ +// 织忆 MemoryWeave — 2026-10-17 指标口径修复回归测试 +// +// 覆盖三处缺陷的修复: +// ① recall_hit_rate 退役(恒=1、无区分度)⇒ Metrics() 不再输出该键 +// ② gap_closure_rate 越界(4.5)⇒ 统一为「缺口个数」口径 + 越界断言拦截 +// ③ 最小样本门 N=20 ⇒ 样本 < 20 一律 -1 + valid=0(杜绝 auto_resolve 2 样本满分) +// +// 本测试用【生产同款计数器】复现改前状态(recall_hit 21252/1、gap 4.5/2、 +// auto_resolve 1/2、usefulness 207/0.98),并断言改后 /api/v1/metrics 语义。 +package selfoptimize + +import ( + "encoding/json" + "os" + "testing" +) + +// prodLikeDash 复现生产 /api/v1/metrics 的改前计数状态(2026-10-17 19:10 实测值)。 +func prodLikeDash() *Dashboard { + return &Dashboard{ + TotalRecalls: 21252, + HitCount: 21252, + UsefulCount: 203, + NotUsefulCount: 4, + DistillLossSum: 2732.58, // avg 0.51 * 5358(近似) + DistillLossCount: 5358, + ClosedGaps: 9, // 4.5 * 2(改前 ClosedGaps/TotalGaps 口径错混,越界) + TotalGaps: 2, + TotalFixes: 0, + CascadeFixedTotal: 0, + AutoResolvedConflicts: 2, + TotalConflicts: 2, + DeprecatedToday: 31, + DistillSuccessCount: 3163, + } +} + +// TestMetricsRetireHitRate —— ① recall_hit_rate 退役:Metrics() 不再输出该键及配套键。 +func TestMetricsRetireHitRate(t *testing.T) { + d := prodLikeDash() + m := d.Metrics() + if _, ok := m["recall_hit_rate"]; ok { + t.Fatalf("recall_hit_rate 应已退役,Metrics() 不得再输出") + } + if _, ok := m["recall_hit_samples"]; ok { + t.Fatalf("recall_hit_samples 应已退役") + } + if _, ok := m["recall_hit_valid"]; ok { + t.Fatalf("recall_hit_valid 应已退役") + } +} + +// TestMetricsGapClosureBounded —— ② gap_closure_rate 越界断言: +// 改前 4.5(ClosedGaps=9/TotalGaps=2,写入侧把关闭事件数混入分子)⇒ 改后断言拦截为 -1 + valid=0。 +func TestMetricsGapClosureBounded(t *testing.T) { + d := prodLikeDash() + m := d.Metrics() + if v := m["gap_closure_rate"]; v != metricNA { + t.Fatalf("越界 gap_closure_rate 应被断言拦截为 -1,got %v", v) + } + if v := m["gap_closure_valid"]; v != 0 { + t.Fatalf("越界时 gap_closure_valid 应为 0,got %v", v) + } +} + +// TestMetricsMinSampleGate —— ③ 最小样本门 N=20: +// auto_resolve 仅 2 样本(改前 1 分/valid=1)⇒ 改后 -1 + valid=0;gap 2 样本同样 -1。 +func TestMetricsMinSampleGate(t *testing.T) { + d := prodLikeDash() + m := d.Metrics() + if v := m["auto_resolve_rate"]; v != metricNA { + t.Fatalf("auto_resolve 样本 2<20 应为 -1,got %v", v) + } + if v := m["auto_resolve_valid"]; v != 0 { + t.Fatalf("auto_resolve 样本 2<20 应为 valid=0,got %v", v) + } + // usefulness 有 207 样本 ≥ 20 且负反馈存在 ⇒ 可读(0.98),证明门没把好数据焊死 + if v := m["recall_usefulness_rate"]; v < 0.97 || v > 0.99 { + t.Fatalf("usefulness 207 样本应可读为 ~0.98,got %v", v) + } + if v := m["recall_usefulness_valid"]; v != 1 { + t.Fatalf("usefulness 应 valid=1,got %v", v) + } +} + +// TestMetricsProdBeforeAfterJSON —— 复现改前/改后 /api/v1/metrics 的 JSON 输出, +// 供验收附「改前/改后两次原始 JSON」(前置断言保证改后语义;本测试打印改后 JSON)。 +func TestMetricsProdBeforeAfterJSON(t *testing.T) { + d := prodLikeDash() + // 断言改后 key 集(与改前对比:recall_hit_* 移除) + m := d.Metrics() + for _, absent := range []string{"recall_hit_rate", "recall_hit_samples", "recall_hit_valid"} { + if _, ok := m[absent]; ok { + t.Fatalf("%s 不应出现在改后 Metrics()", absent) + } + } + // 打印改后 JSON(同 /api/v1/metrics 序列化路径) + out, err := json.MarshalIndent(m, "", " ") + if err != nil { + t.Fatal(err) + } + t.Logf("改后 /api/v1/metrics JSON:\n%s", string(out)) + if os.Getenv("PRINT_METRICS_JSON") == "1" { + println(string(out)) + } +} \ No newline at end of file diff --git a/go/internal/selfoptimize/selfoptimize.go b/go/internal/selfoptimize/selfoptimize.go index 51b41b2..85b9435 100644 --- a/go/internal/selfoptimize/selfoptimize.go +++ b/go/internal/selfoptimize/selfoptimize.go @@ -260,6 +260,13 @@ func rateOrNA(num, den int) float64 { return math.Round(float64(num)/float64(den)*100) / 100 } +// minRateSamples 是率类指标的最小样本门。 +// +// 样本太少(< 20)时,即使比率为真值也**统计上不可解读** —— 例如 auto_resolve_rate +// 曾仅 2 个样本即判 valid=1 满分。现在样本不足一律按退化处理(metricNA + valid=0), +// 不参与面板解读,避免「小样本满分」误导。 +const minRateSamples = 20 + // b2f 布尔转 0/1,用于 *_valid 键。 func b2f(b bool) float64 { if b { @@ -278,50 +285,80 @@ func (d *Dashboard) Metrics() map[string]float64 { // 负样本通道为空 ⇒ 无从判断好坏(详见 metricNA 注释)。 usefulRate := metricNA + usefulSamples := d.UsefulCount + d.NotUsefulCount if d.NotUsefulCount > 0 { - usefulRate = rateOrNA(d.UsefulCount, d.UsefulCount+d.NotUsefulCount) + usefulRate = rateOrNA(d.UsefulCount, usefulSamples) } - hitRate := rateOrNA(d.HitCount, d.TotalRecalls) + // recall_hit_rate 已于 2026-10-17 退役:其分子原本是 core.go 的 `len(results) > 0`, + // 量的是「召回有没有返回东西」而非「返回的对不对」,恒为 1、无区分度。 + // 「返回且被使用」的语义已由 recall_usefulness_rate 承担(198 样本 0.98)。 + // 禁止保留恒等于 1 的指标挂在面板上误导人 ⇒ 从 Metrics() 移除该键。 gapRate := rateOrNA(d.ClosedGaps, d.TotalGaps) cascadeRate := rateOrNA(d.CascadeFixedTotal, d.TotalFixes) autoRate := rateOrNA(d.AutoResolvedConflicts, d.TotalConflicts) + // gap_closure_rate 断言:ClosedGaps/TotalGaps 统一为「缺口个数」口径后,比率必落 [0,1]。 + // 若写入侧再次把「关闭事件数」混进来导致越界,断言拦截 → -1 + valid=0(禁止越界值上板)。 + if gapRate > 1 { + gapRate = metricNA + } + // 均值项:零样本的均值同样是退化,按同一规则返回 metricNA。 avgLoss := metricNA if d.DistillLossCount > 0 { avgLoss = math.Round(d.DistillLossSum/float64(d.DistillLossCount)*100) / 100 } + // 最小样本门:samples < minRateSamples ⇒ 值 = -1 且 valid = 0,不参与面板解读。 + usefulVal, usefulValid := usefulRate, b2f(usefulSamples >= minRateSamples && d.NotUsefulCount > 0) + gapVal, gapValid := gapRate, b2f(d.TotalGaps >= minRateSamples) + cascadeVal, cascadeValid := cascadeRate, b2f(d.TotalFixes >= minRateSamples) + autoVal, autoValid := autoRate, b2f(d.TotalConflicts >= minRateSamples) + lossVal, lossValid := avgLoss, b2f(d.DistillLossCount >= minRateSamples) + if usefulSamples < minRateSamples || d.NotUsefulCount == 0 { + usefulVal = metricNA + } + if d.TotalGaps < minRateSamples || gapVal == metricNA { + // 样本不足 或 越界断言触发 ⇒ 值 -1 且 valid=0(值与 flag 保持同步)。 + gapVal = metricNA + gapValid = 0 + } + if d.TotalFixes < minRateSamples { + cascadeVal = metricNA + } + if d.TotalConflicts < minRateSamples { + autoVal = metricNA + } + if d.DistillLossCount < minRateSamples { + lossVal = metricNA + } + return map[string]float64{ - "recall_usefulness_rate": usefulRate, - "recall_usefulness_samples": float64(d.UsefulCount + d.NotUsefulCount), + "recall_usefulness_rate": usefulVal, + "recall_usefulness_samples": float64(usefulSamples), "recall_usefulness_negative": float64(d.NotUsefulCount), "recall_usefulness_positive": float64(d.UsefulCount), - "recall_usefulness_valid": b2f(d.NotUsefulCount > 0), + "recall_usefulness_valid": usefulValid, - "recall_hit_rate": hitRate, - "recall_hit_samples": float64(d.TotalRecalls), - "recall_hit_valid": b2f(d.TotalRecalls > 0), - - "gap_closure_rate": gapRate, + "gap_closure_rate": gapVal, "gap_closure_samples": float64(d.TotalGaps), - "gap_closure_valid": b2f(d.TotalGaps > 0), + "gap_closure_valid": gapValid, - "cascade_fix_rate": cascadeRate, + "cascade_fix_rate": cascadeVal, "cascade_fix_samples": float64(d.TotalFixes), - "cascade_fix_valid": b2f(d.TotalFixes > 0), + "cascade_fix_valid": cascadeValid, - "auto_resolve_rate": autoRate, + "auto_resolve_rate": autoVal, "auto_resolve_samples": float64(d.TotalConflicts), - "auto_resolve_valid": b2f(d.TotalConflicts > 0), + "auto_resolve_valid": autoValid, "deprecated_per_day": float64(d.DeprecatedToday), "distill_success_count": float64(d.DistillSuccessCount), - "avg_distill_loss": avgLoss, + "avg_distill_loss": lossVal, "avg_distill_loss_samples": float64(d.DistillLossCount), - "avg_distill_loss_valid": b2f(d.DistillLossCount > 0), + "avg_distill_loss_valid": lossValid, } } @@ -604,7 +641,9 @@ func (gd *GapDetector) List() []*Gap { func (gd *GapDetector) Close(topic string) { gd.mu.Lock() defer gd.mu.Unlock() - if g, ok := gd.gaps[topic]; ok { + // 统一「缺口个数」口径:仅当缺口从未关闭时才算一次闭环(与 RecordHit 一致), + // 否则同一缺口被 Close 重复调用会让 ClosedGaps 虚增、gap_closure_rate 越界 > 1。 + if g, ok := gd.gaps[topic]; ok && !g.Closed { g.Closed = true gd.persistGaps() Dash.RecordGapClosed() // 同步仪表盘 diff --git a/go/internal/selfoptimize/selfoptimize_test.go b/go/internal/selfoptimize/selfoptimize_test.go index 7704627..24f9b5b 100644 --- a/go/internal/selfoptimize/selfoptimize_test.go +++ b/go/internal/selfoptimize/selfoptimize_test.go @@ -50,8 +50,10 @@ func TestDashboard_InitialMetricsNoData(t *testing.T) { if m["recall_usefulness_rate"] != metricNA { t.Errorf("expected metricNA(%v), got %f", metricNA, m["recall_usefulness_rate"]) } - if m["recall_hit_rate"] != metricNA { - t.Errorf("expected metricNA(%v), got %f", metricNA, m["recall_hit_rate"]) + // 2026-10-17:recall_hit_rate 已退役(恒=1、无区分度),其「返回且被使用」语义由 + // recall_usefulness_rate 承担。Metrics() 不再输出该键 ⇒ 断言键不存在(读它应得 0,会误导)。 + if _, ok := m["recall_hit_rate"]; ok { + t.Errorf("recall_hit_rate 已退役,Metrics() 不应再输出该键") } if m["recall_usefulness_valid"] != 0 { t.Errorf("expected valid=0, got %f", m["recall_usefulness_valid"]) @@ -60,10 +62,13 @@ func TestDashboard_InitialMetricsNoData(t *testing.T) { func TestDashboard_UsefulnessRate(t *testing.T) { d := &Dashboard{} - d.RecordFeedback(true) - d.RecordFeedback(true) - d.RecordFeedback(true) - d.RecordFeedback(false) + // 满足最小样本门 N=20:15 有用 + 5 无用 = 20 样本 ⇒ 15/20 = 0.75 + for i := 0; i < 15; i++ { + d.RecordFeedback(true) + } + for i := 0; i < 5; i++ { + d.RecordFeedback(false) + } m := d.Metrics() if m["recall_usefulness_rate"] != 0.75 { @@ -71,6 +76,21 @@ func TestDashboard_UsefulnessRate(t *testing.T) { } } +// TestDashboard_UsefulnessRateBelowMinSamples —— 最小样本门(N=20):反馈样本 < 20 ⇒ -1 + valid=0。 +func TestDashboard_UsefulnessRateBelowMinSamples(t *testing.T) { + d := &Dashboard{} + for i := 0; i < 2; i++ { // 仅 2 样本(旧 auto_resolve_rate 同类误判满分) + d.RecordFeedback(true) + } + m := d.Metrics() + if m["recall_usefulness_rate"] != metricNA { + t.Errorf("样本 < 20 应为 metricNA(-1), got %f", m["recall_usefulness_rate"]) + } + if m["recall_usefulness_valid"] != 0 { + t.Errorf("样本 < 20 应为 valid=0, got %f", m["recall_usefulness_valid"]) + } +} + // TestDashboard_DistillDoesNotPolluteUsefulnessRate —— 2026-10-14 t_c06ad4d0 解污染回归。 // 蒸馏回调改走 RecordDistillSuccess()(独立计数 distill_success_count), // 不得再污染 recall_usefulness_rate 的分子/分母。10 次蒸馏成功 ⇒ UsefulCount 仍为 0。 @@ -98,28 +118,62 @@ func TestDashboard_DistillDoesNotPolluteUsefulnessRate(t *testing.T) { } } -func TestDashboard_HitRate(t *testing.T) { +// TestDashboard_HitRateRetired —— 2026-10-17:recall_hit_rate 已退役。 +// 其分子原是 core.go 的 `len(results)>0`(恒=1、无区分度),「返回且被使用」语义由 +// recall_usefulness_rate 承担。RecordRecall 仍累积 TotalRecalls/HitCount(供持久化/审计), +// 但 Metrics() 不再输出 recall_hit_rate 键。 +func TestDashboard_HitRateRetired(t *testing.T) { d := &Dashboard{} d.RecordRecall(true) d.RecordRecall(true) d.RecordRecall(false) m := d.Metrics() - if m["recall_hit_rate"] < 0.66 || m["recall_hit_rate"] > 0.67 { - t.Errorf("expected ~0.67, got %f", m["recall_hit_rate"]) + if _, ok := m["recall_hit_rate"]; ok { + t.Errorf("recall_hit_rate 已退役,Metrics() 不应再输出该键") + } + if m["recall_hit_samples"] != 0 { + t.Errorf("recall_hit_samples 已退役,不应输出(got %f)", m["recall_hit_samples"]) } } func TestDashboard_GapClosureRate(t *testing.T) { d := &Dashboard{} - d.TotalGaps = 10 - d.ClosedGaps = 5 + d.TotalGaps = 20 // ≥ 最小样本门 N=20 + d.ClosedGaps = 10 m := d.Metrics() if m["gap_closure_rate"] != 0.5 { t.Errorf("expected 0.5, got %f", m["gap_closure_rate"]) } } +// TestDashboard_GapClosureBelowMinSamples —— 最小样本门(N=20)+ 越界断言: +// TotalGaps < 20 ⇒ -1 + valid=0;ClosedGaps > TotalGaps(写入侧口径错)⇒ 断言拦截为 -1 + valid=0。 +func TestDashboard_GapClosureBelowMinSamples(t *testing.T) { + d := &Dashboard{} + d.TotalGaps = 2 + d.ClosedGaps = 1 + m := d.Metrics() + if m["gap_closure_rate"] != metricNA { + t.Errorf("TotalGaps < 20 应为 metricNA(-1), got %f", m["gap_closure_rate"]) + } + if m["gap_closure_valid"] != 0 { + t.Errorf("TotalGaps < 20 应为 valid=0, got %f", m["gap_closure_valid"]) + } + + // 越界断言:ClosedGaps > TotalGaps(同类 4.5 越界值)⇒ 拦截为 -1,禁止上板 + d2 := &Dashboard{} + d2.TotalGaps = 20 + d2.ClosedGaps = 30 // 关闭事件数混入 ⇒ 越界 + m2 := d2.Metrics() + if m2["gap_closure_rate"] != metricNA { + t.Errorf("ClosedGaps > TotalGaps 越界应被断言拦截为 metricNA(-1), got %f", m2["gap_closure_rate"]) + } + if m2["gap_closure_valid"] != 0 { + t.Errorf("越界时 valid 应为 0, got %f", m2["gap_closure_valid"]) + } +} + func TestDashboard_RecordDeprecation(t *testing.T) { d := &Dashboard{} d.RecordDeprecation() @@ -143,13 +197,18 @@ func TestDashboard_RecordCorrection(t *testing.T) { func TestDashboard_RecordConflictResolved(t *testing.T) { d := &Dashboard{} - d.RecordConflictResolved(true) - d.RecordConflictResolved(false) - if d.TotalConflicts != 2 { - t.Errorf("expected 2 conflicts, got %d", d.TotalConflicts) + // 满足最小样本门 N=20:10 自动 + 10 手动 = 20 冲突 ⇒ 10/20 = 0.5 + for i := 0; i < 10; i++ { + d.RecordConflictResolved(true) } - if d.AutoResolvedConflicts != 1 { - t.Errorf("expected 1 auto, got %d", d.AutoResolvedConflicts) + for i := 0; i < 10; i++ { + d.RecordConflictResolved(false) + } + if d.TotalConflicts != 20 { + t.Errorf("expected 20 conflicts, got %d", d.TotalConflicts) + } + if d.AutoResolvedConflicts != 10 { + t.Errorf("expected 10 auto, got %d", d.AutoResolvedConflicts) } m := d.Metrics() if m["auto_resolve_rate"] != 0.5 { @@ -384,10 +443,10 @@ func TestDashboardLocalPersistence_RoundTrip(t *testing.T) { t.Setenv("ZHIYI_DASHBOARD_PATH", p) src := &Dashboard{ - TotalRecalls: 7, HitCount: 6, UsefulCount: 5, NotUsefulCount: 1, - DistillLossSum: 1.05, DistillLossCount: 5, - ClosedGaps: 2, TotalGaps: 4, TotalFixes: 3, CascadeFixedTotal: 1, - AutoResolvedConflicts: 2, TotalConflicts: 3, DeprecatedToday: 4, + TotalRecalls: 7, HitCount: 6, UsefulCount: 15, NotUsefulCount: 5, // 20 样本 ≥ N + DistillLossSum: 4.2, DistillLossCount: 20, + ClosedGaps: 10, TotalGaps: 20, TotalFixes: 30, CascadeFixedTotal: 10, + AutoResolvedConflicts: 20, TotalConflicts: 30, DeprecatedToday: 4, } src.mu.Lock() src.persist() // 测试环境无 Redis → 走本地 JSON 分支 @@ -403,13 +462,12 @@ func TestDashboardLocalPersistence_RoundTrip(t *testing.T) { } m := Dash.Metrics() want := map[string]float64{ - "recall_hit_rate": 0.86, // 6/7 - "recall_usefulness_rate": 0.83, // 5/6 - "gap_closure_rate": 0.5, // 2/4 - "cascade_fix_rate": 0.33, // 1/3 - "auto_resolve_rate": 0.67, // 2/3 + "recall_usefulness_rate": 0.75, // 15/20 + "gap_closure_rate": 0.5, // 10/20 + "cascade_fix_rate": 0.33, // 10/30 + "auto_resolve_rate": 0.67, // 20/30 "deprecated_per_day": 4, - "avg_distill_loss": 0.21, // 1.05/5 + "avg_distill_loss": 0.21, // 4.2/20 } for k, v := range want { if m[k] != v { @@ -434,8 +492,10 @@ func TestDashboardLocalPersistence_StaleDeprecatedResetsToZero(t *testing.T) { if m["deprecated_per_day"] != 0 { t.Errorf("跨天未归零: %v", m["deprecated_per_day"]) } - if m["recall_hit_rate"] != 0.67 { // 2/3 - t.Errorf("其他计数应保留, got %v", m["recall_hit_rate"]) + // 2026-10-17:recall_hit_rate 已退役。blob 无反馈计数 ⇒ 用 recall_usefulness_rate + // 确认「其余计数状态被保留」(无反馈 = metricNA,未被 load 改坏)。 + if m["recall_usefulness_rate"] != metricNA { + t.Errorf("recall_usefulness_rate 应保留为 metricNA(blob 无反馈计数), got %v", m["recall_usefulness_rate"]) } } @@ -445,13 +505,13 @@ func TestDashboardLocalPersistence_MissingOrBrokenFileIsSafe(t *testing.T) { zeroDash() // 本测试的本意是「load 失败不得改动 Dash」——所以基准值取「零状态的 Metrics 值」, - // 而不是硬编码 0。2026-10-14 语义变更后,零状态的 recall_hit_rate 是 metricNA(-1) - // 而非 0(无样本 ⇒ 无结论,不是「0% 命中」)。 - want := Dash.Metrics()["recall_hit_rate"] + // 而不是硬编码 0。2026-10-17 起 recall_hit_rate 已退役,改用 recall_usefulness_rate + // 作基准(零状态为 metricNA(-1),无样本 ⇒ 无结论)。 + want := Dash.Metrics()["recall_usefulness_rate"] if err := loadDashboardLocal(); err == nil { t.Errorf("缺文件应返回 err(调用方忽略,行为退回仅内存)") } - if got := Dash.Metrics()["recall_hit_rate"]; got != want { + if got := Dash.Metrics()["recall_usefulness_rate"]; got != want { t.Errorf("缺文件不应改动 Dash:期望 %v,实际 %v", want, got) } @@ -461,7 +521,7 @@ func TestDashboardLocalPersistence_MissingOrBrokenFileIsSafe(t *testing.T) { if err := loadDashboardLocal(); err == nil { t.Errorf("坏文件应返回 err") } - if got := Dash.Metrics()["recall_hit_rate"]; got != want { + if got := Dash.Metrics()["recall_usefulness_rate"]; got != want { t.Errorf("坏文件不应改动 Dash:期望 %v,实际 %v", want, got) } } diff --git a/scripts/daily-check.sh b/scripts/daily-check.sh index 2a7beaf..1515046 100755 --- a/scripts/daily-check.sh +++ b/scripts/daily-check.sh @@ -26,13 +26,15 @@ METRICS=$(curl -s $BASE/api/v1/metrics/self -H "X-API-Key: $API_KEY" 2>/dev/null echo "$METRICS" | python3 -c " import sys,json d=json.load(sys.stdin) -hit = d.get('recall_hit_rate',0) -loss = d.get('avg_distill_loss',0) -gap = d.get('gap_closure_rate',0) -auto = d.get('auto_resolve_rate',0) -print(f'命中率: {hit:.0%}, 蒸馏损失: {loss:.2f}, gap闭合: {gap}/天, auto_resolve: {auto}') +# 2026-10-17:recall_hit_rate 已退役(恒=1、无区分度)。召回质量改用 recall_usefulness_rate +#(返回且被使用);-1 表示样本不足/无负反馈 = 数据不可解读,跳过告警,不误报。 +hit = d.get('recall_usefulness_rate', -1) +loss = d.get('avg_distill_loss', 0) +gap = d.get('gap_closure_rate', 0) +auto = d.get('auto_resolve_rate', 0) +print(f'召回有用率: {hit:.0%}, 蒸馏损失: {loss:.2f}, gap闭合: {gap}/天, auto_resolve: {auto}') if loss > 0.3: print('⚠️ avg_distill_loss 超标') -if hit < 0.5: print('⚠️ recall_hit_rate 低于阈值') +if hit >= 0 and hit < 0.5: print('⚠️ recall_usefulness_rate 低于阈值') " 2>/dev/null # 4. 蒸馏状态 diff --git a/skills/zhiyi/scripts/daily-check.sh b/skills/zhiyi/scripts/daily-check.sh index 2a7beaf..1515046 100755 --- a/skills/zhiyi/scripts/daily-check.sh +++ b/skills/zhiyi/scripts/daily-check.sh @@ -26,13 +26,15 @@ METRICS=$(curl -s $BASE/api/v1/metrics/self -H "X-API-Key: $API_KEY" 2>/dev/null echo "$METRICS" | python3 -c " import sys,json d=json.load(sys.stdin) -hit = d.get('recall_hit_rate',0) -loss = d.get('avg_distill_loss',0) -gap = d.get('gap_closure_rate',0) -auto = d.get('auto_resolve_rate',0) -print(f'命中率: {hit:.0%}, 蒸馏损失: {loss:.2f}, gap闭合: {gap}/天, auto_resolve: {auto}') +# 2026-10-17:recall_hit_rate 已退役(恒=1、无区分度)。召回质量改用 recall_usefulness_rate +#(返回且被使用);-1 表示样本不足/无负反馈 = 数据不可解读,跳过告警,不误报。 +hit = d.get('recall_usefulness_rate', -1) +loss = d.get('avg_distill_loss', 0) +gap = d.get('gap_closure_rate', 0) +auto = d.get('auto_resolve_rate', 0) +print(f'召回有用率: {hit:.0%}, 蒸馏损失: {loss:.2f}, gap闭合: {gap}/天, auto_resolve: {auto}') if loss > 0.3: print('⚠️ avg_distill_loss 超标') -if hit < 0.5: print('⚠️ recall_hit_rate 低于阈值') +if hit >= 0 and hit < 0.5: print('⚠️ recall_usefulness_rate 低于阈值') " 2>/dev/null # 4. 蒸馏状态