xiaoxue_admin/memoryweave
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(metrics): t_c70f385c 指标口径修复 — recall_hit_rate 退役 + gap_closure_rate 断言 + 最小样本门 N=20

Browse Source 
- recall_hit_rate 退役：分子原为 core.go 的 len(results)>0（恒=1、无区分度），
  「返回且被使用」语义已由 recall_usefulness_rate 承担（207 样本 0.98）。
  Metrics()/distributed/metrics-gauge/executor/tuning/CLI/daily-check 全链移除该键。
- gap_closure_rate 统一为「缺口个数」口径：Close() 加 !g.Closed 守卫（与 RecordHit 一致），
  根治同一缺口重复关闭导致 ClosedGaps 虚增、越界 4.5；Metrics() 加越界断言 gapRate>1 → -1。
- 所有率类指标加最小样本门 minRateSamples=20：样本<20 ⇒ -1 + valid=0，
  杜绝 auto_resolve_rate 2 样本判满分。
- 测试：metrics_retire_regression_test.go 用生产同款计数复现改前/改后 JSON；
  selfoptimize 全套通过。
This commit is contained in:
小唯
2026-10-17 20:32:34 +08:00
parent cee68592e3
commit b2a016e90f
12 changed files with 306 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cli-anything/cli_anything/zhiyi/zhiyi_cli.py
+1 -2
View File
@@ -197,7 +197,6 @@ def stats(stat_type, json_flag):
if "metrics" in data:
m = data["metrics"]
click.echo(f"\n📊 自优化指标:")
click.echo(f" 召回命中率: {m.get('recall_hit_rate', '?'):.1%}")
click.echo(f" 召回有用率: {m.get('recall_usefulness_rate', '?'):.1%}")
click.echo(f" 记忆总数: {m.get('total_memories', '?')}")
@@ -429,7 +428,7 @@ def _repl_with_skin(c: ZhiYiClient, skin):
["记忆数", str(s.get("total_memories", "?"))],
["图谱节点", str(g.get("node_count", "?"))],
["图谱边", str(g.get("edge_count", "?"))],
["召回命中率", f"{m.get('recall_hit_rate', '?'):.1%}"],
["召回有用率", f"{m.get('recall_usefulness_rate', '?'):.1%}"],
],
)
elif cmd == "graph" and args and args[0] == "navigate":
go/internal/api/routes/tuning.go
+8 -6
View File
@@ -204,20 +204,22 @@ func (tc *TuningConfig) AutoTune(metrics map[string]float64) []TuningEvent {
skipped = append(skipped, rule+": "+why)
}
// 1. 召回命中率 < 0.5 → 提高 diversity（扩大搜索范围）
hitRate, hitOK := ratioFor(metrics, "recall_hit_rate")
// 1. 召回质量低 → 提高 diversity（扩大搜索范围）。
// 2026-10-17：原信号 recall_hit_rate 已退役（恒=1、从不及 0.5 ⇒ 本安全阀长期死寂）。
// 改用 recall_usefulness_rate（返回且被使用）作为召回质量的实时信号：< 0.5 ⇒ 召回太窄、扩大范围。
hitRate, hitOK := ratioFor(metrics, "recall_usefulness_rate")
if !hitOK {
note("rule1_hit_rate", "指标不可用（无数据/退化/样本<30）")
note("rule1_usefulness", "指标不可用（无数据/退化/样本<20）")
}
if hitOK && hitRate < 0.5 && !tc.canMove("diversity") {
note("rule1_hit_rate", "冷却中（距上次调整 <"+tuneCooldown.String()+"）")
note("rule1_usefulness", "冷却中（距上次调整 <"+tuneCooldown.String()+"）")
}
if hitOK && hitRate < 0.5 && tc.canMove("diversity") {
newDiversity := tc.Diversity + 0.1
if newDiversity > 1.0 {
newDiversity = 1.0
}
events = append(events, recordTuning(tc, "diversity", tc.Diversity, newDiversity, "recall_hit_rate < 0.5"))
events = append(events, recordTuning(tc, "diversity", tc.Diversity, newDiversity, "recall_usefulness_rate < 0.5"))
tc.Diversity = newDiversity
tc.markMove("diversity")
}
@@ -358,7 +360,7 @@ func (tc *TuningConfig) AnalyticsHandler(w http.ResponseWriter, r *http.Request)
m := selfoptimize.Dash.Metrics()
suggestions := []string{}
if m["recall_hit_rate"] < 0.5 && m["avg_distill_loss"] > 0.3 {
if m["recall_usefulness_rate"] < 0.5 && m["avg_distill_loss"] > 0.3 {
suggestions = append(suggestions, "蒸馏质量低导致召回率低 — 建议检查 Embedding 模型")
}
if m["deprecated_per_day"] > 10 {
go/internal/api/routes/tuning_guard_test.go
-2
View File
@@ -23,10 +23,8 @@ func TestAutoTuneGuard_SentinelAndSamples(t *testing.T) {
"gap_closure_samples": 0,
"gap_closure_valid": 0,
"recall_usefulness_rate": -1,
"recall_hit_rate": -1,
"avg_distill_loss": -1,
"recall_usefulness_valid": 0,
"recall_hit_valid": 0,
"avg_distill_loss_valid": 0,
}
if ev := tc.AutoTune(na); len(ev) != 0 {
go/internal/distributed/distributed.go
+1 -9
View File
@@ -233,15 +233,7 @@ func (mh *MetricsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "# TYPE zhiyi_recall_usefulness_samples gauge\n")
fmt.Fprintf(w, "zhiyi_recall_usefulness_samples %f\n", metrics["recall_usefulness_samples"])
fmt.Fprintf(w, "# HELP zhiyi_recall_hit_rate Recall命中率（分子=召回返回非空）；-1=无数据\n")
fmt.Fprintf(w, "# TYPE zhiyi_recall_hit_rate gauge\n")
fmt.Fprintf(w, "zhiyi_recall_hit_rate %f\n", metrics["recall_hit_rate"])
fmt.Fprintf(w, "# HELP zhiyi_recall_hit_valid 该率是否可解读；0 ⇒ 值为 -1\n")
fmt.Fprintf(w, "# TYPE zhiyi_recall_hit_valid gauge\n")
fmt.Fprintf(w, "zhiyi_recall_hit_valid %f\n", metrics["recall_hit_valid"])
fmt.Fprintf(w, "# HELP zhiyi_recall_hit_samples 该率的分母样本数\n")
fmt.Fprintf(w, "# TYPE zhiyi_recall_hit_samples gauge\n")
fmt.Fprintf(w, "zhiyi_recall_hit_samples %f\n", metrics["recall_hit_samples"])
// 2026-10-17：zhiyi_recall_hit_rate 已退役（恒=1、无区分度），由 recall_usefulness_rate 承担「返回且被使用」语义。
fmt.Fprintf(w, "# HELP zhiyi_gap_closure_rate 缺口闭环率；-1=无数据\n")
fmt.Fprintf(w, "# TYPE zhiyi_gap_closure_rate gauge\n")
go/internal/metrics/metrics.go
-10
View File
@@ -84,12 +84,6 @@ var (
Help: "recall 有用率 (0-1)",
},
)
RecallHitRate = prometheus.NewGauge(
prometheus.GaugeOpts{
Name: "zhiyi_recall_hit_rate",
Help: "recall 命中率 (0-1)",
},
)
GapClosureRate = prometheus.NewGauge(
prometheus.GaugeOpts{
Name: "zhiyi_gap_closure_rate",
@@ -136,7 +130,6 @@ func init() {
MemoryRSSBytes,
SQLiteDBSizeBytes,
RecallUsefulnessRate,
RecallHitRate,
GapClosureRate,
DeprecatedPerDay,
AutoResolveRate,
@@ -155,9 +148,6 @@ func SyncFromDashboard(m map[string]float64) {
if v, ok := m["recall_usefulness_rate"]; ok {
RecallUsefulnessRate.Set(v)
}
if v, ok := m["recall_hit_rate"]; ok {
RecallHitRate.Set(v)
}
if v, ok := m["gap_closure_rate"]; ok {
GapClosureRate.Set(v)
}
go/internal/selfoptimize/executor.go
+4 -2
View File
@@ -119,8 +119,10 @@ func (te *TriggerExecutor) evaluate(t *TriggerDef, metrics map[string]float64) b
threshold, _ := t.Condition.(float64)
return hours >= threshold
case TRecallMiss:
// 检查最近的 miss 计数
rate, ok := metrics["recall_hit_rate"]
// 检查最近的 miss 计数。
// 2026-10-17：recall_hit_rate 已退役（恒=1、无区分度），改用 recall_usefulness_rate
// （返回且被使用）作为「召回已产生可用结果、应清除 miss」的信号。
rate, ok := metrics["recall_usefulness_rate"]
return ok && rate > 0 && t.LastFired.IsZero()
case TQualityDrop:
threshold, _ := t.Condition.(float64)
go/internal/selfoptimize/metrics_na_test.go
+19 -17
View File
@@ -19,7 +19,6 @@ func TestMetricsDegenerateReturnsNA(t *testing.T) {
{"无反馈样本", &Dashboard{}, "recall_usefulness_rate"},
{"有正反馈但无负反馈（旧实现返回 1 的情形）",
&Dashboard{UsefulCount: 46, NotUsefulCount: 0}, "recall_usefulness_rate"},
{"从未召回", &Dashboard{}, "recall_hit_rate"},
{"无缺口记录", &Dashboard{}, "gap_closure_rate"},
{"无修复记录", &Dashboard{}, "cascade_fix_rate"},
{"无冲突记录", &Dashboard{}, "auto_resolve_rate"},
@@ -50,25 +49,24 @@ func TestMetricsHealthyPathStillComputes(t *testing.T) {
NotUsefulCount: 30,
TotalRecalls: 200,
HitCount: 150,
TotalGaps: 10,
ClosedGaps: 4,
TotalGaps: 20, // ≥ 最小样本门 N=20
ClosedGaps: 8,
TotalFixes: 25,
CascadeFixedTotal: 10,
TotalConflicts: 8,
AutoResolvedConflicts: 2,
DistillLossSum: 1.0,
DistillLossCount: 2,
TotalConflicts: 20, // ≥ 最小样本门 N=20
AutoResolvedConflicts: 5,
DistillLossSum: 10.0,
DistillLossCount: 20, // ≥ 最小样本门 N=20
DeprecatedToday: 31,
}
m := d.Metrics()
want := map[string]float64{
"recall_usefulness_rate": 0.70, // 70/(70+30)
"recall_hit_rate": 0.75, // 150/200
"gap_closure_rate": 0.40, // 4/10
"gap_closure_rate": 0.40, // 8/20
"cascade_fix_rate": 0.40, // 10/25
"auto_resolve_rate": 0.25, // 2/8
"avg_distill_loss": 0.50, // 1.0/2
"auto_resolve_rate": 0.25, // 5/20
"avg_distill_loss": 0.50, // 10.0/20
"deprecated_per_day": 31,
}
for k, w := range want {
@@ -77,9 +75,9 @@ func TestMetricsHealthyPathStillComputes(t *testing.T) {
}
}
// 健康路径所有 *_valid 必须为 1
// 健康路径所有 *_valid 必须为 1（recall_hit_valid 已于 2026-10-17 随 recall_hit_rate 退役）
for _, k := range []string{
"recall_usefulness_valid", "recall_hit_valid", "gap_closure_valid",
"recall_usefulness_valid", "gap_closure_valid",
"cascade_fix_valid", "auto_resolve_valid", "avg_distill_loss_valid",
} {
if m[k] != 1 {
@@ -93,9 +91,8 @@ func TestMetricsSamplesReported(t *testing.T) {
d := Dashboard{UsefulCount: 46, TotalRecalls: 1000, TotalGaps: 3}
m := d.Metrics()
want := map[string]float64{
"recall_usefulness_samples": 46, // 46+0
"recall_hit_samples": 1000, // 1000
"gap_closure_samples": 3, // 3
"recall_usefulness_samples": 46, // 46+0
"gap_closure_samples": 3, // 3
}
for k, w := range want {
if got := m[k]; got != w {
@@ -106,9 +103,10 @@ func TestMetricsSamplesReported(t *testing.T) {
// TestMetricsKeyNamesAreBackwardCompatible —— 只增键、不改键名：
// 所有旧键必须仍然存在（否则 SyncFromDashboard / 各 JSON 端点会静默丢指标）。
// 2026-10-17 例外：recall_hit_rate 是**有意退役**（恒=1、无区分度），不在兼容契约内。
func TestMetricsKeyNamesAreBackwardCompatible(t *testing.T) {
legacy := []string{
"recall_usefulness_rate", "recall_hit_rate", "gap_closure_rate",
"recall_usefulness_rate", "gap_closure_rate",
"cascade_fix_rate", "deprecated_per_day", "avg_distill_loss",
"auto_resolve_rate",
}
@@ -118,6 +116,10 @@ func TestMetricsKeyNamesAreBackwardCompatible(t *testing.T) {
t.Errorf("旧键 %q 丢失——本改动必须只增不改", k)
}
}
// 退役键不得复出
if _, ok := m["recall_hit_rate"]; ok {
t.Errorf("recall_hit_rate 应已退役，不得复出")
}
}
// TestRateOrNA —— 辅助函数边界：den<=0 一律 -1；否则两位四舍五入。
go/internal/selfoptimize/metrics_retire_regression_test.go
+106
View File
@@ -0,0 +1,106 @@
// 织忆 MemoryWeave — 2026-10-17 指标口径修复回归测试
//
// 覆盖三处缺陷的修复：
// ① recall_hit_rate 退役（恒=1、无区分度）⇒ Metrics() 不再输出该键
// ② gap_closure_rate 越界（4.5）⇒ 统一为「缺口个数」口径 + 越界断言拦截
// ③ 最小样本门 N=20 ⇒ 样本 < 20 一律 -1 + valid=0（杜绝 auto_resolve 2 样本满分）
//
// 本测试用【生产同款计数器】复现改前状态（recall_hit 21252/1、gap 4.5/2、
// auto_resolve 1/2、usefulness 207/0.98），并断言改后 /api/v1/metrics 语义。
package selfoptimize
import (
"encoding/json"
"os"
"testing"
)
// prodLikeDash 复现生产 /api/v1/metrics 的改前计数状态（2026-10-17 19:10 实测值）。
func prodLikeDash() *Dashboard {
return &Dashboard{
TotalRecalls: 21252,
HitCount: 21252,
UsefulCount: 203,
NotUsefulCount: 4,
DistillLossSum: 2732.58, // avg 0.51 * 5358（近似）
DistillLossCount: 5358,
ClosedGaps: 9, // 4.5 * 2（改前 ClosedGaps/TotalGaps 口径错混，越界）
TotalGaps: 2,
TotalFixes: 0,
CascadeFixedTotal: 0,
AutoResolvedConflicts: 2,
TotalConflicts: 2,
DeprecatedToday: 31,
DistillSuccessCount: 3163,
}
}
// TestMetricsRetireHitRate —— ① recall_hit_rate 退役：Metrics() 不再输出该键及配套键。
func TestMetricsRetireHitRate(t *testing.T) {
d := prodLikeDash()
m := d.Metrics()
if _, ok := m["recall_hit_rate"]; ok {
t.Fatalf("recall_hit_rate 应已退役，Metrics() 不得再输出")
}
if _, ok := m["recall_hit_samples"]; ok {
t.Fatalf("recall_hit_samples 应已退役")
}
if _, ok := m["recall_hit_valid"]; ok {
t.Fatalf("recall_hit_valid 应已退役")
}
}
// TestMetricsGapClosureBounded —— ② gap_closure_rate 越界断言：
// 改前 4.5（ClosedGaps=9/TotalGaps=2，写入侧把关闭事件数混入分子）⇒ 改后断言拦截为 -1 + valid=0。
func TestMetricsGapClosureBounded(t *testing.T) {
d := prodLikeDash()
m := d.Metrics()
if v := m["gap_closure_rate"]; v != metricNA {
t.Fatalf("越界 gap_closure_rate 应被断言拦截为 -1，got %v", v)
}
if v := m["gap_closure_valid"]; v != 0 {
t.Fatalf("越界时 gap_closure_valid 应为 0，got %v", v)
}
}
// TestMetricsMinSampleGate —— ③ 最小样本门 N=20：
// auto_resolve 仅 2 样本（改前 1 分/valid=1）⇒ 改后 -1 + valid=0；gap 2 样本同样 -1。
func TestMetricsMinSampleGate(t *testing.T) {
d := prodLikeDash()
m := d.Metrics()
if v := m["auto_resolve_rate"]; v != metricNA {
t.Fatalf("auto_resolve 样本 2<20 应为 -1，got %v", v)
}
if v := m["auto_resolve_valid"]; v != 0 {
t.Fatalf("auto_resolve 样本 2<20 应为 valid=0，got %v", v)
}
// usefulness 有 207 样本 ≥ 20 且负反馈存在 ⇒ 可读（0.98），证明门没把好数据焊死
if v := m["recall_usefulness_rate"]; v < 0.97 || v > 0.99 {
t.Fatalf("usefulness 207 样本应可读为 ~0.98，got %v", v)
}
if v := m["recall_usefulness_valid"]; v != 1 {
t.Fatalf("usefulness 应 valid=1，got %v", v)
}
}
// TestMetricsProdBeforeAfterJSON —— 复现改前/改后 /api/v1/metrics 的 JSON 输出，
// 供验收附「改前/改后两次原始 JSON」（前置断言保证改后语义；本测试打印改后 JSON）。
func TestMetricsProdBeforeAfterJSON(t *testing.T) {
d := prodLikeDash()
// 断言改后 key 集（与改前对比：recall_hit_* 移除）
m := d.Metrics()
for _, absent := range []string{"recall_hit_rate", "recall_hit_samples", "recall_hit_valid"} {
if _, ok := m[absent]; ok {
t.Fatalf("%s 不应出现在改后 Metrics()", absent)
}
}
// 打印改后 JSON（同 /api/v1/metrics 序列化路径）
out, err := json.MarshalIndent(m, "", " ")
if err != nil {
t.Fatal(err)
}
t.Logf("改后 /api/v1/metrics JSON:\n%s", string(out))
if os.Getenv("PRINT_METRICS_JSON") == "1" {
println(string(out))
}
}
go/internal/selfoptimize/selfoptimize.go
+57 -18
View File
@@ -260,6 +260,13 @@ func rateOrNA(num, den int) float64 {
return math.Round(float64(num)/float64(den)*100) / 100
}
// minRateSamples 是率类指标的最小样本门。
//
// 样本太少（< 20）时，即使比率为真值也**统计上不可解读** —— 例如 auto_resolve_rate
// 曾仅 2 个样本即判 valid=1 满分。现在样本不足一律按退化处理（metricNA + valid=0），
// 不参与面板解读，避免「小样本满分」误导。
const minRateSamples = 20
// b2f 布尔转 0/1，用于 *_valid 键。
func b2f(b bool) float64 {
if b {
@@ -278,50 +285,80 @@ func (d *Dashboard) Metrics() map[string]float64 {
// 负样本通道为空 ⇒ 无从判断好坏（详见 metricNA 注释）。
usefulRate := metricNA
usefulSamples := d.UsefulCount + d.NotUsefulCount
if d.NotUsefulCount > 0 {
usefulRate = rateOrNA(d.UsefulCount, d.UsefulCount+d.NotUsefulCount)
usefulRate = rateOrNA(d.UsefulCount, usefulSamples)
}
hitRate := rateOrNA(d.HitCount, d.TotalRecalls)
// recall_hit_rate 已于 2026-10-17 退役：其分子原本是 core.go 的 `len(results) > 0`，
// 量的是「召回有没有返回东西」而非「返回的对不对」，恒为 1、无区分度。
// 「返回且被使用」的语义已由 recall_usefulness_rate 承担（198 样本 0.98）。
// 禁止保留恒等于 1 的指标挂在面板上误导人 ⇒ 从 Metrics() 移除该键。
gapRate := rateOrNA(d.ClosedGaps, d.TotalGaps)
cascadeRate := rateOrNA(d.CascadeFixedTotal, d.TotalFixes)
autoRate := rateOrNA(d.AutoResolvedConflicts, d.TotalConflicts)
// gap_closure_rate 断言：ClosedGaps/TotalGaps 统一为「缺口个数」口径后，比率必落 [0,1]。
// 若写入侧再次把「关闭事件数」混进来导致越界，断言拦截 → -1 + valid=0（禁止越界值上板）。
if gapRate > 1 {
gapRate = metricNA
}
// 均值项：零样本的均值同样是退化，按同一规则返回 metricNA。
avgLoss := metricNA
if d.DistillLossCount > 0 {
avgLoss = math.Round(d.DistillLossSum/float64(d.DistillLossCount)*100) / 100
}
// 最小样本门：samples < minRateSamples ⇒ 值 = -1 且 valid = 0，不参与面板解读。
usefulVal, usefulValid := usefulRate, b2f(usefulSamples >= minRateSamples && d.NotUsefulCount > 0)
gapVal, gapValid := gapRate, b2f(d.TotalGaps >= minRateSamples)
cascadeVal, cascadeValid := cascadeRate, b2f(d.TotalFixes >= minRateSamples)
autoVal, autoValid := autoRate, b2f(d.TotalConflicts >= minRateSamples)
lossVal, lossValid := avgLoss, b2f(d.DistillLossCount >= minRateSamples)
if usefulSamples < minRateSamples || d.NotUsefulCount == 0 {
usefulVal = metricNA
}
if d.TotalGaps < minRateSamples || gapVal == metricNA {
// 样本不足 或 越界断言触发 ⇒ 值 -1 且 valid=0（值与 flag 保持同步）。
gapVal = metricNA
gapValid = 0
}
if d.TotalFixes < minRateSamples {
cascadeVal = metricNA
}
if d.TotalConflicts < minRateSamples {
autoVal = metricNA
}
if d.DistillLossCount < minRateSamples {
lossVal = metricNA
}
return map[string]float64{
"recall_usefulness_rate": usefulRate,
"recall_usefulness_samples": float64(d.UsefulCount + d.NotUsefulCount),
"recall_usefulness_rate": usefulVal,
"recall_usefulness_samples": float64(usefulSamples),
"recall_usefulness_negative": float64(d.NotUsefulCount),
"recall_usefulness_positive": float64(d.UsefulCount),
"recall_usefulness_valid": b2f(d.NotUsefulCount > 0),
"recall_usefulness_valid": usefulValid,
"recall_hit_rate": hitRate,
"recall_hit_samples": float64(d.TotalRecalls),
"recall_hit_valid": b2f(d.TotalRecalls > 0),
"gap_closure_rate": gapRate,
"gap_closure_rate": gapVal,
"gap_closure_samples": float64(d.TotalGaps),
"gap_closure_valid": b2f(d.TotalGaps > 0),
"gap_closure_valid": gapValid,
"cascade_fix_rate": cascadeRate,
"cascade_fix_rate": cascadeVal,
"cascade_fix_samples": float64(d.TotalFixes),
"cascade_fix_valid": b2f(d.TotalFixes > 0),
"cascade_fix_valid": cascadeValid,
"auto_resolve_rate": autoRate,
"auto_resolve_rate": autoVal,
"auto_resolve_samples": float64(d.TotalConflicts),
"auto_resolve_valid": b2f(d.TotalConflicts > 0),
"auto_resolve_valid": autoValid,
"deprecated_per_day": float64(d.DeprecatedToday),
"distill_success_count": float64(d.DistillSuccessCount),
"avg_distill_loss": avgLoss,
"avg_distill_loss": lossVal,
"avg_distill_loss_samples": float64(d.DistillLossCount),
"avg_distill_loss_valid": b2f(d.DistillLossCount > 0),
"avg_distill_loss_valid": lossValid,
}
}
@@ -604,7 +641,9 @@ func (gd *GapDetector) List() []*Gap {
func (gd *GapDetector) Close(topic string) {
gd.mu.Lock()
defer gd.mu.Unlock()
if g, ok := gd.gaps[topic]; ok {
// 统一「缺口个数」口径：仅当缺口从未关闭时才算一次闭环（与 RecordHit 一致），
// 否则同一缺口被 Close 重复调用会让 ClosedGaps 虚增、gap_closure_rate 越界 > 1。
if g, ok := gd.gaps[topic]; ok && !g.Closed {
g.Closed = true
gd.persistGaps()
Dash.RecordGapClosed() // 同步仪表盘
go/internal/selfoptimize/selfoptimize_test.go
+94 -34
View File
@@ -50,8 +50,10 @@ func TestDashboard_InitialMetricsNoData(t *testing.T) {
if m["recall_usefulness_rate"] != metricNA {
t.Errorf("expected metricNA(%v), got %f", metricNA, m["recall_usefulness_rate"])
}
if m["recall_hit_rate"] != metricNA {
t.Errorf("expected metricNA(%v), got %f", metricNA, m["recall_hit_rate"])
// 2026-10-17：recall_hit_rate 已退役（恒=1、无区分度），其「返回且被使用」语义由
// recall_usefulness_rate 承担。Metrics() 不再输出该键 ⇒ 断言键不存在（读它应得 0，会误导）。
if _, ok := m["recall_hit_rate"]; ok {
t.Errorf("recall_hit_rate 已退役，Metrics() 不应再输出该键")
}
if m["recall_usefulness_valid"] != 0 {
t.Errorf("expected valid=0, got %f", m["recall_usefulness_valid"])
@@ -60,10 +62,13 @@ func TestDashboard_InitialMetricsNoData(t *testing.T) {
func TestDashboard_UsefulnessRate(t *testing.T) {
d := &Dashboard{}
d.RecordFeedback(true)
d.RecordFeedback(true)
d.RecordFeedback(true)
d.RecordFeedback(false)
// 满足最小样本门 N=20：15 有用 + 5 无用 = 20 样本 ⇒ 15/20 = 0.75
for i := 0; i < 15; i++ {
d.RecordFeedback(true)
}
for i := 0; i < 5; i++ {
d.RecordFeedback(false)
}
m := d.Metrics()
if m["recall_usefulness_rate"] != 0.75 {
@@ -71,6 +76,21 @@ func TestDashboard_UsefulnessRate(t *testing.T) {
}
}
// TestDashboard_UsefulnessRateBelowMinSamples —— 最小样本门（N=20）：反馈样本 < 20 ⇒ -1 + valid=0。
func TestDashboard_UsefulnessRateBelowMinSamples(t *testing.T) {
d := &Dashboard{}
for i := 0; i < 2; i++ { // 仅 2 样本（旧 auto_resolve_rate 同类误判满分）
d.RecordFeedback(true)
}
m := d.Metrics()
if m["recall_usefulness_rate"] != metricNA {
t.Errorf("样本 < 20 应为 metricNA(-1), got %f", m["recall_usefulness_rate"])
}
if m["recall_usefulness_valid"] != 0 {
t.Errorf("样本 < 20 应为 valid=0, got %f", m["recall_usefulness_valid"])
}
}
// TestDashboard_DistillDoesNotPolluteUsefulnessRate —— 2026-10-14 t_c06ad4d0 解污染回归。
// 蒸馏回调改走 RecordDistillSuccess()（独立计数 distill_success_count），
// 不得再污染 recall_usefulness_rate 的分子/分母。10 次蒸馏成功 ⇒ UsefulCount 仍为 0。
@@ -98,28 +118,62 @@ func TestDashboard_DistillDoesNotPolluteUsefulnessRate(t *testing.T) {
}
}
func TestDashboard_HitRate(t *testing.T) {
// TestDashboard_HitRateRetired —— 2026-10-17：recall_hit_rate 已退役。
// 其分子原是 core.go 的 `len(results)>0`（恒=1、无区分度），「返回且被使用」语义由
// recall_usefulness_rate 承担。RecordRecall 仍累积 TotalRecalls/HitCount（供持久化/审计），
// 但 Metrics() 不再输出 recall_hit_rate 键。
func TestDashboard_HitRateRetired(t *testing.T) {
d := &Dashboard{}
d.RecordRecall(true)
d.RecordRecall(true)
d.RecordRecall(false)
m := d.Metrics()
if m["recall_hit_rate"] < 0.66 || m["recall_hit_rate"] > 0.67 {
t.Errorf("expected ~0.67, got %f", m["recall_hit_rate"])
if _, ok := m["recall_hit_rate"]; ok {
t.Errorf("recall_hit_rate 已退役，Metrics() 不应再输出该键")
}
if m["recall_hit_samples"] != 0 {
t.Errorf("recall_hit_samples 已退役，不应输出（got %f）", m["recall_hit_samples"])
}
}
func TestDashboard_GapClosureRate(t *testing.T) {
d := &Dashboard{}
d.TotalGaps = 10
d.ClosedGaps = 5
d.TotalGaps = 20 // ≥ 最小样本门 N=20
d.ClosedGaps = 10
m := d.Metrics()
if m["gap_closure_rate"] != 0.5 {
t.Errorf("expected 0.5, got %f", m["gap_closure_rate"])
}
}
// TestDashboard_GapClosureBelowMinSamples —— 最小样本门（N=20）+ 越界断言：
// TotalGaps < 20 ⇒ -1 + valid=0；ClosedGaps > TotalGaps（写入侧口径错）⇒ 断言拦截为 -1 + valid=0。
func TestDashboard_GapClosureBelowMinSamples(t *testing.T) {
d := &Dashboard{}
d.TotalGaps = 2
d.ClosedGaps = 1
m := d.Metrics()
if m["gap_closure_rate"] != metricNA {
t.Errorf("TotalGaps < 20 应为 metricNA(-1), got %f", m["gap_closure_rate"])
}
if m["gap_closure_valid"] != 0 {
t.Errorf("TotalGaps < 20 应为 valid=0, got %f", m["gap_closure_valid"])
}
// 越界断言：ClosedGaps > TotalGaps（同类 4.5 越界值）⇒ 拦截为 -1，禁止上板
d2 := &Dashboard{}
d2.TotalGaps = 20
d2.ClosedGaps = 30 // 关闭事件数混入 ⇒ 越界
m2 := d2.Metrics()
if m2["gap_closure_rate"] != metricNA {
t.Errorf("ClosedGaps > TotalGaps 越界应被断言拦截为 metricNA(-1), got %f", m2["gap_closure_rate"])
}
if m2["gap_closure_valid"] != 0 {
t.Errorf("越界时 valid 应为 0, got %f", m2["gap_closure_valid"])
}
}
func TestDashboard_RecordDeprecation(t *testing.T) {
d := &Dashboard{}
d.RecordDeprecation()
@@ -143,13 +197,18 @@ func TestDashboard_RecordCorrection(t *testing.T) {
func TestDashboard_RecordConflictResolved(t *testing.T) {
d := &Dashboard{}
d.RecordConflictResolved(true)
d.RecordConflictResolved(false)
if d.TotalConflicts != 2 {
t.Errorf("expected 2 conflicts, got %d", d.TotalConflicts)
// 满足最小样本门 N=20：10 自动 + 10 手动 = 20 冲突 ⇒ 10/20 = 0.5
for i := 0; i < 10; i++ {
d.RecordConflictResolved(true)
}
if d.AutoResolvedConflicts != 1 {
t.Errorf("expected 1 auto, got %d", d.AutoResolvedConflicts)
for i := 0; i < 10; i++ {
d.RecordConflictResolved(false)
}
if d.TotalConflicts != 20 {
t.Errorf("expected 20 conflicts, got %d", d.TotalConflicts)
}
if d.AutoResolvedConflicts != 10 {
t.Errorf("expected 10 auto, got %d", d.AutoResolvedConflicts)
}
m := d.Metrics()
if m["auto_resolve_rate"] != 0.5 {
@@ -384,10 +443,10 @@ func TestDashboardLocalPersistence_RoundTrip(t *testing.T) {
t.Setenv("ZHIYI_DASHBOARD_PATH", p)
src := &Dashboard{
TotalRecalls: 7, HitCount: 6, UsefulCount: 5, NotUsefulCount: 1,
DistillLossSum: 1.05, DistillLossCount: 5,
ClosedGaps: 2, TotalGaps: 4, TotalFixes: 3, CascadeFixedTotal: 1,
AutoResolvedConflicts: 2, TotalConflicts: 3, DeprecatedToday: 4,
TotalRecalls: 7, HitCount: 6, UsefulCount: 15, NotUsefulCount: 5, // 20 样本 ≥ N
DistillLossSum: 4.2, DistillLossCount: 20,
ClosedGaps: 10, TotalGaps: 20, TotalFixes: 30, CascadeFixedTotal: 10,
AutoResolvedConflicts: 20, TotalConflicts: 30, DeprecatedToday: 4,
}
src.mu.Lock()
src.persist() // 测试环境无 Redis → 走本地 JSON 分支
@@ -403,13 +462,12 @@ func TestDashboardLocalPersistence_RoundTrip(t *testing.T) {
}
m := Dash.Metrics()
want := map[string]float64{
"recall_hit_rate": 0.86, // 6/7
"recall_usefulness_rate": 0.83, // 5/6
"gap_closure_rate": 0.5, // 2/4
"cascade_fix_rate": 0.33, // 1/3
"auto_resolve_rate": 0.67, // 2/3
"recall_usefulness_rate": 0.75, // 15/20
"gap_closure_rate": 0.5, // 10/20
"cascade_fix_rate": 0.33, // 10/30
"auto_resolve_rate": 0.67, // 20/30
"deprecated_per_day": 4,
"avg_distill_loss": 0.21, // 1.05/5
"avg_distill_loss": 0.21, // 4.2/20
}
for k, v := range want {
if m[k] != v {
@@ -434,8 +492,10 @@ func TestDashboardLocalPersistence_StaleDeprecatedResetsToZero(t *testing.T) {
if m["deprecated_per_day"] != 0 {
t.Errorf("跨天未归零: %v", m["deprecated_per_day"])
}
if m["recall_hit_rate"] != 0.67 { // 2/3
t.Errorf("其他计数应保留, got %v", m["recall_hit_rate"])
// 2026-10-17：recall_hit_rate 已退役。blob 无反馈计数 ⇒ 用 recall_usefulness_rate
// 确认「其余计数状态被保留」（无反馈 = metricNA，未被 load 改坏）。
if m["recall_usefulness_rate"] != metricNA {
t.Errorf("recall_usefulness_rate 应保留为 metricNA（blob 无反馈计数）, got %v", m["recall_usefulness_rate"])
}
}
@@ -445,13 +505,13 @@ func TestDashboardLocalPersistence_MissingOrBrokenFileIsSafe(t *testing.T) {
zeroDash()
// 本测试的本意是「load 失败不得改动 Dash」——所以基准值取「零状态的 Metrics 值」，
// 而不是硬编码 0。2026-10-14 语义变更后，零状态的 recall_hit_rate 是 metricNA(-1)
// 而非 0（无样本 ⇒ 无结论，不是「0% 命中」）。
want := Dash.Metrics()["recall_hit_rate"]
// 而不是硬编码 0。2026-10-17 起 recall_hit_rate 已退役，改用 recall_usefulness_rate
// 作基准（零状态为 metricNA(-1)，无样本 ⇒ 无结论）。
want := Dash.Metrics()["recall_usefulness_rate"]
if err := loadDashboardLocal(); err == nil {
t.Errorf("缺文件应返回 err（调用方忽略，行为退回仅内存）")
}
if got := Dash.Metrics()["recall_hit_rate"]; got != want {
if got := Dash.Metrics()["recall_usefulness_rate"]; got != want {
t.Errorf("缺文件不应改动 Dash：期望 %v，实际 %v", want, got)
}
@@ -461,7 +521,7 @@ func TestDashboardLocalPersistence_MissingOrBrokenFileIsSafe(t *testing.T) {
if err := loadDashboardLocal(); err == nil {
t.Errorf("坏文件应返回 err")
}
if got := Dash.Metrics()["recall_hit_rate"]; got != want {
if got := Dash.Metrics()["recall_usefulness_rate"]; got != want {
t.Errorf("坏文件不应改动 Dash：期望 %v，实际 %v", want, got)
}
}
scripts/daily-check.sh
+8 -6
View File
@@ -26,13 +26,15 @@ METRICS=$(curl -s $BASE/api/v1/metrics/self -H "X-API-Key: $API_KEY" 2>/dev/null
echo "$METRICS" | python3 -c "
import sys,json
d=json.load(sys.stdin)
hit = d.get('recall_hit_rate',0)
loss = d.get('avg_distill_loss',0)
gap = d.get('gap_closure_rate',0)
auto = d.get('auto_resolve_rate',0)
print(f'命中率: {hit:.0%}, 蒸馏损失: {loss:.2f}, gap闭合: {gap}/天, auto_resolve: {auto}')
# 2026-10-17：recall_hit_rate 已退役（恒=1、无区分度）。召回质量改用 recall_usefulness_rate
#（返回且被使用）；-1 表示样本不足/无负反馈 = 数据不可解读，跳过告警，不误报。
hit = d.get('recall_usefulness_rate', -1)
loss = d.get('avg_distill_loss', 0)
gap = d.get('gap_closure_rate', 0)
auto = d.get('auto_resolve_rate', 0)
print(f'召回有用率: {hit:.0%}, 蒸馏损失: {loss:.2f}, gap闭合: {gap}/天, auto_resolve: {auto}')
if loss > 0.3: print('⚠️ avg_distill_loss 超标')
if hit < 0.5: print('⚠️ recall_hit_rate 低于阈值')
if hit >= 0 and hit < 0.5: print('⚠️ recall_usefulness_rate 低于阈值')
" 2>/dev/null
# 4. 蒸馏状态
skills/zhiyi/scripts/daily-check.sh
+8 -6
View File
@@ -26,13 +26,15 @@ METRICS=$(curl -s $BASE/api/v1/metrics/self -H "X-API-Key: $API_KEY" 2>/dev/null
echo "$METRICS" | python3 -c "
import sys,json
d=json.load(sys.stdin)
hit = d.get('recall_hit_rate',0)
loss = d.get('avg_distill_loss',0)
gap = d.get('gap_closure_rate',0)
auto = d.get('auto_resolve_rate',0)
print(f'命中率: {hit:.0%}, 蒸馏损失: {loss:.2f}, gap闭合: {gap}/天, auto_resolve: {auto}')
# 2026-10-17：recall_hit_rate 已退役（恒=1、无区分度）。召回质量改用 recall_usefulness_rate
#（返回且被使用）；-1 表示样本不足/无负反馈 = 数据不可解读，跳过告警，不误报。
hit = d.get('recall_usefulness_rate', -1)
loss = d.get('avg_distill_loss', 0)
gap = d.get('gap_closure_rate', 0)
auto = d.get('auto_resolve_rate', 0)
print(f'召回有用率: {hit:.0%}, 蒸馏损失: {loss:.2f}, gap闭合: {gap}/天, auto_resolve: {auto}')
if loss > 0.3: print('⚠️ avg_distill_loss 超标')
if hit < 0.5: print('⚠️ recall_hit_rate 低于阈值')
if hit >= 0 and hit < 0.5: print('⚠️ recall_usefulness_rate 低于阈值')
" 2>/dev/null
# 4. 蒸馏状态