From 44fd0e95b4de64c116fa1552df4f00075028d624 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=94=AF?= Date: Sat, 17 Oct 2026 21:22:28 +0800 Subject: [PATCH] =?UTF-8?q?fix(metrics):=20t=5Fc70f385c=20=E6=B6=88?= =?UTF-8?q?=E8=B4=B9=E8=80=85=20N/A=20=E6=8A=A4=E6=A0=8F=20+=20daily-check?= =?UTF-8?q?=20N/A=20=E6=98=BE=E7=A4=BA=20+=20=E5=8F=8D=E4=BE=8B=E6=B5=8B?= =?UTF-8?q?=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 小怡终审 2026-10-17 21:05 必须修的 4 项之一(-1 哨兵被当低分 ⇒ 触发器恒 fire): 1. selfoptimize.go: 新增 IsNA(v) 判 N/A(metricNA<0)。 2. executor.go: TGapDetected/TQualityDrop/TVValueLow 阈值比较前显式判 N/A, -1 不再被当「低于阈值」每轮触发(与旧 recall_hit_rate 恒=1 同类失真、方向相反)。 配 executor_na_test.go 反例(v=-1 断言不触发)+ 正向对照。 3. tuning.go: AnalyticsHandler 建议 builder 抽为纯函数 AnalyticsSuggestions, 统一走 ratioFor(-1/valid=0/样本不足 ⇒ 不产生建议);配 tuning_analytics_test.go 反例。 4. 两个 daily-check.sh: gap/auto/usefulness 值为 -1 时显示 —(N/A),不再打印误导性 -1/天。 测试命令(go.mod 在 go/ 下,须 cd go): cd go && go test ./internal/selfoptimize/... ./internal/api/routes/ (TestCommitConflictGate_E2E 为既有环境性 LLM 蒸馏用例,非本次改动路径,单独 FAIL) 生产部署仍待小怡终审 + 牧尘放行(本卡边界)。 --- go/internal/api/routes/tuning.go | 41 ++++++++------ .../api/routes/tuning_analytics_test.go | 38 +++++++++++++ go/internal/selfoptimize/executor.go | 14 +++-- go/internal/selfoptimize/executor_na_test.go | 53 +++++++++++++++++++ go/internal/selfoptimize/selfoptimize.go | 9 ++++ scripts/daily-check.sh | 17 +++--- skills/zhiyi/scripts/daily-check.sh | 17 +++--- 7 files changed, 158 insertions(+), 31 deletions(-) create mode 100644 go/internal/api/routes/tuning_analytics_test.go create mode 100644 go/internal/selfoptimize/executor_na_test.go diff --git a/go/internal/api/routes/tuning.go b/go/internal/api/routes/tuning.go index c87a680..5eb879e 100644 --- a/go/internal/api/routes/tuning.go +++ b/go/internal/api/routes/tuning.go @@ -356,25 +356,34 @@ func (tc *TuningConfig) RunHandler(w http.ResponseWriter, r *http.Request) { // POST /api/v1/tuning/analytics — 运行因果分析 func (tc *TuningConfig) AnalyticsHandler(w http.ResponseWriter, r *http.Request) { - // 基于指标间的关系给出建议 m := selfoptimize.Dash.Metrics() - - suggestions := []string{} - if m["recall_usefulness_rate"] < 0.5 && m["avg_distill_loss"] > 0.3 { - suggestions = append(suggestions, "蒸馏质量低导致召回率低 — 建议检查 Embedding 模型") - } - if m["deprecated_per_day"] > 10 { - suggestions = append(suggestions, "废弃率过高 — 正在流失大量记忆,检查冲突检测阈值") - } - if m["gap_closure_rate"] < 0.3 { - suggestions = append(suggestions, "缺口闭环率过低 — Agent 未回应缺口学习请求") - } - if m["auto_resolve_rate"] < 0.5 { - suggestions = append(suggestions, "自动裁决率低 — 规则覆盖不全,考虑扩展策略") - } - + suggestions := AnalyticsSuggestions(m) respond(w, 200, map[string]interface{}{ "metrics": m, "suggestions": suggestions, }) } + +// AnalyticsSuggestions 依据指标间关系给出调优建议(纯函数,便于单测反例:N/A 不产生建议)。 +// +// ⚠️ 每条建议的比率比较都必须先判 N/A(ratioFor 已封装:-1 哨兵 / valid=0 / 样本不足 ⇒ 不可用)。 +// 若直接 `m[key] < 阈值`,-1 会恒真 ⇒ 无数据也误报建议(本卡 t_c70f385c 事故面)。 +// deprecated_per_day 是计数(恒 ≥0,非比率),不走 ratioFor。 +func AnalyticsSuggestions(m map[string]float64) []string { + suggestions := []string{} + if hr, ok := ratioFor(m, "recall_usefulness_rate"); ok && hr < 0.5 { + if dl, ok2 := ratioFor(m, "avg_distill_loss"); ok2 && dl > 0.3 { + suggestions = append(suggestions, "蒸馏质量低导致召回率低 — 建议检查 Embedding 模型") + } + } + if m["deprecated_per_day"] > 10 { + suggestions = append(suggestions, "废弃率过高 — 正在流失大量记忆,检查冲突检测阈值") + } + if g, ok := ratioFor(m, "gap_closure_rate"); ok && g < 0.3 { + suggestions = append(suggestions, "缺口闭环率过低 — Agent 未回应缺口学习请求") + } + if a, ok := ratioFor(m, "auto_resolve_rate"); ok && a < 0.5 { + suggestions = append(suggestions, "自动裁决率低 — 规则覆盖不全,考虑扩展策略") + } + return suggestions +} diff --git a/go/internal/api/routes/tuning_analytics_test.go b/go/internal/api/routes/tuning_analytics_test.go new file mode 100644 index 0000000..c648155 --- /dev/null +++ b/go/internal/api/routes/tuning_analytics_test.go @@ -0,0 +1,38 @@ +// AnalyticsSuggestions N/A 哨兵护栏 —— 反例 + 正向对照。 +// +// 本卡 t_c70f385c:AnalyticsHandler 的建议 builder 原为裸 `m[key] < 阈值`, +// -1 哨兵(样本不足/越界被拦)会让 `-1 < 0.5` 恒真 ⇒ 无数据也误报建议。 +// 现统一走 ratioFor(已封装 N/A + valid=0 + 样本不足判定)。 +package routes + +import "testing" + +// TestAnalyticsSuggestions_NA_NoFiring 反例:全 N/A 哨兵(-1)不产生任何建议(避免误报)。 +func TestAnalyticsSuggestions_NA_NoFiring(t *testing.T) { + na := map[string]float64{ + "recall_usefulness_rate": -1, "recall_usefulness_samples": 0, "recall_usefulness_valid": 0, + "avg_distill_loss": -1, "avg_distill_loss_samples": 0, "avg_distill_loss_valid": 0, + "gap_closure_rate": -1, "gap_closure_samples": 0, "gap_closure_valid": 0, + "auto_resolve_rate": -1, "auto_resolve_samples": 0, "auto_resolve_valid": 0, + "deprecated_per_day": 0, + } + if s := AnalyticsSuggestions(na); len(s) != 0 { + t.Fatalf("全 N/A 不应产生任何建议,实际 %d 条: %v", len(s), s) + } +} + +// TestAnalyticsSuggestions_ValidLow 正向对照:有效且低于阈值时产生建议(护栏没把功能焊死)。 +func TestAnalyticsSuggestions_ValidLow(t *testing.T) { + m := map[string]float64{ + "recall_usefulness_rate": 0.2, "recall_usefulness_samples": 100, "recall_usefulness_valid": 1, + "gap_closure_rate": 0.1, "gap_closure_samples": 100, "gap_closure_valid": 1, + "auto_resolve_rate": 0.2, "auto_resolve_samples": 100, "auto_resolve_valid": 1, + "avg_distill_loss": 0.1, "avg_distill_loss_samples": 100, "avg_distill_loss_valid": 1, + "deprecated_per_day": 0, + } + s := AnalyticsSuggestions(m) + // 至少触发「gap 闭环率过低」+「自动裁决率低」两条 + if len(s) < 2 { + t.Fatalf("有效且低于阈值应产生 ≥2 条建议,实际 %d 条: %v", len(s), s) + } +} diff --git a/go/internal/selfoptimize/executor.go b/go/internal/selfoptimize/executor.go index a9a2628..5f95103 100644 --- a/go/internal/selfoptimize/executor.go +++ b/go/internal/selfoptimize/executor.go @@ -122,20 +122,28 @@ func (te *TriggerExecutor) evaluate(t *TriggerDef, metrics map[string]float64) b // 检查最近的 miss 计数。 // 2026-10-17:recall_hit_rate 已退役(恒=1、无区分度),改用 recall_usefulness_rate // (返回且被使用)作为「召回已产生可用结果、应清除 miss」的信号。 + // rate>0 天然排除 N/A 哨兵(-1>0 为 false)⇒ 无数据时不 fire。 rate, ok := metrics["recall_usefulness_rate"] return ok && rate > 0 && t.LastFired.IsZero() case TQualityDrop: threshold, _ := t.Condition.(float64) - return metrics["recall_usefulness_rate"] < threshold + // N/A 哨兵(-1,样本不足/越界被拦)不得当「低于阈值」触发 —— 先排除。 + v, ok := metrics["recall_usefulness_rate"] + return ok && !IsNA(v) && v < threshold case TGapDetected: - return metrics["gap_closure_rate"] < 0.5 + // ⚠️ 事故面:gap_closure_rate=-1(样本不足/越界)时 `-1 < 0.5` 恒真 ⇒ 每轮 fire。 + // 必须先判 N/A:无数据 ⇒ 不触发缺口学习。 + v, ok := metrics["gap_closure_rate"] + return ok && !IsNA(v) && v < 0.5 case TConflict: // 原判据 metrics["auto_resolve_rate"] < 0.5 —— 该值恒为 0,故恒 true,t6 空转。 // 新判据:人侧数据(画像/心迹/牵挂)有冲突时才 fire。 return hasProfileOrHeartConflict(metrics) case TVValueLow: threshold, _ := t.Condition.(float64) - return metrics["recall_usefulness_rate"] < threshold + // N/A 哨兵先排除(同 TQualityDrop)。 + v, ok := metrics["recall_usefulness_rate"] + return ok && !IsNA(v) && v < threshold case TDecayCritical: return metrics["deprecated_per_day"] > 5 } diff --git a/go/internal/selfoptimize/executor_na_test.go b/go/internal/selfoptimize/executor_na_test.go new file mode 100644 index 0000000..83a31c5 --- /dev/null +++ b/go/internal/selfoptimize/executor_na_test.go @@ -0,0 +1,53 @@ +// 触发器 N/A 哨兵护栏 —— 反例 + 正向对照。 +// +// 本卡 t_c70f385c(小怡终审 2026-10-17 21:05)事故面: +// 加了「samples<20 ⇒ -1」的最小样本门后,gap_closure_rate=-1 时 +// `executor.go:131 case TGapDetected: return metrics["gap_closure_rate"] < 0.5` 的 +// `-1 < 0.5` 恒真 ⇒ 触发器每轮都 fire。这与旧 recall_hit_rate 恒=1 是**同一类 +// 判定失真(方向相反)**。要求:所有消费点先显式判 N/A,且每个消费点配一条反例测试。 +package selfoptimize + +import "testing" + +// TestTriggerEvaluate_NA_Sentinel 反例:N/A 哨兵(-1)不得触发任何比率类触发器。 +func TestTriggerEvaluate_NA_Sentinel(t *testing.T) { + te := &TriggerExecutor{} + na := map[string]float64{ + "recall_usefulness_rate": -1, + "gap_closure_rate": -1, + "auto_resolve_rate": -1, + "avg_distill_loss": -1, + } + if te.evaluate(&TriggerDef{Type: TGapDetected, Condition: true}, na) { + t.Fatal("gap_closure_rate=-1(N/A)不应触发 t5,实际触发了") + } + if te.evaluate(&TriggerDef{Type: TQualityDrop, Condition: 0.3}, na) { + t.Fatal("recall_usefulness_rate=-1(N/A)不应触发 t4,实际触发了") + } + if te.evaluate(&TriggerDef{Type: TVValueLow, Condition: 0.2}, na) { + t.Fatal("recall_usefulness_rate=-1(N/A)不应触发 t7,实际触发了") + } + if te.evaluate(&TriggerDef{Type: TRecallMiss, Condition: 3}, na) { + t.Fatal("recall_usefulness_rate=-1(N/A)不应触发 t3(rate>0 不成立),实际触发了") + } +} + +// TestTriggerEvaluate_ValidLow 正向对照:有效且低于阈值时仍要触发(护栏没把功能焊死)。 +func TestTriggerEvaluate_ValidLow(t *testing.T) { + te := &TriggerExecutor{} + valid := map[string]float64{ + "recall_usefulness_rate": 0.2, + "gap_closure_rate": 0.2, + } + if !te.evaluate(&TriggerDef{Type: TGapDetected, Condition: true}, valid) { + t.Fatal("gap_closure_rate=0.2 (<0.5, 有效) 应触发 t5") + } + if !te.evaluate(&TriggerDef{Type: TQualityDrop, Condition: 0.3}, valid) { + t.Fatal("recall_usefulness_rate=0.2 (<0.3, 有效) 应触发 t4") + } + // 有效且高于阈值 ⇒ 不触发 + hi := map[string]float64{"gap_closure_rate": 0.9, "recall_usefulness_rate": 0.9} + if te.evaluate(&TriggerDef{Type: TGapDetected, Condition: true}, hi) { + t.Fatal("gap_closure_rate=0.9 (>0.5, 有效) 不应触发 t5") + } +} diff --git a/go/internal/selfoptimize/selfoptimize.go b/go/internal/selfoptimize/selfoptimize.go index 85b9435..a42b291 100644 --- a/go/internal/selfoptimize/selfoptimize.go +++ b/go/internal/selfoptimize/selfoptimize.go @@ -260,6 +260,15 @@ func rateOrNA(num, den int) float64 { return math.Round(float64(num)/float64(den)*100) / 100 } +// IsNA 判定指标值是否为「无数据」哨兵(metricNA < 0)。 +// +// ⚠️ 消费者铁律:触发器 / 调优建议 / 面板解读等所有消费点在拿比率值跟阈值 +// 比较之前,必须先排除 N/A。-1 是哨兵(样本不足 或 越界被拦),不是「低分」, +// 若直接 `v < 阈值` 会恒为 true ⇒ 每轮误触发(本卡 t_c70f385c 事故面: +// gap_closure_rate=-1 让「降低 gap 阈值」触发器每轮 fire,与旧 recall_hit_rate 恒=1 +// 是同类判定失真、方向相反)。故:v < 0 ⇒ 跳过 / 不触发。 +func IsNA(v float64) bool { return v < 0 } + // minRateSamples 是率类指标的最小样本门。 // // 样本太少(< 20)时,即使比率为真值也**统计上不可解读** —— 例如 auto_resolve_rate diff --git a/scripts/daily-check.sh b/scripts/daily-check.sh index 1515046..d7a6552 100755 --- a/scripts/daily-check.sh +++ b/scripts/daily-check.sh @@ -29,12 +29,17 @@ d=json.load(sys.stdin) # 2026-10-17:recall_hit_rate 已退役(恒=1、无区分度)。召回质量改用 recall_usefulness_rate #(返回且被使用);-1 表示样本不足/无负反馈 = 数据不可解读,跳过告警,不误报。 hit = d.get('recall_usefulness_rate', -1) -loss = d.get('avg_distill_loss', 0) -gap = d.get('gap_closure_rate', 0) -auto = d.get('auto_resolve_rate', 0) -print(f'召回有用率: {hit:.0%}, 蒸馏损失: {loss:.2f}, gap闭合: {gap}/天, auto_resolve: {auto}') -if loss > 0.3: print('⚠️ avg_distill_loss 超标') -if hit >= 0 and hit < 0.5: print('⚠️ recall_usefulness_rate 低于阈值') +loss = d.get('avg_distill_loss', -1) +gap = d.get('gap_closure_rate', -1) +auto = d.get('auto_resolve_rate', -1) +# -1 哨兵 = 样本不足/越界被拦,不可解读 ⇒ 显示 —(N/A),不得当成具体数字误导。 +def fmt(v, pct=False): + if v is None or v < 0: + return '—(N/A)' + return ('%.0f%%' % (v*100)) if pct else ('%.2f' % v) +print('召回有用率: %s, 蒸馏损失: %s, gap闭合率: %s, auto_resolve率: %s' % (fmt(hit, True), fmt(loss), fmt(gap, True), fmt(auto, True))) +if loss is not None and loss >= 0 and loss > 0.3: print('⚠️ avg_distill_loss 超标') +if hit is not None and hit >= 0 and hit < 0.5: print('⚠️ recall_usefulness_rate 低于阈值') " 2>/dev/null # 4. 蒸馏状态 diff --git a/skills/zhiyi/scripts/daily-check.sh b/skills/zhiyi/scripts/daily-check.sh index 1515046..d7a6552 100755 --- a/skills/zhiyi/scripts/daily-check.sh +++ b/skills/zhiyi/scripts/daily-check.sh @@ -29,12 +29,17 @@ d=json.load(sys.stdin) # 2026-10-17:recall_hit_rate 已退役(恒=1、无区分度)。召回质量改用 recall_usefulness_rate #(返回且被使用);-1 表示样本不足/无负反馈 = 数据不可解读,跳过告警,不误报。 hit = d.get('recall_usefulness_rate', -1) -loss = d.get('avg_distill_loss', 0) -gap = d.get('gap_closure_rate', 0) -auto = d.get('auto_resolve_rate', 0) -print(f'召回有用率: {hit:.0%}, 蒸馏损失: {loss:.2f}, gap闭合: {gap}/天, auto_resolve: {auto}') -if loss > 0.3: print('⚠️ avg_distill_loss 超标') -if hit >= 0 and hit < 0.5: print('⚠️ recall_usefulness_rate 低于阈值') +loss = d.get('avg_distill_loss', -1) +gap = d.get('gap_closure_rate', -1) +auto = d.get('auto_resolve_rate', -1) +# -1 哨兵 = 样本不足/越界被拦,不可解读 ⇒ 显示 —(N/A),不得当成具体数字误导。 +def fmt(v, pct=False): + if v is None or v < 0: + return '—(N/A)' + return ('%.0f%%' % (v*100)) if pct else ('%.2f' % v) +print('召回有用率: %s, 蒸馏损失: %s, gap闭合率: %s, auto_resolve率: %s' % (fmt(hit, True), fmt(loss), fmt(gap, True), fmt(auto, True))) +if loss is not None and loss >= 0 and loss > 0.3: print('⚠️ avg_distill_loss 超标') +if hit is not None and hit >= 0 and hit < 0.5: print('⚠️ recall_usefulness_rate 低于阈值') " 2>/dev/null # 4. 蒸馏状态