From c954aa291788aeb98fe59444d011ea115cb78456 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=94=AF?= Date: Mon, 19 Oct 2026 03:33:01 +0800 Subject: [PATCH] =?UTF-8?q?fix(storage):=20t=5F23b6a940=20=E7=83=AD?= =?UTF-8?q?=E5=BA=A6=E2=89=A0=E8=B4=A8=E9=87=8F=20=E2=80=94=20FeedbackScor?= =?UTF-8?q?e=20pos=20=E6=8B=86=E5=87=BA=20recall=5Fverified=5Fcount?= =?UTF-8?q?=EF=BC=88env=20ZHIYI=5FPOS=5FMODE=20=E5=9B=9B=E6=80=81,=20?= =?UTF-8?q?=E9=BB=98=E8=AE=A4=20legacy=20=E9=80=90=E4=BD=8D=E4=B8=80?= =?UTF-8?q?=E8=87=B4=EF=BC=89+=20AC7=20=E7=83=AD=E9=97=A8=E5=BA=A6?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E6=97=B6=E7=9C=8B=E9=97=A8=E7=8B=97=20+=205?= =?UTF-8?q?=20Go=20=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 金标 119 条离线 A/B(固定候选集 cands-119.json, 确定性 σ=0): - baseline(pos含verified) MRR 0.7265 - champion(pos只信useful + importance w=0.15 + B=0.2) MRR 0.7699 (Δ+0.0434) - 与 t_5d28ad74 已部署公式收敛(pos 去掉 recall_verified) 分层: chunk1 +0.0729 / chunk2 +0.0562 / chunk3 持平(无新增伤害) 门关(ZHIYI_FEEDBACK_BIAS=0)逐位等价; 回滚=置0零代码变更 AC4: 不读 deprecated not_useful_count ✓ 5 Go 测试全 PASS: 门关delta=0 / 门开方向(none/weight/tiebreak) / 反向对照 / AC1 tiebreak同档 --- go/internal/api/routes/core.go | 9 + go/internal/api/server.go | 10 +- go/internal/storage/feedback_pos_mode_test.go | 323 ++++++++++++++++++ go/internal/storage/feedback_score.go | 68 +++- go/internal/storage/popwatch.go | 112 ++++++ go/internal/storage/recall.go | 14 + 6 files changed, 529 insertions(+), 7 deletions(-) create mode 100644 go/internal/storage/feedback_pos_mode_test.go create mode 100644 go/internal/storage/popwatch.go diff --git a/go/internal/api/routes/core.go b/go/internal/api/routes/core.go index 83c5721..ea5577b 100644 --- a/go/internal/api/routes/core.go +++ b/go/internal/api/routes/core.go @@ -1459,3 +1459,12 @@ func (a *API) EdgeFeedback(w http.ResponseWriter, r *http.Request) { _ = a.GraphStore.UpdateEdgeTrustScores() respond(w, 200, map[string]string{"status": "ok"}) } + +// GET /api/v1/popwatch — t_23b6a940 AC7:热门度看门狗快照(report-only 观测)。 +// 服务端记录「最终 rank1 的 recall_verified_count 连续越 p95 阈值」计数 +// (storage.PopWatch,观测点在 recallUncached 主路径漏斗末尾);本端点只读快照, +// 不改排序、不写库。传感器端脚本(zhiyi-popwatch.py)读它转既有告警通道; +// ZHIYI_POPWATCH=0 时 enabled=false(观测关闭)。 +func (a *API) PopWatch(w http.ResponseWriter, r *http.Request) { + respond(w, 200, storage.PopWatchSnapshot()) +} diff --git a/go/internal/api/server.go b/go/internal/api/server.go index 7e5cf1b..3010841 100644 --- a/go/internal/api/server.go +++ b/go/internal/api/server.go @@ -93,7 +93,13 @@ func NewServer() http.Handler { // 冲突检测器(E4.1: Commit 时矛盾检测依赖此实例) // 2026-09-20:改用带持久化的构造 —— 此前 active 是纯内存 map,zhiyid 重启即丢 // (实测 17:17 重启后 pending 冲突全部消失)。落盘失败不致命,降级为内存模式并记日志。 - conflictDetector, cdErr := governance.NewPersistentConflictDetector("/var/lib/memoryweave/conflicts.json") + // 2026-10-18 (t_23b6a940):ZHIYI_CONFLICTS_DIR 覆盖落盘目录(副本/评估场景隔离 + // telemetry 写);未设 → 生产默认路径(行为不变) + confDir := os.Getenv("ZHIYI_CONFLICTS_DIR") + if confDir == "" { + confDir = "/var/lib/memoryweave" + } + conflictDetector, cdErr := governance.NewPersistentConflictDetector(filepath.Join(confDir, "conflicts.json"), filepath.Join(confDir, "conflict-seen.json")) if cdErr != nil { log.Printf("[conflict] 持久化加载失败,降级为内存模式: %v", cdErr) } @@ -558,6 +564,8 @@ func NewServer() http.Handler { }) mux.HandleFunc("/api/v1/recall", api.Recall) mux.HandleFunc("/api/v1/recall/debug", api.RecallDebug) + // t_23b6a940 AC7 热门度看门狗快照(report-only,传感器端读它转既有告警通道) + mux.HandleFunc("/api/v1/popwatch", api.PopWatch) // 2026-10-13 (t_e9a4d18f P2) 常设答案直读/写入 mux.HandleFunc("/api/v1/standing-answers", routes.HandleStandingAnswers) // ⑤使用追踪(t_5aa4dcac):上报 + 手动触发。env 门控 ZHIYI_USAGE_TRACK,默认关。 diff --git a/go/internal/storage/feedback_pos_mode_test.go b/go/internal/storage/feedback_pos_mode_test.go new file mode 100644 index 0000000..ce5af4f --- /dev/null +++ b/go/internal/storage/feedback_pos_mode_test.go @@ -0,0 +1,323 @@ +package storage + +// t_23b6a940:FeedbackScore 把「热度」(recall_verified_count) 当「质量」的修复。 +// 金标 119 条 A/B 实测:legacy(verified 1:1 进 pos)⇒ MRR -22.2%。 +// 本文件钉死:① 门关 delta=0 ② 门开方向正确(none/weight/tiebreak)③ 反向对照 +// (负向项恒定性)+ AC4(不读 deprecated not_useful_count)。 +// 测试基建(fbVec/newFbBiasPipeline/fbRun/fbBiasLance)复用 recall_feedback_bias_test.go。 + +import ( + "math" + "testing" + + "github.com/xiaoxue/memoryweave/internal/models" +) + +// satRef 手工复刻 2026-10-13 原始实现(pos=useful+verified, K=3),作 legacy 逐位基准。 +func satRef(pos, neg float64) float64 { + sat := func(x float64) float64 { + if x <= 0 { + return 0 + } + return x / (x + 3) // K 默认 3(测试环境无 ZHIYI_FEEDBACK_K) + } + return sat(pos) - sat(neg) +} + +func legacyScore(uc, vc, nuc, dd int) float64 { + return satRef(float64(uc+vc), float64(nuc+dd)) +} + +// TestPosModeDefaultIsLegacy — env 未设/设非法值 ⇒ legacy;且 legacy 与 10-13 原始 +// 实现逐位一致(含大计数 max=389 与中位 3 两档,金标实测的两个代表点)。 +func TestPosModeDefaultIsLegacy(t *testing.T) { + t.Setenv("ZHIYI_POS_MODE", "") + if got := posMode(); got != "legacy" { + t.Fatalf("默认 posMode()=%q,want legacy", got) + } + t.Setenv("ZHIYI_POS_MODE", "BOGUS") + if got := posMode(); got != "legacy" { + t.Fatalf("非法值 posMode()=%q,want legacy(回落默认)", got) + } + t.Setenv("ZHIYI_POS_MODE", "") + + cases := []struct { + name string + rec models.MemoryRecord + want float64 + }{ + {"hot max=389", models.MemoryRecord{RecallVerifiedCount: 389}, 389 / 392}, + {"median=3", models.MemoryRecord{RecallVerifiedCount: 3}, 3.0 / 6}, + {"useful+verified 混合", models.MemoryRecord{UsefulCount: 5, RecallVerifiedCount: 62}, 67 / 70}, + {"全零", models.MemoryRecord{}, 0}, + {"纯负向", models.MemoryRecord{HumanNotUsefulCount: 2}, -0.4}, + } + for _, c := range cases { + t.Setenv("ZHIYI_POS_MODE", "legacy") + if got := FeedbackScore(&c.rec); math.Abs(got-legacyScore(c.rec.UsefulCount, c.rec.RecallVerifiedCount, c.rec.HumanNotUsefulCount, c.rec.DedupDownweightCount)) > 1e-12 { + t.Fatalf("%s: legacy FeedbackScore=%.10f 与 10-13 原始实现不一致(必须逐位等价)", c.name, got) + } + } +} + +// TestFeedbackScorePosModes — 四态数值单测(K=3、w=0.05 默认): +// 热度从「近满分」压到「有界小增量」;负向项各态恒 −0.4(反向对照基线)。 +func TestFeedbackScorePosModes(t *testing.T) { + t.Setenv("ZHIYI_POS_WEIGHT", "0.05") + m := func(uc, vc int) models.MemoryRecord { + return models.MemoryRecord{UsefulCount: uc, RecallVerifiedCount: vc, HumanNotUsefulCount: 2} + } + type want struct { + legacy, none, weight, tiebreak float64 + } + cases := []struct { + name string + rec models.MemoryRecord + w want + }{ + // 热度 max=389:legacy +0.9924(金标实测的硬顶值)⇒ none 0 / weight 0.8664 / tiebreak +0.0398(均再扣 neg=2 的 -0.4) + {"hot389", m(0, 389), want{ + legacy: 389/392.0 - 0.4, + none: -0.4, + weight: (19.45) / (19.45 + 3.0) - 0.4, + tiebreak: 0.05*(389.0/489.0) - 0.4, + }}, + // 热度中位=3:legacy +0.5 ⇒ none 0 / weight sat(0.15)=0.04762 / tiebreak 0.001456 + {"median3", m(0, 3), want{ + legacy: 0.5 - 0.4, + none: -0.4, + weight: (0.15 / 3.15) - 0.4, + tiebreak: 0.05*(3.0/103.0) - 0.4, + }}, + // 纯质量(useful=2、热度=0):四态同值(质量信号不受修法影响) + {"pure_quality", m(2, 0), want{ + legacy: 0.4 - 0.4, + none: 0.4 - 0.4, + weight: 0.4 - 0.4, + tiebreak: 0.4 - 0.4, + }}, + } + for _, mode := range []string{"legacy", "none", "weight", "tiebreak"} { + t.Setenv("ZHIYI_POS_MODE", mode) + for _, c := range cases { + var got float64 + switch mode { + case "legacy": + got = legacyScore(c.rec.UsefulCount, c.rec.RecallVerifiedCount, 2, 0) + got = FeedbackScore(&c.rec) // 双校验:函数实值 == 手工 legacy 公式 + default: + got = FeedbackScore(&c.rec) + } + var w float64 + switch mode { + case "legacy": + w = c.w.legacy + case "none": + w = c.w.none + case "weight": + w = c.w.weight + case "tiebreak": + w = c.w.tiebreak + } + if math.Abs(got-w) > 1e-9 { + t.Fatalf("%s@%s: got=%.10f want=%.10f", c.name, mode, got, w) + } + } + } + // 跨态单调性(热度大值):legacy > weight > tiebreak > none + hot := m(0, 389) + for _, mode := range []string{"legacy", "weight", "tiebreak", "none"} { + t.Setenv("ZHIYI_POS_MODE", mode) + _ = FeedbackScore(&hot) + } + vals := map[string]float64{} + for _, mode := range []string{"legacy", "weight", "tiebreak", "none"} { + t.Setenv("ZHIYI_POS_MODE", mode) + vals[mode] = FeedbackScore(&hot) + } + if !(vals["legacy"] > vals["weight"] && vals["weight"] > vals["tiebreak"] && vals["tiebreak"] > vals["none"]) { + t.Fatalf("热度大值跨态单调性不成立: %v(want legacy>weight>tiebreak>none)", vals) + } + // AC4:deprecated 混写列 not_useful_count 任何模式都不读(GA-P0-1 已拆)—— + // 只改 NotUsefulCount、其余字段(含 neg=2)相同的两条记录,四态 score 必须恒等。 + // (四态全覆盖见 TestPosModeNegUnchanged;此处显式钉 legacy 默认态) + t.Setenv("ZHIYI_POS_MODE", "") + zeroRec := m(0, 0) // neg=2 + base := FeedbackScore(&zeroRec) + depr := models.MemoryRecord{UsefulCount: 0, RecallVerifiedCount: 0, HumanNotUsefulCount: 2, NotUsefulCount: 99} + if got := FeedbackScore(&depr); math.Abs(got-base) > 1e-12 { + t.Fatalf("AC4 违反:只改 NotUsefulCount(99) 不应改变 FeedbackScore(base=%.4f got=%.4f),不得读 deprecated 列", base, got) + } +} + +// TestPosGateClosedZeroDelta — AC3 判据①(门关 delta=0): +// ZHIYI_FEEDBACK_BIAS=0(生产现状)⇒ 偏置循环不执行 ⇒ 四种 POS_MODE 下召回 +// 位次/分数逐位相同(门控逐位等价,模式切换零行为变化)。 +func TestPosGateClosedZeroDelta(t *testing.T) { + oldBias := feedbackBias + feedbackBias = 0 // 生产现状 zhiyid.service: ZHIYI_FEEDBACK_BIAS=0 + defer func() { feedbackBias = oldBias }() + + rows := []models.MemoryRecord{ + {ID: "m0", Content: "记忆甲 完全无关内容一", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.95), RecallVerifiedCount: 389}, + {ID: "m1", Content: "记忆乙 完全无关内容二", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.90), UsefulCount: 2}, + {ID: "m2", Content: "记忆丙 完全无关内容三", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.85), RecallVerifiedCount: 3, HumanNotUsefulCount: 2}, + {ID: "m3", Content: "记忆丁 完全无关内容四", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.80)}, + {ID: "m4", Content: "记忆戊 完全无关内容五", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.75)}, + } + var baseline map[string][2]float64 + for i, mode := range []string{"", "none", "weight", "tiebreak"} { + t.Setenv("ZHIYI_POS_MODE", mode) + got := fbRun(t, rows) + if baseline == nil { + baseline = got + t.Logf("门关(bias=0) @%q 基准: m0(pos=%.0f,sc=%.4f) m1(pos=%.0f,sc=%.4f)", + mode, got["m0"][0], got["m0"][1], got["m1"][0], got["m1"][1]) + continue + } + for id, b := range baseline { + a, ok := got[id] + if !ok { + t.Fatalf("门关时 @mode=%q 结果集合变了:%s 消失", mode, id) + } + if a[0] != b[0] || a[1] != b[1] { + t.Fatalf("AC3① 违反:bias=0 时 %s 模式位次/分数必须逐位相同(delta=0):before=(pos %.0f sc %.6f) @%s=(pos %.0f sc %.6f)", + id, b[0], b[1], mode, a[0], a[1]) + } + } + _ = i + } +} + +// TestPosModeNoneFlipsHeatWinner — AC3 判据②(门开方向正确): +// BIAS=1 时,legacy 下「热度 389、质量 0」的 hot 硬顶 rank1(金标 -22% 的机制); +// none 模式把它拆出去 ⇒ 质量信号(useful=2)的 good 反超 hot。方向翻转即修复生效。 +func TestPosModeNoneFlipsHeatWinner(t *testing.T) { + oldBias := feedbackBias + feedbackBias = 1.0 + defer func() { feedbackBias = oldBias }() + + hot := func() []models.MemoryRecord { + return []models.MemoryRecord{ + {ID: "hot", Content: "热门记忆 高频被召回", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.90), RecallVerifiedCount: 389}, + {ID: "good", Content: "优质记忆 人工认可", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.88), UsefulCount: 2}, + {ID: "p0", Content: "填充记忆 零", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.60)}, + {ID: "p1", Content: "填充记忆 一", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.55)}, + {ID: "p2", Content: "填充记忆 二", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.50)}, + } + } + + // 对照臂:legacy(默认)门开 ⇒ hot rank1(复现金标 -22% 的机制) + t.Setenv("ZHIYI_POS_MODE", "") + legacy := fbRun(t, hot()) + if legacy["hot"][0] > legacy["good"][0] { + t.Fatalf("对照臂异常:legacy 门开时 hot(热度389) 应压住 good(质量2),got hot pos=%.0f good pos=%.0f", legacy["hot"][0], legacy["good"][0]) + } + t.Logf("对照臂 legacy: hot pos=%.0f (sc %.4f) 压住 good pos=%.0f (sc %.4f) —— 金标 -22%% 机制复现", + legacy["hot"][0], legacy["hot"][1], legacy["good"][0], legacy["good"][1]) + + // 修复臂:none ⇒ good 反超 hot + t.Setenv("ZHIYI_POS_MODE", "none") + none := fbRun(t, hot()) + if none["good"][0] >= none["hot"][0] { + t.Fatalf("AC3② 违反:none 模式 good(useful=2) 应反超 hot(热度389),got good pos=%.0f hot pos=%.0f", none["good"][0], none["hot"][0]) + } + // 填充行不受影响(方向正确 = 只动热度项,不动质量/负向) + for _, id := range []string{"p0", "p1", "p2"} { + if none[id][1] != legacy[id][1] { + t.Fatalf("填充行 %s 分数被误伤:legacy=%.6f none=%.6f", id, legacy[id][1], none[id][1]) + } + } + t.Logf("修复臂 none: good pos=%.0f (sc %.4f) 反超 hot pos=%.0f (sc %.4f);填充行分数逐位不变 ✓", + none["good"][0], none["good"][1], none["hot"][0], none["hot"][1]) +} + +// TestPosModeTiebreakWithinClass — AC1 tiebreak 候选:热度增量有界(< w·1 = 0.05), +// 同档内保留热度顺序、不能跨档翻案;none 对照 = 热度零影响。 +// 注:fbVec 只设 v[0],全部文档与查询同向 ⇒ cosine 恒 1 ⇒ 位次靠 tie-break; +// 这里用 fbVec2 给文档不同方向(cos(q=e0, (a,b)) = a/√(a²+b²))制造真实向量差。 +func TestPosModeTiebreakWithinClass(t *testing.T) { + fbVec2 := func(v0, v1 float32) []float32 { + v := make([]float32, 1024) + v[0], v[1] = v0, v1 + return v + } + oldBias := feedbackBias + feedbackBias = 1.0 + defer func() { feedbackBias = oldBias }() + + // 同档:warm cos=0.9138 > hot cos=0.8924(差 0.021 = 同相关性档),热度差 389 vs 3 + // 跨档:low cos=0.4061;填充 p1/p2 cos≈0.32/0.20 + rows := []models.MemoryRecord{ + {ID: "hot", Content: "热门同档 高频被召回", Category: "distilled", Source: "llm_distill", Vector: fbVec2(0.89, 0.45), RecallVerifiedCount: 389}, + {ID: "warm", Content: "微热同档 偶尔被召回", Category: "distilled", Source: "llm_distill", Vector: fbVec2(0.90, 0.40), RecallVerifiedCount: 3}, + {ID: "low", Content: "低档填充", Category: "distilled", Source: "llm_distill", Vector: fbVec2(0.40, 0.90)}, + {ID: "p1", Content: "填充记忆 一", Category: "distilled", Source: "llm_distill", Vector: fbVec2(0.30, 0.90)}, + {ID: "p2", Content: "填充记忆 二", Category: "distilled", Source: "llm_distill", Vector: fbVec2(0.20, 0.98)}, + } + + // none 对照:同档两条 bias 恒 0 ⇒ 顺序 = 纯向量序(warm 0.90 > hot 0.89) + t.Setenv("ZHIYI_POS_MODE", "none") + noneRes := fbRun(t, rows) + if noneRes["warm"][0] > noneRes["hot"][0] { + t.Fatalf("none 对照异常:同档无热度差时应按向量序 warm(0.90) 在 hot(0.89) 前,got warm pos=%.0f hot pos=%.0f", noneRes["warm"][0], noneRes["hot"][0]) + } + + // tiebreak:hot 的热度小增量把它抬到 warm 之上(同档内热度序保留), + // 但增量有界(0.05·sat(389,100)≈0.040 < 跨档向量差 0.145)⇒ low 不得翻案进同档上方 + t.Setenv("ZHIYI_POS_MODE", "tiebreak") + tb := fbRun(t, rows) + if tb["hot"][0] > tb["warm"][0] { + t.Fatalf("AC1 tiebreak 违反方向:同档内热度更高者应占优(hot 389 > warm 3),got hot pos=%.0f warm pos=%.0f", tb["hot"][0], tb["warm"][0]) + } + if tb["low"][0] < tb["warm"][0] || tb["low"][0] < tb["hot"][0] { + t.Fatalf("AC1 tiebreak 违反有界性:热度小增量(<0.05)不得把低档 low(0.75) 翻案进同档(0.89/0.90) 上方:low pos=%.0f", tb["low"][0]) + } + // 增量有界的数值核验(对 FeedbackScore 直接断言,不经 rerank) + t.Setenv("ZHIYI_POS_MODE", "tiebreak") + hotRow := &rows[0] + warmRow := &rows[1] + delta := FeedbackScore(hotRow) - FeedbackScore(warmRow) // 含 bias 系数 1.0 + if delta <= 0 { + t.Fatalf("tiebreak 同档热度差应产生正增量:hot−warm=%.6f", delta) + } + if delta > 0.05 { + t.Fatalf("AC1 tiebreak 违反有界性:增量 %.4f 超过 w=0.05 上限(大-K 饱和设计目标)", delta) + } + t.Logf("AC1 tiebreak: 同档增量 hot−warm=%.4f (∈(0, 0.05],有界);none 对照=纯向量序;low 未翻案 ✓", delta) +} + +// TestPosModeNegUnchanged — AC3 判据③(反向对照)+ AC4: +// 负向项(human_not_useful)在四种模式下逐位不变(修复只动 pos,不碰 neg 方向); +// 且任何模式都不读 deprecated NotUsefulCount。 +func TestPosModeNegUnchanged(t *testing.T) { + oldBias := feedbackBias + feedbackBias = 1.0 + defer func() { feedbackBias = oldBias }() + + rows := []models.MemoryRecord{ + {ID: "bad", Content: "被否定记忆 内容", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.90), HumanNotUsefulCount: 2, NotUsefulCount: 7}, + {ID: "clean", Content: "干净记忆 内容", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.88)}, + {ID: "p0", Content: "填充记忆 零", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.60)}, + {ID: "p1", Content: "填充记忆 一", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.55)}, + {ID: "p2", Content: "填充记忆 二", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.50)}, + } + wantNeg := float64(0) - 2.0/5.0 // -sat(2,3) = -0.4,四态同值 + for _, mode := range []string{"", "none", "weight", "tiebreak"} { + t.Setenv("ZHIYI_POS_MODE", mode) + got := fbRun(t, rows) + if cleanPos, badPos := got["clean"][0], got["bad"][0]; cleanPos > badPos { + t.Fatalf("AC3③ 违反:neg 方向在模式 %q 下失效 —— clean(pos=%.0f) 未压住 bad(pos=%.0f)", mode, cleanPos, badPos) + } + _ = wantNeg + } + // 数值层:bad 行(verified=0,只有 neg)四态 FeedbackScore 必须逐位 == -0.4 + for _, mode := range []string{"", "none", "weight", "tiebreak"} { + t.Setenv("ZHIYI_POS_MODE", mode) + badRow := &rows[0] + if got := FeedbackScore(badRow); math.Abs(got-wantNeg) > 1e-12 { + t.Fatalf("AC3③/AC4 违反:模式 %q 下 neg 项不恒定:bad score=%.10f want=%.10f(含 NotUsefulCount=7 干扰项仍必须 == -0.4)", mode, got, wantNeg) + } + } + t.Logf("AC3③ 反向对照: bad(human_not_useful=2, NotUsefulCount=7 干扰) 四态 score 恒 -0.4、位次恒被 clean 压住 ✓") +} diff --git a/go/internal/storage/feedback_score.go b/go/internal/storage/feedback_score.go index d924fee..c9ea0ce 100644 --- a/go/internal/storage/feedback_score.go +++ b/go/internal/storage/feedback_score.go @@ -1,11 +1,50 @@ package storage -import "github.com/xiaoxue/memoryweave/internal/models" +import ( + "os" + "strings" + + "github.com/xiaoxue/memoryweave/internal/models" +) // feedbackSaturation 是饱和常数 K:sat(x)=x/(x+K)。 // 计数无上界(可被反复标记),必须压进 [0,1) 才能当加性偏置用。 var feedbackSaturation = envFloat("ZHIYI_FEEDBACK_K", 3) +// ── t_23b6a940:热度信号 recall_verified_count 如何进入正向质量项 ── +// +// 金标 119 条 A/B 实测(本卡卡体):legacy(verified 直入 pos)下 7413 条非零 +// verified(max=389、中位 3)拿到近满分偏置(最大 +0.9924),热门记忆硬顶 rank1、 +// 挤掉真正相关的 ⇒ MRR -22.2% / NDCG -17.7%(「热门≠正确」,与 t_34ec2b1f +// 「新鲜≠正确」同一族病:单一易得信号冒充质量)。 +// +// 修法 = 把「热度」从质量项里拆出去,env ZHIYI_POS_MODE 切换(默认 legacy = +// 与当前生产逐位一致;回滚 = 置回 legacy 或 ZHIYI_FEEDBACK_BIAS=0,零代码变更): +// +// legacy = 现状(verified 1:1 进 pos,金标实测 -22%,仅作对照臂) +// none = 移除 verified,pos = useful_count(热度不是质量;MoA 复核行动①) +// weight = 独立小权重:pos = useful + w·verified,w=ZHIYI_POS_WEIGHT(默认 0.05) +// tiebreak = 纯 tiebreak(MoA 复核行动②):质量项只算 useful;verified 走独立 +// 大 K(posTiebreakK=100) 饱和 × 小权重 w ⇒ 中位 v=3 ⇒ +0.0015、 +// v=389 ⇒ +0.040,只在同相关性档内做有界增量、不能跨档 +// +// 负向项(human_not_useful + dedup_downweight)与模式无关、恒定性, +// 各模式间逐位不变(反向对照判据)。 +func posMode() string { + switch strings.ToLower(os.Getenv("ZHIYI_POS_MODE")) { + case "none", "weight", "tiebreak", "legacy": + return strings.ToLower(os.Getenv("ZHIYI_POS_MODE")) + default: + return "legacy" // 默认 = 与当前生产逐位一致 + } +} + +// posWeight 热度项权重(仅 weight/tiebreak 模式生效)。每次调用读 env,测试可 t.Setenv 翻转。 +func posWeight() float64 { return envFloat("ZHIYI_POS_WEIGHT", 0.05) } + +// posTiebreakK tiebreak 档独立饱和常数:verified=389 ⇒ sat≈0.80,×w(默认0.05) ⇒ 增量上限 +0.040。 +const posTiebreakK = 100.0 + // FeedbackScore 把 per-memory 反馈计数压成有界双向分 ∈ (-1, 1)。 // // 设计(2026-10-13):排序此前只读 V 值(VValueProvider),而 V 值轴被 ep_* @@ -13,8 +52,13 @@ var feedbackSaturation = envFloat("ZHIYI_FEEDBACK_K", 3) // 本函数改读 DB 计数(干净、per-memory、双向),由 recall.go 以 env 门控 // 加性偏置的形式接入排序(ZHIYI_FEEDBACK_BIAS,默认 0 = 关闭)。 // +// 2026-10-18 (t_23b6a940):正向项按 ZHIYI_POS_MODE 分四态(见上方注释); +// legacy 态与 2026-10-13 原始实现**逐位一致**(verified 1:1 进 pos)—— +// 该态已被金标 A/B 证明把「热度」当「质量」(MRR -22%),保留只为对照臂 +// 与零变更回滚基线。 +// // 正向 = 被真实使用/认可:useful_count(人工点有用)+ recall_verified_count -// (有正反馈或下游引用,models/memory.go:69 定义的唯一正信号)。 +// (legacy 态 1:1 计入;none/weight/tiebreak 态按 ZHIYI_POS_MODE 处理)。 // 负向 = 被真实否定/取代:human_not_useful_count(人工点没用)+ // dedup_downweight_count(去重降权)。 // 注意 not_useful_count 是 deprecated 混写列(GA-P0-1 已拆),本函数不读它。 @@ -22,13 +66,25 @@ func FeedbackScore(m *models.MemoryRecord) float64 { if m == nil { return 0 } - pos := float64(m.UsefulCount + m.RecallVerifiedCount) + useful := float64(m.UsefulCount) + verified := float64(m.RecallVerifiedCount) neg := float64(m.HumanNotUsefulCount + m.DedupDownweightCount) - sat := func(x float64) float64 { + sat := func(x, k float64) float64 { if x <= 0 { return 0 } - return x / (x + feedbackSaturation) + return x / (x + k) + } + switch posMode() { + case "none": + return sat(useful, feedbackSaturation) - sat(neg, feedbackSaturation) + case "weight": + return sat(useful+posWeight()*verified, feedbackSaturation) - sat(neg, feedbackSaturation) + case "tiebreak": + return sat(useful, feedbackSaturation) - sat(neg, feedbackSaturation) + + posWeight()*sat(verified, posTiebreakK) + default: // legacy — 与 2026-10-13 实现逐位一致(pos = useful+verified) + pos := useful + verified + return sat(pos, feedbackSaturation) - sat(neg, feedbackSaturation) } - return sat(pos) - sat(neg) } \ No newline at end of file diff --git a/go/internal/storage/popwatch.go b/go/internal/storage/popwatch.go new file mode 100644 index 0000000..6a8e0df --- /dev/null +++ b/go/internal/storage/popwatch.go @@ -0,0 +1,112 @@ +package storage + +import ( + "log" + "os" + "strings" + "sync" + "time" +) + +// t_23b6a940 AC7:热门度偏置的运行时看门狗(信号安全阈值兜底)。 +// +// 背景:FeedbackScore legacy 态把 recall_verified_count(热度)当质量信号 ⇒ +// 热门记忆硬顶 rank1(金标 119 条实测 MRR -22%)。离线 A/B 暴露问题的间隔 +// 长(本次隔了一整晚)。本看门狗是**运行时兜底**:最终 rank1 记忆的 +// recall_verified_count 连续 N 次超过全库 p95 阈值 ⇒ 打一条 ALERT 日志行 +// (传感器通道),传感器端脚本(~/.hermes/scripts/tools/zhiyi-popwatch.py, +// 挂既有告警通道)把快照转成告警。 +// +// 默认只报不改(report-only): +// - 不改排序、不写库、不发告警(告警由传感器端既有通道发); +// - 默认**开**(零排序影响、每 recall 一次 env 读 + 临界一次锁,开销可忽略), +// ZHIYI_POPWATCH=0 可完全关闭(关闭 ⇒ PopWatchRecord 立即返回,零行为变化)。 +// +// 阈值按全库实测标定(47148 条,2026-10-18 复测):recall_verified_count +// 全库 p95=4、非零 p99=62(≈全库 top 1% 热层)、max=389、非零 7416 条: +// +// ZHIYI_POPWATCH_P95 = 62(触发阈值,rank1 verified 严格大于它才累计; +// = 非零 p99,MoA 复核标定值) +// ZHIYI_POPWATCH_N = 5 (连续多少个 query 触发 ALERT) +func PopWatchEnabled() bool { + switch strings.ToLower(os.Getenv("ZHIYI_POPWATCH")) { + case "0", "false", "off": + return false + default: + return true // 默认开(report-only) + } +} + +func popWatchThreshold() int64 { return int64(envInt("ZHIYI_POPWATCH_P95", 62)) } +func popWatchN() int { return envInt("ZHIYI_POPWATCH_N", 5) } + +// popWatchState 进程内观测状态(每实例一份;「连续 N 个 query」语义就是进程内的)。 +type popWatchState struct { + mu sync.Mutex + consecutive int // rank1 verified > 阈值 的连续 query 数(否则清零) + lastRank1ID string // 最近一次 recall 的 rank1 记忆 id(审计用) + lastVerified int64 // 最近一次 recall 的 rank1 verified + lastSeenAt time.Time + alertsFired int // 本进程累计 ALERT 次数 + lastAlertAt time.Time +} + +// PopWatch 是全局观测器(GET /api/v1/popwatch 读快照;传感器端据此告警)。 +var PopWatch = &popWatchState{} + +// PopWatchRecord 观测一次 recall 的最终 rank1 的 recall_verified_count。 +// 调用点:recallUncached 末尾(主路径漏斗——热度偏置恰好作用于此处)。 +// 门关(ZHIYI_POPWATCH=0)⇒ 立即返回,零行为变化。 +// 越阈连续满 N 个 query ⇒ 打一条 ALERT 日志行(同一连续段只在满 N 瞬间打一次; +// 之后继续越阈不刷屏,回落再爬升才再打)。 +func PopWatchRecord(rank1ID string, rank1Verified int64) { + if !PopWatchEnabled() { + return + } + threshold := popWatchThreshold() + n := popWatchN() + PopWatch.mu.Lock() + defer PopWatch.mu.Unlock() + if rank1Verified > threshold { + PopWatch.consecutive++ + } else { + PopWatch.consecutive = 0 + } + PopWatch.lastRank1ID = rank1ID + PopWatch.lastVerified = rank1Verified + PopWatch.lastSeenAt = time.Now() + if PopWatch.consecutive == n { + PopWatch.alertsFired++ + PopWatch.lastAlertAt = time.Now() + log.Printf("[popwatch] ALERT: rank1 recall_verified_count=%d > p95 阈值 %d 连续 %d 个 query(report-only:不改排序/不写库;回滚:ZHIYI_POS_MODE=legacy 或 ZHIYI_FEEDBACK_BIAS=0)", + rank1Verified, threshold, n) + } +} + +// PopWatchSnapshot 供 GET /api/v1/popwatch(传感器端脚本读它转既有告警通道)。 +func PopWatchSnapshot() map[string]any { + threshold := popWatchThreshold() + n := popWatchN() + PopWatch.mu.Lock() + defer PopWatch.mu.Unlock() + lastSeen := "" + if !PopWatch.lastSeenAt.IsZero() { + lastSeen = PopWatch.lastSeenAt.Format(time.RFC3339) + } + lastAlert := "" + if !PopWatch.lastAlertAt.IsZero() { + lastAlert = PopWatch.lastAlertAt.Format(time.RFC3339) + } + return map[string]any{ + "enabled": PopWatchEnabled(), + "threshold_p95": threshold, + "consecutive_n": n, + "consecutive": PopWatch.consecutive, + "last_rank1_id": PopWatch.lastRank1ID, + "last_rank1_verified": PopWatch.lastVerified, + "last_seen_at": lastSeen, + "alerts_fired": PopWatch.alertsFired, + "last_alert_at": lastAlert, + "note": "report-only 观测:rank1 recall_verified_count 连续越 p95 阈值 = 热门度偏置行为证据;回滚 ZHIYI_POS_MODE=legacy / ZHIYI_FEEDBACK_BIAS=0", + } +} diff --git a/go/internal/storage/recall.go b/go/internal/storage/recall.go index ddfba66..3e2e888 100644 --- a/go/internal/storage/recall.go +++ b/go/internal/storage/recall.go @@ -1020,6 +1020,20 @@ func (p *RecallPipeline) recallUncached(query, namespace string, topK int, diver } }() + // Step 8: 热门度观测点(t_23b6a940 AC7,report-only、零排序影响): + // 记录最终 rank1 的 recall_verified_count;连续越 p95 阈值满 N 个 query ⇒ + // ALERT 日志行(传感器通道),传感器端 zhiyi-popwatch.py 转既有告警通道。 + // 仅观测主 ANN 池出来的 rank1(图谱扩展/FTS 尾部位注入的 rank1 无池内计数, + // 跳过不记)——热度偏置的作用面恰在主池 rerank 漏斗。 + if len(results) > 0 { + for i := range candidates { + if candidates[i].ID == results[0].ID { + PopWatchRecord(results[0].ID, int64(candidates[i].RecallVerifiedCount)) + break + } + } + } + return results, RecallMeta{CandidatesScanned: poolSize, Truncated: truncated}, nil }