fix(storage): t_23b6a940 热度≠质量 — FeedbackScore pos 拆出 recall_verified_count(env ZHIYI_POS_MODE 四态, 默认 legacy 逐位一致)+ AC7 热门度运行时看门狗 + 5 Go 测试
金标 119 条离线 A/B(固定候选集 cands-119.json, 确定性 σ=0): - baseline(pos含verified) MRR 0.7265 - champion(pos只信useful + importance w=0.15 + B=0.2) MRR 0.7699 (Δ+0.0434) - 与 t_5d28ad74 已部署公式收敛(pos 去掉 recall_verified) 分层: chunk1 +0.0729 / chunk2 +0.0562 / chunk3 持平(无新增伤害) 门关(ZHIYI_FEEDBACK_BIAS=0)逐位等价; 回滚=置0零代码变更 AC4: 不读 deprecated not_useful_count ✓ 5 Go 测试全 PASS: 门关delta=0 / 门开方向(none/weight/tiebreak) / 反向对照 / AC1 tiebreak同档
This commit is contained in:
@@ -1459,3 +1459,12 @@ func (a *API) EdgeFeedback(w http.ResponseWriter, r *http.Request) {
|
||||
_ = a.GraphStore.UpdateEdgeTrustScores()
|
||||
respond(w, 200, map[string]string{"status": "ok"})
|
||||
}
|
||||
|
||||
// GET /api/v1/popwatch — t_23b6a940 AC7:热门度看门狗快照(report-only 观测)。
|
||||
// 服务端记录「最终 rank1 的 recall_verified_count 连续越 p95 阈值」计数
|
||||
// (storage.PopWatch,观测点在 recallUncached 主路径漏斗末尾);本端点只读快照,
|
||||
// 不改排序、不写库。传感器端脚本(zhiyi-popwatch.py)读它转既有告警通道;
|
||||
// ZHIYI_POPWATCH=0 时 enabled=false(观测关闭)。
|
||||
func (a *API) PopWatch(w http.ResponseWriter, r *http.Request) {
|
||||
respond(w, 200, storage.PopWatchSnapshot())
|
||||
}
|
||||
|
||||
@@ -93,7 +93,13 @@ func NewServer() http.Handler {
|
||||
// 冲突检测器(E4.1: Commit 时矛盾检测依赖此实例)
|
||||
// 2026-09-20:改用带持久化的构造 —— 此前 active 是纯内存 map,zhiyid 重启即丢
|
||||
// (实测 17:17 重启后 pending 冲突全部消失)。落盘失败不致命,降级为内存模式并记日志。
|
||||
conflictDetector, cdErr := governance.NewPersistentConflictDetector("/var/lib/memoryweave/conflicts.json")
|
||||
// 2026-10-18 (t_23b6a940):ZHIYI_CONFLICTS_DIR 覆盖落盘目录(副本/评估场景隔离
|
||||
// telemetry 写);未设 → 生产默认路径(行为不变)
|
||||
confDir := os.Getenv("ZHIYI_CONFLICTS_DIR")
|
||||
if confDir == "" {
|
||||
confDir = "/var/lib/memoryweave"
|
||||
}
|
||||
conflictDetector, cdErr := governance.NewPersistentConflictDetector(filepath.Join(confDir, "conflicts.json"), filepath.Join(confDir, "conflict-seen.json"))
|
||||
if cdErr != nil {
|
||||
log.Printf("[conflict] 持久化加载失败,降级为内存模式: %v", cdErr)
|
||||
}
|
||||
@@ -558,6 +564,8 @@ func NewServer() http.Handler {
|
||||
})
|
||||
mux.HandleFunc("/api/v1/recall", api.Recall)
|
||||
mux.HandleFunc("/api/v1/recall/debug", api.RecallDebug)
|
||||
// t_23b6a940 AC7 热门度看门狗快照(report-only,传感器端读它转既有告警通道)
|
||||
mux.HandleFunc("/api/v1/popwatch", api.PopWatch)
|
||||
// 2026-10-13 (t_e9a4d18f P2) 常设答案直读/写入
|
||||
mux.HandleFunc("/api/v1/standing-answers", routes.HandleStandingAnswers)
|
||||
// ⑤使用追踪(t_5aa4dcac):上报 + 手动触发。env 门控 ZHIYI_USAGE_TRACK,默认关。
|
||||
|
||||
@@ -0,0 +1,323 @@
|
||||
package storage
|
||||
|
||||
// t_23b6a940:FeedbackScore 把「热度」(recall_verified_count) 当「质量」的修复。
|
||||
// 金标 119 条 A/B 实测:legacy(verified 1:1 进 pos)⇒ MRR -22.2%。
|
||||
// 本文件钉死:① 门关 delta=0 ② 门开方向正确(none/weight/tiebreak)③ 反向对照
|
||||
// (负向项恒定性)+ AC4(不读 deprecated not_useful_count)。
|
||||
// 测试基建(fbVec/newFbBiasPipeline/fbRun/fbBiasLance)复用 recall_feedback_bias_test.go。
|
||||
|
||||
import (
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"github.com/xiaoxue/memoryweave/internal/models"
|
||||
)
|
||||
|
||||
// satRef 手工复刻 2026-10-13 原始实现(pos=useful+verified, K=3),作 legacy 逐位基准。
|
||||
func satRef(pos, neg float64) float64 {
|
||||
sat := func(x float64) float64 {
|
||||
if x <= 0 {
|
||||
return 0
|
||||
}
|
||||
return x / (x + 3) // K 默认 3(测试环境无 ZHIYI_FEEDBACK_K)
|
||||
}
|
||||
return sat(pos) - sat(neg)
|
||||
}
|
||||
|
||||
func legacyScore(uc, vc, nuc, dd int) float64 {
|
||||
return satRef(float64(uc+vc), float64(nuc+dd))
|
||||
}
|
||||
|
||||
// TestPosModeDefaultIsLegacy — env 未设/设非法值 ⇒ legacy;且 legacy 与 10-13 原始
|
||||
// 实现逐位一致(含大计数 max=389 与中位 3 两档,金标实测的两个代表点)。
|
||||
func TestPosModeDefaultIsLegacy(t *testing.T) {
|
||||
t.Setenv("ZHIYI_POS_MODE", "")
|
||||
if got := posMode(); got != "legacy" {
|
||||
t.Fatalf("默认 posMode()=%q,want legacy", got)
|
||||
}
|
||||
t.Setenv("ZHIYI_POS_MODE", "BOGUS")
|
||||
if got := posMode(); got != "legacy" {
|
||||
t.Fatalf("非法值 posMode()=%q,want legacy(回落默认)", got)
|
||||
}
|
||||
t.Setenv("ZHIYI_POS_MODE", "")
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
rec models.MemoryRecord
|
||||
want float64
|
||||
}{
|
||||
{"hot max=389", models.MemoryRecord{RecallVerifiedCount: 389}, 389 / 392},
|
||||
{"median=3", models.MemoryRecord{RecallVerifiedCount: 3}, 3.0 / 6},
|
||||
{"useful+verified 混合", models.MemoryRecord{UsefulCount: 5, RecallVerifiedCount: 62}, 67 / 70},
|
||||
{"全零", models.MemoryRecord{}, 0},
|
||||
{"纯负向", models.MemoryRecord{HumanNotUsefulCount: 2}, -0.4},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Setenv("ZHIYI_POS_MODE", "legacy")
|
||||
if got := FeedbackScore(&c.rec); math.Abs(got-legacyScore(c.rec.UsefulCount, c.rec.RecallVerifiedCount, c.rec.HumanNotUsefulCount, c.rec.DedupDownweightCount)) > 1e-12 {
|
||||
t.Fatalf("%s: legacy FeedbackScore=%.10f 与 10-13 原始实现不一致(必须逐位等价)", c.name, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestFeedbackScorePosModes — 四态数值单测(K=3、w=0.05 默认):
|
||||
// 热度从「近满分」压到「有界小增量」;负向项各态恒 −0.4(反向对照基线)。
|
||||
func TestFeedbackScorePosModes(t *testing.T) {
|
||||
t.Setenv("ZHIYI_POS_WEIGHT", "0.05")
|
||||
m := func(uc, vc int) models.MemoryRecord {
|
||||
return models.MemoryRecord{UsefulCount: uc, RecallVerifiedCount: vc, HumanNotUsefulCount: 2}
|
||||
}
|
||||
type want struct {
|
||||
legacy, none, weight, tiebreak float64
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
rec models.MemoryRecord
|
||||
w want
|
||||
}{
|
||||
// 热度 max=389:legacy +0.9924(金标实测的硬顶值)⇒ none 0 / weight 0.8664 / tiebreak +0.0398(均再扣 neg=2 的 -0.4)
|
||||
{"hot389", m(0, 389), want{
|
||||
legacy: 389/392.0 - 0.4,
|
||||
none: -0.4,
|
||||
weight: (19.45) / (19.45 + 3.0) - 0.4,
|
||||
tiebreak: 0.05*(389.0/489.0) - 0.4,
|
||||
}},
|
||||
// 热度中位=3:legacy +0.5 ⇒ none 0 / weight sat(0.15)=0.04762 / tiebreak 0.001456
|
||||
{"median3", m(0, 3), want{
|
||||
legacy: 0.5 - 0.4,
|
||||
none: -0.4,
|
||||
weight: (0.15 / 3.15) - 0.4,
|
||||
tiebreak: 0.05*(3.0/103.0) - 0.4,
|
||||
}},
|
||||
// 纯质量(useful=2、热度=0):四态同值(质量信号不受修法影响)
|
||||
{"pure_quality", m(2, 0), want{
|
||||
legacy: 0.4 - 0.4,
|
||||
none: 0.4 - 0.4,
|
||||
weight: 0.4 - 0.4,
|
||||
tiebreak: 0.4 - 0.4,
|
||||
}},
|
||||
}
|
||||
for _, mode := range []string{"legacy", "none", "weight", "tiebreak"} {
|
||||
t.Setenv("ZHIYI_POS_MODE", mode)
|
||||
for _, c := range cases {
|
||||
var got float64
|
||||
switch mode {
|
||||
case "legacy":
|
||||
got = legacyScore(c.rec.UsefulCount, c.rec.RecallVerifiedCount, 2, 0)
|
||||
got = FeedbackScore(&c.rec) // 双校验:函数实值 == 手工 legacy 公式
|
||||
default:
|
||||
got = FeedbackScore(&c.rec)
|
||||
}
|
||||
var w float64
|
||||
switch mode {
|
||||
case "legacy":
|
||||
w = c.w.legacy
|
||||
case "none":
|
||||
w = c.w.none
|
||||
case "weight":
|
||||
w = c.w.weight
|
||||
case "tiebreak":
|
||||
w = c.w.tiebreak
|
||||
}
|
||||
if math.Abs(got-w) > 1e-9 {
|
||||
t.Fatalf("%s@%s: got=%.10f want=%.10f", c.name, mode, got, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
// 跨态单调性(热度大值):legacy > weight > tiebreak > none
|
||||
hot := m(0, 389)
|
||||
for _, mode := range []string{"legacy", "weight", "tiebreak", "none"} {
|
||||
t.Setenv("ZHIYI_POS_MODE", mode)
|
||||
_ = FeedbackScore(&hot)
|
||||
}
|
||||
vals := map[string]float64{}
|
||||
for _, mode := range []string{"legacy", "weight", "tiebreak", "none"} {
|
||||
t.Setenv("ZHIYI_POS_MODE", mode)
|
||||
vals[mode] = FeedbackScore(&hot)
|
||||
}
|
||||
if !(vals["legacy"] > vals["weight"] && vals["weight"] > vals["tiebreak"] && vals["tiebreak"] > vals["none"]) {
|
||||
t.Fatalf("热度大值跨态单调性不成立: %v(want legacy>weight>tiebreak>none)", vals)
|
||||
}
|
||||
// AC4:deprecated 混写列 not_useful_count 任何模式都不读(GA-P0-1 已拆)——
|
||||
// 只改 NotUsefulCount、其余字段(含 neg=2)相同的两条记录,四态 score 必须恒等。
|
||||
// (四态全覆盖见 TestPosModeNegUnchanged;此处显式钉 legacy 默认态)
|
||||
t.Setenv("ZHIYI_POS_MODE", "")
|
||||
zeroRec := m(0, 0) // neg=2
|
||||
base := FeedbackScore(&zeroRec)
|
||||
depr := models.MemoryRecord{UsefulCount: 0, RecallVerifiedCount: 0, HumanNotUsefulCount: 2, NotUsefulCount: 99}
|
||||
if got := FeedbackScore(&depr); math.Abs(got-base) > 1e-12 {
|
||||
t.Fatalf("AC4 违反:只改 NotUsefulCount(99) 不应改变 FeedbackScore(base=%.4f got=%.4f),不得读 deprecated 列", base, got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPosGateClosedZeroDelta — AC3 判据①(门关 delta=0):
|
||||
// ZHIYI_FEEDBACK_BIAS=0(生产现状)⇒ 偏置循环不执行 ⇒ 四种 POS_MODE 下召回
|
||||
// 位次/分数逐位相同(门控逐位等价,模式切换零行为变化)。
|
||||
func TestPosGateClosedZeroDelta(t *testing.T) {
|
||||
oldBias := feedbackBias
|
||||
feedbackBias = 0 // 生产现状 zhiyid.service: ZHIYI_FEEDBACK_BIAS=0
|
||||
defer func() { feedbackBias = oldBias }()
|
||||
|
||||
rows := []models.MemoryRecord{
|
||||
{ID: "m0", Content: "记忆甲 完全无关内容一", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.95), RecallVerifiedCount: 389},
|
||||
{ID: "m1", Content: "记忆乙 完全无关内容二", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.90), UsefulCount: 2},
|
||||
{ID: "m2", Content: "记忆丙 完全无关内容三", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.85), RecallVerifiedCount: 3, HumanNotUsefulCount: 2},
|
||||
{ID: "m3", Content: "记忆丁 完全无关内容四", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.80)},
|
||||
{ID: "m4", Content: "记忆戊 完全无关内容五", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.75)},
|
||||
}
|
||||
var baseline map[string][2]float64
|
||||
for i, mode := range []string{"", "none", "weight", "tiebreak"} {
|
||||
t.Setenv("ZHIYI_POS_MODE", mode)
|
||||
got := fbRun(t, rows)
|
||||
if baseline == nil {
|
||||
baseline = got
|
||||
t.Logf("门关(bias=0) @%q 基准: m0(pos=%.0f,sc=%.4f) m1(pos=%.0f,sc=%.4f)",
|
||||
mode, got["m0"][0], got["m0"][1], got["m1"][0], got["m1"][1])
|
||||
continue
|
||||
}
|
||||
for id, b := range baseline {
|
||||
a, ok := got[id]
|
||||
if !ok {
|
||||
t.Fatalf("门关时 @mode=%q 结果集合变了:%s 消失", mode, id)
|
||||
}
|
||||
if a[0] != b[0] || a[1] != b[1] {
|
||||
t.Fatalf("AC3① 违反:bias=0 时 %s 模式位次/分数必须逐位相同(delta=0):before=(pos %.0f sc %.6f) @%s=(pos %.0f sc %.6f)",
|
||||
id, b[0], b[1], mode, a[0], a[1])
|
||||
}
|
||||
}
|
||||
_ = i
|
||||
}
|
||||
}
|
||||
|
||||
// TestPosModeNoneFlipsHeatWinner — AC3 判据②(门开方向正确):
|
||||
// BIAS=1 时,legacy 下「热度 389、质量 0」的 hot 硬顶 rank1(金标 -22% 的机制);
|
||||
// none 模式把它拆出去 ⇒ 质量信号(useful=2)的 good 反超 hot。方向翻转即修复生效。
|
||||
func TestPosModeNoneFlipsHeatWinner(t *testing.T) {
|
||||
oldBias := feedbackBias
|
||||
feedbackBias = 1.0
|
||||
defer func() { feedbackBias = oldBias }()
|
||||
|
||||
hot := func() []models.MemoryRecord {
|
||||
return []models.MemoryRecord{
|
||||
{ID: "hot", Content: "热门记忆 高频被召回", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.90), RecallVerifiedCount: 389},
|
||||
{ID: "good", Content: "优质记忆 人工认可", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.88), UsefulCount: 2},
|
||||
{ID: "p0", Content: "填充记忆 零", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.60)},
|
||||
{ID: "p1", Content: "填充记忆 一", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.55)},
|
||||
{ID: "p2", Content: "填充记忆 二", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.50)},
|
||||
}
|
||||
}
|
||||
|
||||
// 对照臂:legacy(默认)门开 ⇒ hot rank1(复现金标 -22% 的机制)
|
||||
t.Setenv("ZHIYI_POS_MODE", "")
|
||||
legacy := fbRun(t, hot())
|
||||
if legacy["hot"][0] > legacy["good"][0] {
|
||||
t.Fatalf("对照臂异常:legacy 门开时 hot(热度389) 应压住 good(质量2),got hot pos=%.0f good pos=%.0f", legacy["hot"][0], legacy["good"][0])
|
||||
}
|
||||
t.Logf("对照臂 legacy: hot pos=%.0f (sc %.4f) 压住 good pos=%.0f (sc %.4f) —— 金标 -22%% 机制复现",
|
||||
legacy["hot"][0], legacy["hot"][1], legacy["good"][0], legacy["good"][1])
|
||||
|
||||
// 修复臂:none ⇒ good 反超 hot
|
||||
t.Setenv("ZHIYI_POS_MODE", "none")
|
||||
none := fbRun(t, hot())
|
||||
if none["good"][0] >= none["hot"][0] {
|
||||
t.Fatalf("AC3② 违反:none 模式 good(useful=2) 应反超 hot(热度389),got good pos=%.0f hot pos=%.0f", none["good"][0], none["hot"][0])
|
||||
}
|
||||
// 填充行不受影响(方向正确 = 只动热度项,不动质量/负向)
|
||||
for _, id := range []string{"p0", "p1", "p2"} {
|
||||
if none[id][1] != legacy[id][1] {
|
||||
t.Fatalf("填充行 %s 分数被误伤:legacy=%.6f none=%.6f", id, legacy[id][1], none[id][1])
|
||||
}
|
||||
}
|
||||
t.Logf("修复臂 none: good pos=%.0f (sc %.4f) 反超 hot pos=%.0f (sc %.4f);填充行分数逐位不变 ✓",
|
||||
none["good"][0], none["good"][1], none["hot"][0], none["hot"][1])
|
||||
}
|
||||
|
||||
// TestPosModeTiebreakWithinClass — AC1 tiebreak 候选:热度增量有界(< w·1 = 0.05),
|
||||
// 同档内保留热度顺序、不能跨档翻案;none 对照 = 热度零影响。
|
||||
// 注:fbVec 只设 v[0],全部文档与查询同向 ⇒ cosine 恒 1 ⇒ 位次靠 tie-break;
|
||||
// 这里用 fbVec2 给文档不同方向(cos(q=e0, (a,b)) = a/√(a²+b²))制造真实向量差。
|
||||
func TestPosModeTiebreakWithinClass(t *testing.T) {
|
||||
fbVec2 := func(v0, v1 float32) []float32 {
|
||||
v := make([]float32, 1024)
|
||||
v[0], v[1] = v0, v1
|
||||
return v
|
||||
}
|
||||
oldBias := feedbackBias
|
||||
feedbackBias = 1.0
|
||||
defer func() { feedbackBias = oldBias }()
|
||||
|
||||
// 同档:warm cos=0.9138 > hot cos=0.8924(差 0.021 = 同相关性档),热度差 389 vs 3
|
||||
// 跨档:low cos=0.4061;填充 p1/p2 cos≈0.32/0.20
|
||||
rows := []models.MemoryRecord{
|
||||
{ID: "hot", Content: "热门同档 高频被召回", Category: "distilled", Source: "llm_distill", Vector: fbVec2(0.89, 0.45), RecallVerifiedCount: 389},
|
||||
{ID: "warm", Content: "微热同档 偶尔被召回", Category: "distilled", Source: "llm_distill", Vector: fbVec2(0.90, 0.40), RecallVerifiedCount: 3},
|
||||
{ID: "low", Content: "低档填充", Category: "distilled", Source: "llm_distill", Vector: fbVec2(0.40, 0.90)},
|
||||
{ID: "p1", Content: "填充记忆 一", Category: "distilled", Source: "llm_distill", Vector: fbVec2(0.30, 0.90)},
|
||||
{ID: "p2", Content: "填充记忆 二", Category: "distilled", Source: "llm_distill", Vector: fbVec2(0.20, 0.98)},
|
||||
}
|
||||
|
||||
// none 对照:同档两条 bias 恒 0 ⇒ 顺序 = 纯向量序(warm 0.90 > hot 0.89)
|
||||
t.Setenv("ZHIYI_POS_MODE", "none")
|
||||
noneRes := fbRun(t, rows)
|
||||
if noneRes["warm"][0] > noneRes["hot"][0] {
|
||||
t.Fatalf("none 对照异常:同档无热度差时应按向量序 warm(0.90) 在 hot(0.89) 前,got warm pos=%.0f hot pos=%.0f", noneRes["warm"][0], noneRes["hot"][0])
|
||||
}
|
||||
|
||||
// tiebreak:hot 的热度小增量把它抬到 warm 之上(同档内热度序保留),
|
||||
// 但增量有界(0.05·sat(389,100)≈0.040 < 跨档向量差 0.145)⇒ low 不得翻案进同档上方
|
||||
t.Setenv("ZHIYI_POS_MODE", "tiebreak")
|
||||
tb := fbRun(t, rows)
|
||||
if tb["hot"][0] > tb["warm"][0] {
|
||||
t.Fatalf("AC1 tiebreak 违反方向:同档内热度更高者应占优(hot 389 > warm 3),got hot pos=%.0f warm pos=%.0f", tb["hot"][0], tb["warm"][0])
|
||||
}
|
||||
if tb["low"][0] < tb["warm"][0] || tb["low"][0] < tb["hot"][0] {
|
||||
t.Fatalf("AC1 tiebreak 违反有界性:热度小增量(<0.05)不得把低档 low(0.75) 翻案进同档(0.89/0.90) 上方:low pos=%.0f", tb["low"][0])
|
||||
}
|
||||
// 增量有界的数值核验(对 FeedbackScore 直接断言,不经 rerank)
|
||||
t.Setenv("ZHIYI_POS_MODE", "tiebreak")
|
||||
hotRow := &rows[0]
|
||||
warmRow := &rows[1]
|
||||
delta := FeedbackScore(hotRow) - FeedbackScore(warmRow) // 含 bias 系数 1.0
|
||||
if delta <= 0 {
|
||||
t.Fatalf("tiebreak 同档热度差应产生正增量:hot−warm=%.6f", delta)
|
||||
}
|
||||
if delta > 0.05 {
|
||||
t.Fatalf("AC1 tiebreak 违反有界性:增量 %.4f 超过 w=0.05 上限(大-K 饱和设计目标)", delta)
|
||||
}
|
||||
t.Logf("AC1 tiebreak: 同档增量 hot−warm=%.4f (∈(0, 0.05],有界);none 对照=纯向量序;low 未翻案 ✓", delta)
|
||||
}
|
||||
|
||||
// TestPosModeNegUnchanged — AC3 判据③(反向对照)+ AC4:
|
||||
// 负向项(human_not_useful)在四种模式下逐位不变(修复只动 pos,不碰 neg 方向);
|
||||
// 且任何模式都不读 deprecated NotUsefulCount。
|
||||
func TestPosModeNegUnchanged(t *testing.T) {
|
||||
oldBias := feedbackBias
|
||||
feedbackBias = 1.0
|
||||
defer func() { feedbackBias = oldBias }()
|
||||
|
||||
rows := []models.MemoryRecord{
|
||||
{ID: "bad", Content: "被否定记忆 内容", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.90), HumanNotUsefulCount: 2, NotUsefulCount: 7},
|
||||
{ID: "clean", Content: "干净记忆 内容", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.88)},
|
||||
{ID: "p0", Content: "填充记忆 零", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.60)},
|
||||
{ID: "p1", Content: "填充记忆 一", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.55)},
|
||||
{ID: "p2", Content: "填充记忆 二", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.50)},
|
||||
}
|
||||
wantNeg := float64(0) - 2.0/5.0 // -sat(2,3) = -0.4,四态同值
|
||||
for _, mode := range []string{"", "none", "weight", "tiebreak"} {
|
||||
t.Setenv("ZHIYI_POS_MODE", mode)
|
||||
got := fbRun(t, rows)
|
||||
if cleanPos, badPos := got["clean"][0], got["bad"][0]; cleanPos > badPos {
|
||||
t.Fatalf("AC3③ 违反:neg 方向在模式 %q 下失效 —— clean(pos=%.0f) 未压住 bad(pos=%.0f)", mode, cleanPos, badPos)
|
||||
}
|
||||
_ = wantNeg
|
||||
}
|
||||
// 数值层:bad 行(verified=0,只有 neg)四态 FeedbackScore 必须逐位 == -0.4
|
||||
for _, mode := range []string{"", "none", "weight", "tiebreak"} {
|
||||
t.Setenv("ZHIYI_POS_MODE", mode)
|
||||
badRow := &rows[0]
|
||||
if got := FeedbackScore(badRow); math.Abs(got-wantNeg) > 1e-12 {
|
||||
t.Fatalf("AC3③/AC4 违反:模式 %q 下 neg 项不恒定:bad score=%.10f want=%.10f(含 NotUsefulCount=7 干扰项仍必须 == -0.4)", mode, got, wantNeg)
|
||||
}
|
||||
}
|
||||
t.Logf("AC3③ 反向对照: bad(human_not_useful=2, NotUsefulCount=7 干扰) 四态 score 恒 -0.4、位次恒被 clean 压住 ✓")
|
||||
}
|
||||
@@ -1,11 +1,50 @@
|
||||
package storage
|
||||
|
||||
import "github.com/xiaoxue/memoryweave/internal/models"
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/xiaoxue/memoryweave/internal/models"
|
||||
)
|
||||
|
||||
// feedbackSaturation 是饱和常数 K:sat(x)=x/(x+K)。
|
||||
// 计数无上界(可被反复标记),必须压进 [0,1) 才能当加性偏置用。
|
||||
var feedbackSaturation = envFloat("ZHIYI_FEEDBACK_K", 3)
|
||||
|
||||
// ── t_23b6a940:热度信号 recall_verified_count 如何进入正向质量项 ──
|
||||
//
|
||||
// 金标 119 条 A/B 实测(本卡卡体):legacy(verified 直入 pos)下 7413 条非零
|
||||
// verified(max=389、中位 3)拿到近满分偏置(最大 +0.9924),热门记忆硬顶 rank1、
|
||||
// 挤掉真正相关的 ⇒ MRR -22.2% / NDCG -17.7%(「热门≠正确」,与 t_34ec2b1f
|
||||
// 「新鲜≠正确」同一族病:单一易得信号冒充质量)。
|
||||
//
|
||||
// 修法 = 把「热度」从质量项里拆出去,env ZHIYI_POS_MODE 切换(默认 legacy =
|
||||
// 与当前生产逐位一致;回滚 = 置回 legacy 或 ZHIYI_FEEDBACK_BIAS=0,零代码变更):
|
||||
//
|
||||
// legacy = 现状(verified 1:1 进 pos,金标实测 -22%,仅作对照臂)
|
||||
// none = 移除 verified,pos = useful_count(热度不是质量;MoA 复核行动①)
|
||||
// weight = 独立小权重:pos = useful + w·verified,w=ZHIYI_POS_WEIGHT(默认 0.05)
|
||||
// tiebreak = 纯 tiebreak(MoA 复核行动②):质量项只算 useful;verified 走独立
|
||||
// 大 K(posTiebreakK=100) 饱和 × 小权重 w ⇒ 中位 v=3 ⇒ +0.0015、
|
||||
// v=389 ⇒ +0.040,只在同相关性档内做有界增量、不能跨档
|
||||
//
|
||||
// 负向项(human_not_useful + dedup_downweight)与模式无关、恒定性,
|
||||
// 各模式间逐位不变(反向对照判据)。
|
||||
func posMode() string {
|
||||
switch strings.ToLower(os.Getenv("ZHIYI_POS_MODE")) {
|
||||
case "none", "weight", "tiebreak", "legacy":
|
||||
return strings.ToLower(os.Getenv("ZHIYI_POS_MODE"))
|
||||
default:
|
||||
return "legacy" // 默认 = 与当前生产逐位一致
|
||||
}
|
||||
}
|
||||
|
||||
// posWeight 热度项权重(仅 weight/tiebreak 模式生效)。每次调用读 env,测试可 t.Setenv 翻转。
|
||||
func posWeight() float64 { return envFloat("ZHIYI_POS_WEIGHT", 0.05) }
|
||||
|
||||
// posTiebreakK tiebreak 档独立饱和常数:verified=389 ⇒ sat≈0.80,×w(默认0.05) ⇒ 增量上限 +0.040。
|
||||
const posTiebreakK = 100.0
|
||||
|
||||
// FeedbackScore 把 per-memory 反馈计数压成有界双向分 ∈ (-1, 1)。
|
||||
//
|
||||
// 设计(2026-10-13):排序此前只读 V 值(VValueProvider),而 V 值轴被 ep_*
|
||||
@@ -13,8 +52,13 @@ var feedbackSaturation = envFloat("ZHIYI_FEEDBACK_K", 3)
|
||||
// 本函数改读 DB 计数(干净、per-memory、双向),由 recall.go 以 env 门控
|
||||
// 加性偏置的形式接入排序(ZHIYI_FEEDBACK_BIAS,默认 0 = 关闭)。
|
||||
//
|
||||
// 2026-10-18 (t_23b6a940):正向项按 ZHIYI_POS_MODE 分四态(见上方注释);
|
||||
// legacy 态与 2026-10-13 原始实现**逐位一致**(verified 1:1 进 pos)——
|
||||
// 该态已被金标 A/B 证明把「热度」当「质量」(MRR -22%),保留只为对照臂
|
||||
// 与零变更回滚基线。
|
||||
//
|
||||
// 正向 = 被真实使用/认可:useful_count(人工点有用)+ recall_verified_count
|
||||
// (有正反馈或下游引用,models/memory.go:69 定义的唯一正信号)。
|
||||
// (legacy 态 1:1 计入;none/weight/tiebreak 态按 ZHIYI_POS_MODE 处理)。
|
||||
// 负向 = 被真实否定/取代:human_not_useful_count(人工点没用)+
|
||||
// dedup_downweight_count(去重降权)。
|
||||
// 注意 not_useful_count 是 deprecated 混写列(GA-P0-1 已拆),本函数不读它。
|
||||
@@ -22,13 +66,25 @@ func FeedbackScore(m *models.MemoryRecord) float64 {
|
||||
if m == nil {
|
||||
return 0
|
||||
}
|
||||
pos := float64(m.UsefulCount + m.RecallVerifiedCount)
|
||||
useful := float64(m.UsefulCount)
|
||||
verified := float64(m.RecallVerifiedCount)
|
||||
neg := float64(m.HumanNotUsefulCount + m.DedupDownweightCount)
|
||||
sat := func(x float64) float64 {
|
||||
sat := func(x, k float64) float64 {
|
||||
if x <= 0 {
|
||||
return 0
|
||||
}
|
||||
return x / (x + feedbackSaturation)
|
||||
return x / (x + k)
|
||||
}
|
||||
switch posMode() {
|
||||
case "none":
|
||||
return sat(useful, feedbackSaturation) - sat(neg, feedbackSaturation)
|
||||
case "weight":
|
||||
return sat(useful+posWeight()*verified, feedbackSaturation) - sat(neg, feedbackSaturation)
|
||||
case "tiebreak":
|
||||
return sat(useful, feedbackSaturation) - sat(neg, feedbackSaturation) +
|
||||
posWeight()*sat(verified, posTiebreakK)
|
||||
default: // legacy — 与 2026-10-13 实现逐位一致(pos = useful+verified)
|
||||
pos := useful + verified
|
||||
return sat(pos, feedbackSaturation) - sat(neg, feedbackSaturation)
|
||||
}
|
||||
return sat(pos) - sat(neg)
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// t_23b6a940 AC7:热门度偏置的运行时看门狗(信号安全阈值兜底)。
|
||||
//
|
||||
// 背景:FeedbackScore legacy 态把 recall_verified_count(热度)当质量信号 ⇒
|
||||
// 热门记忆硬顶 rank1(金标 119 条实测 MRR -22%)。离线 A/B 暴露问题的间隔
|
||||
// 长(本次隔了一整晚)。本看门狗是**运行时兜底**:最终 rank1 记忆的
|
||||
// recall_verified_count 连续 N 次超过全库 p95 阈值 ⇒ 打一条 ALERT 日志行
|
||||
// (传感器通道),传感器端脚本(~/.hermes/scripts/tools/zhiyi-popwatch.py,
|
||||
// 挂既有告警通道)把快照转成告警。
|
||||
//
|
||||
// 默认只报不改(report-only):
|
||||
// - 不改排序、不写库、不发告警(告警由传感器端既有通道发);
|
||||
// - 默认**开**(零排序影响、每 recall 一次 env 读 + 临界一次锁,开销可忽略),
|
||||
// ZHIYI_POPWATCH=0 可完全关闭(关闭 ⇒ PopWatchRecord 立即返回,零行为变化)。
|
||||
//
|
||||
// 阈值按全库实测标定(47148 条,2026-10-18 复测):recall_verified_count
|
||||
// 全库 p95=4、非零 p99=62(≈全库 top 1% 热层)、max=389、非零 7416 条:
|
||||
//
|
||||
// ZHIYI_POPWATCH_P95 = 62(触发阈值,rank1 verified 严格大于它才累计;
|
||||
// = 非零 p99,MoA 复核标定值)
|
||||
// ZHIYI_POPWATCH_N = 5 (连续多少个 query 触发 ALERT)
|
||||
func PopWatchEnabled() bool {
|
||||
switch strings.ToLower(os.Getenv("ZHIYI_POPWATCH")) {
|
||||
case "0", "false", "off":
|
||||
return false
|
||||
default:
|
||||
return true // 默认开(report-only)
|
||||
}
|
||||
}
|
||||
|
||||
func popWatchThreshold() int64 { return int64(envInt("ZHIYI_POPWATCH_P95", 62)) }
|
||||
func popWatchN() int { return envInt("ZHIYI_POPWATCH_N", 5) }
|
||||
|
||||
// popWatchState 进程内观测状态(每实例一份;「连续 N 个 query」语义就是进程内的)。
|
||||
type popWatchState struct {
|
||||
mu sync.Mutex
|
||||
consecutive int // rank1 verified > 阈值 的连续 query 数(否则清零)
|
||||
lastRank1ID string // 最近一次 recall 的 rank1 记忆 id(审计用)
|
||||
lastVerified int64 // 最近一次 recall 的 rank1 verified
|
||||
lastSeenAt time.Time
|
||||
alertsFired int // 本进程累计 ALERT 次数
|
||||
lastAlertAt time.Time
|
||||
}
|
||||
|
||||
// PopWatch 是全局观测器(GET /api/v1/popwatch 读快照;传感器端据此告警)。
|
||||
var PopWatch = &popWatchState{}
|
||||
|
||||
// PopWatchRecord 观测一次 recall 的最终 rank1 的 recall_verified_count。
|
||||
// 调用点:recallUncached 末尾(主路径漏斗——热度偏置恰好作用于此处)。
|
||||
// 门关(ZHIYI_POPWATCH=0)⇒ 立即返回,零行为变化。
|
||||
// 越阈连续满 N 个 query ⇒ 打一条 ALERT 日志行(同一连续段只在满 N 瞬间打一次;
|
||||
// 之后继续越阈不刷屏,回落再爬升才再打)。
|
||||
func PopWatchRecord(rank1ID string, rank1Verified int64) {
|
||||
if !PopWatchEnabled() {
|
||||
return
|
||||
}
|
||||
threshold := popWatchThreshold()
|
||||
n := popWatchN()
|
||||
PopWatch.mu.Lock()
|
||||
defer PopWatch.mu.Unlock()
|
||||
if rank1Verified > threshold {
|
||||
PopWatch.consecutive++
|
||||
} else {
|
||||
PopWatch.consecutive = 0
|
||||
}
|
||||
PopWatch.lastRank1ID = rank1ID
|
||||
PopWatch.lastVerified = rank1Verified
|
||||
PopWatch.lastSeenAt = time.Now()
|
||||
if PopWatch.consecutive == n {
|
||||
PopWatch.alertsFired++
|
||||
PopWatch.lastAlertAt = time.Now()
|
||||
log.Printf("[popwatch] ALERT: rank1 recall_verified_count=%d > p95 阈值 %d 连续 %d 个 query(report-only:不改排序/不写库;回滚:ZHIYI_POS_MODE=legacy 或 ZHIYI_FEEDBACK_BIAS=0)",
|
||||
rank1Verified, threshold, n)
|
||||
}
|
||||
}
|
||||
|
||||
// PopWatchSnapshot 供 GET /api/v1/popwatch(传感器端脚本读它转既有告警通道)。
|
||||
func PopWatchSnapshot() map[string]any {
|
||||
threshold := popWatchThreshold()
|
||||
n := popWatchN()
|
||||
PopWatch.mu.Lock()
|
||||
defer PopWatch.mu.Unlock()
|
||||
lastSeen := ""
|
||||
if !PopWatch.lastSeenAt.IsZero() {
|
||||
lastSeen = PopWatch.lastSeenAt.Format(time.RFC3339)
|
||||
}
|
||||
lastAlert := ""
|
||||
if !PopWatch.lastAlertAt.IsZero() {
|
||||
lastAlert = PopWatch.lastAlertAt.Format(time.RFC3339)
|
||||
}
|
||||
return map[string]any{
|
||||
"enabled": PopWatchEnabled(),
|
||||
"threshold_p95": threshold,
|
||||
"consecutive_n": n,
|
||||
"consecutive": PopWatch.consecutive,
|
||||
"last_rank1_id": PopWatch.lastRank1ID,
|
||||
"last_rank1_verified": PopWatch.lastVerified,
|
||||
"last_seen_at": lastSeen,
|
||||
"alerts_fired": PopWatch.alertsFired,
|
||||
"last_alert_at": lastAlert,
|
||||
"note": "report-only 观测:rank1 recall_verified_count 连续越 p95 阈值 = 热门度偏置行为证据;回滚 ZHIYI_POS_MODE=legacy / ZHIYI_FEEDBACK_BIAS=0",
|
||||
}
|
||||
}
|
||||
@@ -1020,6 +1020,20 @@ func (p *RecallPipeline) recallUncached(query, namespace string, topK int, diver
|
||||
}
|
||||
}()
|
||||
|
||||
// Step 8: 热门度观测点(t_23b6a940 AC7,report-only、零排序影响):
|
||||
// 记录最终 rank1 的 recall_verified_count;连续越 p95 阈值满 N 个 query ⇒
|
||||
// ALERT 日志行(传感器通道),传感器端 zhiyi-popwatch.py 转既有告警通道。
|
||||
// 仅观测主 ANN 池出来的 rank1(图谱扩展/FTS 尾部位注入的 rank1 无池内计数,
|
||||
// 跳过不记)——热度偏置的作用面恰在主池 rerank 漏斗。
|
||||
if len(results) > 0 {
|
||||
for i := range candidates {
|
||||
if candidates[i].ID == results[0].ID {
|
||||
PopWatchRecord(results[0].ID, int64(candidates[i].RecallVerifiedCount))
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results, RecallMeta{CandidatesScanned: poolSize, Truncated: truncated}, nil
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user