Compare commits
18
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
89d1cf0bde | ||
|
|
c954aa2917 | ||
|
|
772a597a53 | ||
|
|
1f30474cf3 | ||
|
|
a1021bb70a | ||
|
|
e0412747f0 | ||
|
|
5f63b35602 | ||
|
|
3e29617cbe | ||
|
|
941b7eec32 | ||
|
|
71d5f2a7cd | ||
|
|
f6c373ea94 | ||
|
|
978f27a164 | ||
|
|
0bfa7a4bc1 | ||
|
|
25f68831fd | ||
|
|
42fafc495f | ||
|
|
d4b8688148 | ||
|
|
bd399dd007 | ||
|
|
ffc5b108c6 |
@@ -564,8 +564,11 @@ func (a *API) Recall(w http.ResponseWriter, r *http.Request) {
|
||||
// (LanceDB ANN 返回 + 去重 + 原始对话过滤之后、rerank/MMR 截断之前捕获)。
|
||||
// 此前误用 len(results)(最终产出数)作代理值——字段名说 "scanned"(扫描候选池)
|
||||
// 值却是"产出数",属"用文档补假名"。现改为 RecallWithMeta 返回的候选池真实大小。
|
||||
// keyword 路径未做 ANN 扫描 ⇒ 恒 0;graph 兜底路径 = 兜底返回数。
|
||||
// keyword 主路径未做 ANN 扫描 ⇒ 恒 0;graph 兜底路径 = 兜底返回数。
|
||||
annScanned := 0
|
||||
// t_3aa2a8c1:粗检索被 Rust 侧单次重读行数上限截断的标志(来源 RecallMeta.Truncated)。
|
||||
// graph 兜底 / keyword 空匹配兜底结果不经 IPC 粗检索 ⇒ 恒 false。
|
||||
truncated := false
|
||||
|
||||
// H5: Mode routing — hybrid / semantic / keyword
|
||||
if req.Mode == "" {
|
||||
@@ -589,6 +592,7 @@ func (a *API) Recall(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
cacheHit = kwMeta.CacheHit
|
||||
truncated = kwMeta.Truncated // t_3aa2a8c1
|
||||
if len(kwResults) == 0 {
|
||||
// 真无匹配(正常业务):成功 schema,results=[] + candidates_scanned 供区分
|
||||
fallbackResults := a.GraphStore.FallbackTextSearch(req.Query, req.Namespace, req.Limit)
|
||||
@@ -610,11 +614,16 @@ func (a *API) Recall(w http.ResponseWriter, r *http.Request) {
|
||||
if len(kwResults) > req.Limit {
|
||||
kwResults = kwResults[:req.Limit]
|
||||
}
|
||||
respond(w, 200, map[string]interface{}{
|
||||
kwResp := map[string]interface{}{
|
||||
"results": kwResults, "count": len(kwResults), "mode": "keyword",
|
||||
"candidates_scanned": 0,
|
||||
"cache_hit": cacheHit, // t_1c12e000:keyword 主路径命中 search cache 时显式标注
|
||||
})
|
||||
}
|
||||
// t_3aa2a8c1:truncated 只在 true 时暴露(省略 = 未截断,omitempty 语义)。
|
||||
if truncated {
|
||||
kwResp["truncated"] = true
|
||||
}
|
||||
respond(w, 200, kwResp)
|
||||
return
|
||||
case "semantic":
|
||||
// Pure semantic — BM25 off (H1 skipped, Pipeline Recall has BM25 built in — handled by switch)
|
||||
@@ -623,12 +632,14 @@ func (a *API) Recall(w http.ResponseWriter, r *http.Request) {
|
||||
results, meta, err = a.Pipeline.RecallWithMeta(req.Query, req.Namespace, req.Limit, req.Diversity)
|
||||
annScanned = meta.CandidatesScanned
|
||||
cacheHit = meta.CacheHit // t_1c12e000
|
||||
truncated = meta.Truncated // t_3aa2a8c1
|
||||
case "hybrid":
|
||||
// BM25 + vector combined (H1: BM25 scoring inside Pipeline.Recall)
|
||||
var meta storage.RecallMeta
|
||||
results, meta, err = a.Pipeline.RecallWithMeta(req.Query, req.Namespace, req.Limit, req.Diversity)
|
||||
annScanned = meta.CandidatesScanned
|
||||
cacheHit = meta.CacheHit // t_1c12e000
|
||||
truncated = meta.Truncated // t_3aa2a8c1
|
||||
}
|
||||
if err != nil {
|
||||
// P0: 降级到 graph.db 关键词搜索(仅 hybrid/semantic 主路径依赖向量/rerank)
|
||||
@@ -637,6 +648,7 @@ func (a *API) Recall(w http.ResponseWriter, r *http.Request) {
|
||||
results = convertFallbackResults(fallbackResults)
|
||||
annScanned = len(fallbackResults)
|
||||
cacheHit = false // t_1c12e000:graph 兜底结果不来自 search cache
|
||||
truncated = false // t_3aa2a8c1:graph.db 兜底结果不经 IPC 粗检索 ⇒ 无截断可言
|
||||
w.Header().Set("X-Fallback", "graph")
|
||||
} else {
|
||||
// t_2d4e3b92: 主路径与兜底都失败 ⇒ 非 2xx + 统一错误体(绝不在 200 里塞错误)
|
||||
@@ -655,7 +667,15 @@ func (a *API) Recall(w http.ResponseWriter, r *http.Request) {
|
||||
if len(results) > 0 && len(results) < 3 && req.Namespace != "shared" && req.Namespace != "" {
|
||||
vec, encErr := a.Embedder.EncodeQuery(req.Query)
|
||||
if encErr == nil {
|
||||
shared, _ := a.LanceDB.Search("memories", vec, 5, "shared")
|
||||
var shared []models.MemoryRecord
|
||||
// t_3aa2a8c1:补充搜同属粗检索 ⇒ 截断也计入本次 recall 的 truncated
|
||||
if rep, ok := a.LanceDB.(storage.SearchTruncationReporter); ok {
|
||||
var tSup bool
|
||||
shared, tSup, _ = rep.SearchWithTruncation("memories", vec, 5, "shared")
|
||||
truncated = truncated || tSup
|
||||
} else {
|
||||
shared, _ = a.LanceDB.Search("memories", vec, 5, "shared")
|
||||
}
|
||||
for _, m := range shared {
|
||||
// 去重:跳过已在 own namespace 结果中的记忆
|
||||
alreadyHave := false
|
||||
@@ -692,7 +712,15 @@ func (a *API) Recall(w http.ResponseWriter, r *http.Request) {
|
||||
if req.Namespace != "wiki-curator-main" && req.Namespace != "" {
|
||||
if vec, encErr := a.Embedder.EncodeQuery(req.Query); encErr == nil {
|
||||
for _, extra := range []string{"wiki-curator-main"} {
|
||||
extraHits, _ := a.LanceDB.Search("memories", vec, 5, extra)
|
||||
var extraHits []models.MemoryRecord
|
||||
// t_3aa2a8c1:补充搜同属粗检索 ⇒ 截断也计入本次 recall 的 truncated
|
||||
if rep, ok := a.LanceDB.(storage.SearchTruncationReporter); ok {
|
||||
var tSup bool
|
||||
extraHits, tSup, _ = rep.SearchWithTruncation("memories", vec, 5, extra)
|
||||
truncated = truncated || tSup
|
||||
} else {
|
||||
extraHits, _ = a.LanceDB.Search("memories", vec, 5, extra)
|
||||
}
|
||||
for _, m := range extraHits {
|
||||
dup := false
|
||||
for _, r := range results {
|
||||
@@ -787,7 +815,7 @@ func (a *API) Recall(w http.ResponseWriter, r *http.Request) {
|
||||
if results == nil {
|
||||
results = []models.RecallResult{}
|
||||
}
|
||||
respond(w, 200, map[string]interface{}{
|
||||
resp := map[string]interface{}{
|
||||
"results": results,
|
||||
"count": len(results),
|
||||
"candidates_scanned": annScanned,
|
||||
@@ -796,7 +824,15 @@ func (a *API) Recall(w http.ResponseWriter, r *http.Request) {
|
||||
// true ⇒ candidates_scanned=0 是「读了 60s 内的旧快照」(含 L1/L2);
|
||||
// false ⇒ 本次真扫了表(candidates_scanned 为真实候选池大小)。
|
||||
"cache_hit": cacheHit,
|
||||
})
|
||||
}
|
||||
// t_3aa2a8c1:truncated —— 只在 true 时暴露(键缺失 = 未截断,等价 omitempty 语义)。
|
||||
// true = 本次 recall 的粗检索/补充搜至少一次命中 Rust 侧单次重读行数上限
|
||||
// (默认 20000,ZHIYI_HEAVY_READ_MAX_ROWS)被截断为部分结果 ⇒ count/
|
||||
// candidates_scanned 是截断后状态,调用方不得当作「全量」。
|
||||
if truncated {
|
||||
resp["truncated"] = true
|
||||
}
|
||||
respond(w, 200, resp)
|
||||
|
||||
// H3: Async trust score update
|
||||
if a.GraphStore != nil {
|
||||
@@ -1423,3 +1459,12 @@ func (a *API) EdgeFeedback(w http.ResponseWriter, r *http.Request) {
|
||||
_ = a.GraphStore.UpdateEdgeTrustScores()
|
||||
respond(w, 200, map[string]string{"status": "ok"})
|
||||
}
|
||||
|
||||
// GET /api/v1/popwatch — t_23b6a940 AC7:热门度看门狗快照(report-only 观测)。
|
||||
// 服务端记录「最终 rank1 的 recall_verified_count 连续越 p95 阈值」计数
|
||||
// (storage.PopWatch,观测点在 recallUncached 主路径漏斗末尾);本端点只读快照,
|
||||
// 不改排序、不写库。传感器端脚本(zhiyi-popwatch.py)读它转既有告警通道;
|
||||
// ZHIYI_POPWATCH=0 时 enabled=false(观测关闭)。
|
||||
func (a *API) PopWatch(w http.ResponseWriter, r *http.Request) {
|
||||
respond(w, 200, storage.PopWatchSnapshot())
|
||||
}
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
package routes
|
||||
|
||||
// t_3aa2a8c1:`/api/v1/recall` 响应暴露 `truncated`(Rust 侧单次重读行数上限截断标志,
|
||||
// 引擎已算、此前只打日志不上抛)。选定的省略语义 = **键缺失即未截断(omitempty)**。
|
||||
// 本测试经路由层机械判定:
|
||||
// ① 粗检索被截断(后端实现 SearchTruncationReporter 且上报 true)⇒ 响应含 "truncated":true
|
||||
// ② 未截断 / 纯 Go 后端(不实现 reporter)⇒ 键缺失(不出现 truncated:false 以外的值)
|
||||
// ③ 回归:既有字段 results/count/candidates_scanned/mode/cache_hit 行为不变
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/xiaoxue/memoryweave/internal/governance"
|
||||
"github.com/xiaoxue/memoryweave/internal/models"
|
||||
"github.com/xiaoxue/memoryweave/internal/storage"
|
||||
)
|
||||
|
||||
// truncReportingLDB 复用同包 fakeLanceDB(recall_error_shape_test.go),
|
||||
// 额外实现可选扩展接口 SearchWithTruncationReporter(t_3aa2a8c1)。
|
||||
type truncReportingLDB struct {
|
||||
*fakeLanceDB
|
||||
reportTruncated bool
|
||||
}
|
||||
|
||||
func (f *truncReportingLDB) SearchWithTruncation(table string, vector []float32, topK int, namespaceFilter string) ([]models.MemoryRecord, bool, error) {
|
||||
if f.searchErr != nil {
|
||||
return nil, false, f.searchErr
|
||||
}
|
||||
if len(f.searchResults) > topK {
|
||||
return f.searchResults[:topK], f.reportTruncated, nil
|
||||
}
|
||||
return f.searchResults, f.reportTruncated, nil
|
||||
}
|
||||
|
||||
func recallBody(t *testing.T, db storage.LanceDB, body string) (int, map[string]interface{}, string) {
|
||||
t.Helper()
|
||||
// 测试须可复跑(t_3aa2a8c1 实测踩坑):SearchCacheInstance 由 storage 包 init() 恒建
|
||||
// (L2 = 共享 Redis,TTL 60s)⇒ 上一次 run 的 recall 结果会进 L2,下一次 run 同 key
|
||||
// 命中缓存路径(Truncated 恒 false)⇒ 断言 flaky。逐测试摘除缓存(测后恢复),
|
||||
// 恒走未缓存路径。包内无 t.Parallel ⇒ 改全局安全。
|
||||
saved := storage.SearchCacheInstance
|
||||
storage.SearchCacheInstance = nil
|
||||
t.Cleanup(func() { storage.SearchCacheInstance = saved })
|
||||
|
||||
emb := storage.NewEmbedder("")
|
||||
rerank := storage.NewReranker("", emb)
|
||||
api := NewAPI(db, emb, rerank, nil, governance.NewInMemoryGraph())
|
||||
req := httptest.NewRequest("POST", "/api/v1/recall", strings.NewReader(body))
|
||||
w := httptest.NewRecorder()
|
||||
api.Recall(w, req)
|
||||
var m map[string]interface{}
|
||||
_ = json.Unmarshal(w.Body.Bytes(), &m)
|
||||
return w.Code, m, w.Body.String()
|
||||
}
|
||||
|
||||
// 既有响应键回归基线(t_2d4e3b92 成功 schema + t_1c12e000 cache_hit + t_949840b1 candidates_scanned)
|
||||
var recallBaselineKeys = []string{"results", "count", "candidates_scanned", "mode", "cache_hit"}
|
||||
|
||||
func assertBaseline(t *testing.T, m map[string]interface{}, body string) {
|
||||
t.Helper()
|
||||
for _, k := range recallBaselineKeys {
|
||||
if _, ok := m[k]; !ok {
|
||||
t.Fatalf("既有字段 %q 缺失(回归破坏),body=%s", k, body)
|
||||
}
|
||||
}
|
||||
if res, ok := m["results"].([]interface{}); ok {
|
||||
if cnt, _ := m["count"].(float64); int(cnt) != len(res) {
|
||||
t.Fatalf("count=%v 与 len(results)=%d 不一致(回归破坏)", m["count"], len(res))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ① 粗检索被行数上限截断 ⇒ 响应暴露 truncated:true + 既有字段回归不变
|
||||
func TestRecall_TruncatedExposedWhenHitCap(t *testing.T) {
|
||||
db := &truncReportingLDB{
|
||||
fakeLanceDB: &fakeLanceDB{searchResults: []models.MemoryRecord{
|
||||
{ID: "mem_1", Content: "test memory one", Category: "distilled"},
|
||||
{ID: "mem_2", Content: "test memory two", Category: "distilled"},
|
||||
}},
|
||||
reportTruncated: true,
|
||||
}
|
||||
code, m, body := recallBody(t, db, `{"query":"test query","limit":5,"namespace":"shared"}`)
|
||||
if code != 200 {
|
||||
t.Fatalf("expected 200, got %d; body=%s", code, body)
|
||||
}
|
||||
if v, ok := m["truncated"]; !ok || v != true {
|
||||
t.Fatalf("粗检索已截断,响应必须含 truncated:true;实际 key 存在=%v 值=%v body=%s", ok, m["truncated"], body)
|
||||
}
|
||||
assertBaseline(t, m, body)
|
||||
}
|
||||
|
||||
// ①b keyword 主路径(kwMeta 透传)同判据
|
||||
func TestRecall_Keyword_TruncatedExposed(t *testing.T) {
|
||||
db := &truncReportingLDB{
|
||||
fakeLanceDB: &fakeLanceDB{searchResults: []models.MemoryRecord{
|
||||
{ID: "mem_1", Content: "keyword memory", Category: "distilled"},
|
||||
}},
|
||||
reportTruncated: true,
|
||||
}
|
||||
code, m, body := recallBody(t, db, `{"query":"keyword query","limit":5,"mode":"keyword","namespace":"shared"}`)
|
||||
if code != 200 {
|
||||
t.Fatalf("expected 200, got %d; body=%s", code, body)
|
||||
}
|
||||
if v, ok := m["truncated"]; !ok || v != true {
|
||||
t.Fatalf("keyword 路径粗检索已截断,响应必须含 truncated:true;body=%s", body)
|
||||
}
|
||||
assertBaseline(t, m, body)
|
||||
}
|
||||
|
||||
// ② 未截断 ⇒ 键缺失(选定语义:省略=未截断),不得出现 truncated:false 以外的值
|
||||
func TestRecall_NotTruncated_KeyOmitted(t *testing.T) {
|
||||
db := &truncReportingLDB{
|
||||
fakeLanceDB: &fakeLanceDB{searchResults: []models.MemoryRecord{
|
||||
{ID: "mem_1", Content: "test memory one", Category: "distilled"},
|
||||
}},
|
||||
reportTruncated: false,
|
||||
}
|
||||
code, m, body := recallBody(t, db, `{"query":"test query","limit":5,"namespace":"shared"}`)
|
||||
if code != 200 {
|
||||
t.Fatalf("expected 200, got %d; body=%s", code, body)
|
||||
}
|
||||
if v, ok := m["truncated"]; ok {
|
||||
t.Fatalf("未截断场景键应缺失(省略=未截断);实际 truncated=%v body=%s", v, body)
|
||||
}
|
||||
assertBaseline(t, m, body)
|
||||
}
|
||||
|
||||
// ②b 纯 Go 后端(不实现 SearchTruncationReporter)⇒ 键缺失,零值语义、行为不变
|
||||
func TestRecall_PureGoBackend_NoTruncatedKey(t *testing.T) {
|
||||
db := &fakeLanceDB{searchResults: []models.MemoryRecord{
|
||||
{ID: "mem_1", Content: "test memory one", Category: "distilled"},
|
||||
}}
|
||||
code, m, body := recallBody(t, db, `{"query":"test query","limit":5,"namespace":"shared"}`)
|
||||
if code != 200 {
|
||||
t.Fatalf("expected 200, got %d; body=%s", code, body)
|
||||
}
|
||||
if _, ok := m["truncated"]; ok {
|
||||
t.Fatalf("纯 Go 后端无行数上限,truncated 键必须缺失;body=%s", body)
|
||||
}
|
||||
assertBaseline(t, m, body)
|
||||
}
|
||||
@@ -1,8 +1,32 @@
|
||||
// 织忆 MemoryWeave — 路由层测试隔离(t_e5997ca4 起)
|
||||
// 织忆 MemoryWeave — routes 包测试隔离(两个护栏合并 · t_e5997ca4 + t_b7a8ed9c)
|
||||
//
|
||||
// 单元测试不得写到生产落盘点。selfoptimize 包已有同款 TestMain 先例
|
||||
// (曾实测:整套测试跑完把 /var/lib/memoryweave/dashboard.json 写成测试数字)。
|
||||
// ⚠️ 本文件是**两个 TestMain 的合并体**。两卡分支各自加了一个 TestMain:
|
||||
// · t_e5997ca4 → testmain_test.go(落盘点护栏)
|
||||
// · t_b7a8ed9c → scorecache_guard_test.go(打分缓存护栏)
|
||||
// 在各自分支上互不干扰,合并到 main 后同包两个 TestMain **编译冲突**
|
||||
// (multiple definitions of TestMain)⇒ 必须合进一个文件。
|
||||
// (此点由 t_b7a8ed9c 在 scorecache_guard_test.go 顶部预先标注;小怡合并时未读文件
|
||||
// 而漏看,由全套测试的 setup failed 拦下 —— 见 worklog 教训。)
|
||||
//
|
||||
// 护栏①:单元测试不得写到生产落盘点(t_e5997ca4)
|
||||
// ---------------------------------------------------------------
|
||||
// selfoptimize 包已有同款 TestMain 先例(曾实测:整套测试跑完把
|
||||
// /var/lib/memoryweave/dashboard.json 写成测试数字)。
|
||||
// gap close/repair 端点会触发 Dash.persist()(本地 JSON 兜底)⇒ 统一重定向到临时目录。
|
||||
//
|
||||
// 护栏②:默认关闭 distill 打分缓存(t_b7a8ed9c)
|
||||
// ---------------------------------------------------------------
|
||||
// 背景:distill 打分记忆化(0dc571d)+ Redis L2(36db9bb,键 zhiyi:distill-score:*,
|
||||
// TTL 168h)后,**本包 E2E 测试(zz_commit_conflict_gate_test.go)会读写生产 Redis**:
|
||||
// 第一次 run L2 miss → 真调 mock LLM 并把「测试内容哈希 → 结果」写进生产;
|
||||
// 7 天内后续 run L2 hit → LLM 被静默短路 → 数 LLM 调用次数的断言(base+1)必挂。
|
||||
// 实测 2026-10-17:TestCommitConflictGate_E2E 16.5s 固定 FAIL,
|
||||
// 「expected >= 1 LLM distill calls, got 0」。
|
||||
//
|
||||
// 治法(对齐项目门禁规矩③:加缓存必须给该包 TestMain 加关闭开关):
|
||||
// Setenv ZHIYI_SCORE_CACHE=0 —— distill.getScoreCache 的 Once 闭包**活读 env**
|
||||
// (包级变量在 init 时已定型,跨包设不了),本进程内缓存整体禁用:
|
||||
// 不读 L2(LLM 真被调用,计数断言有覆盖)、不写 L2(不再污染生产)。
|
||||
package routes
|
||||
|
||||
import (
|
||||
@@ -12,12 +36,16 @@ import (
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
// 护栏①:落盘点重定向到临时目录(宁可测试直接失败,也不写生产落盘点)
|
||||
dir, err := os.MkdirTemp("", "routes-test-dash-*")
|
||||
if err != nil {
|
||||
// 宁可测试直接失败,也不写生产落盘点
|
||||
panic(err)
|
||||
}
|
||||
_ = os.Setenv("ZHIYI_DASHBOARD_PATH", filepath.Join(dir, "dashboard.json"))
|
||||
|
||||
// 护栏②:关闭 distill 打分缓存(不读/不写生产 Redis L2)
|
||||
_ = os.Setenv("ZHIYI_SCORE_CACHE", "0")
|
||||
|
||||
code := m.Run()
|
||||
_ = os.RemoveAll(dir)
|
||||
os.Exit(code)
|
||||
|
||||
@@ -93,7 +93,13 @@ func NewServer() http.Handler {
|
||||
// 冲突检测器(E4.1: Commit 时矛盾检测依赖此实例)
|
||||
// 2026-09-20:改用带持久化的构造 —— 此前 active 是纯内存 map,zhiyid 重启即丢
|
||||
// (实测 17:17 重启后 pending 冲突全部消失)。落盘失败不致命,降级为内存模式并记日志。
|
||||
conflictDetector, cdErr := governance.NewPersistentConflictDetector("/var/lib/memoryweave/conflicts.json")
|
||||
// 2026-10-18 (t_23b6a940):ZHIYI_CONFLICTS_DIR 覆盖落盘目录(副本/评估场景隔离
|
||||
// telemetry 写);未设 → 生产默认路径(行为不变)
|
||||
confDir := os.Getenv("ZHIYI_CONFLICTS_DIR")
|
||||
if confDir == "" {
|
||||
confDir = "/var/lib/memoryweave"
|
||||
}
|
||||
conflictDetector, cdErr := governance.NewPersistentConflictDetector(filepath.Join(confDir, "conflicts.json"), filepath.Join(confDir, "conflict-seen.json"))
|
||||
if cdErr != nil {
|
||||
log.Printf("[conflict] 持久化加载失败,降级为内存模式: %v", cdErr)
|
||||
}
|
||||
@@ -558,6 +564,8 @@ func NewServer() http.Handler {
|
||||
})
|
||||
mux.HandleFunc("/api/v1/recall", api.Recall)
|
||||
mux.HandleFunc("/api/v1/recall/debug", api.RecallDebug)
|
||||
// t_23b6a940 AC7 热门度看门狗快照(report-only,传感器端读它转既有告警通道)
|
||||
mux.HandleFunc("/api/v1/popwatch", api.PopWatch)
|
||||
// 2026-10-13 (t_e9a4d18f P2) 常设答案直读/写入
|
||||
mux.HandleFunc("/api/v1/standing-answers", routes.HandleStandingAnswers)
|
||||
// ⑤使用追踪(t_5aa4dcac):上报 + 手动触发。env 门控 ZHIYI_USAGE_TRACK,默认关。
|
||||
|
||||
@@ -82,7 +82,12 @@ var (
|
||||
|
||||
func getScoreCache() *scoreCache {
|
||||
scoreCacheOnce.Do(func() {
|
||||
if scoreCacheDisabled {
|
||||
// 活读 env(而非只认包级变量):包级变量在进程 init 时定型,跨包测试
|
||||
// (如 routes)在 TestMain 里 Setenv 时已太迟 —— 让 TestMain 能真正关掉
|
||||
// 缓存(测试进程确定性:数 LLM 调用次数的断言不被 L2 命中短路)。
|
||||
// 生产语义不变:env 在进程生命周期内稳定,启动时的 ZHIYI_SCORE_CACHE=0
|
||||
// 依然在此处被读到。
|
||||
if scoreCacheDisabled || os.Getenv("ZHIYI_SCORE_CACHE") == "0" {
|
||||
log.Printf("[scorecache] 已禁用(环境变量 ZHIYI_SCORE_CACHE=0 或测试环境)")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
// 织忆 MemoryWeave — 图谱扩展(ExpandFromResults)锁死 + 实测(t_818f77be)
|
||||
//
|
||||
// 背景:旧 InMemoryGraph/FileGraph 的 ExpandFromResults 拿 r.Category(类目名,如 "distilled")
|
||||
// 当实体名去 Navigate,永远匹配不到任何图节点 ⇒ 结构性零产出。修正后导航起点 = 从 r.Content
|
||||
// 提实体(复用 extract*Entities + normalize*EntityID,与 ExpandWithSummary 同一套逻辑)。
|
||||
//
|
||||
// 本文件两类断言:
|
||||
// 1) 锁死(AC#4):构造小图 + 一条 recall ⇒ 结果必含 category=graph_expanded,且 Score=0.5 不变。
|
||||
// 2) 实测(AC#3):同批 10 条 recall ⇒ 补入率 > 0,给出具体数字。
|
||||
// 判读方式:改前树副本跑本测试 ⇒ 两条均得 0(复现结构零);改后树 ⇒ 均 >0。
|
||||
//
|
||||
// 说明:FileGraph.AddNode/AddEdge 存在潜伏死锁(save() 在写锁内又取 RLock ⇒ 非重入 RWMutex 自锁),
|
||||
// 故 FileGraph 用例改走「写 JSON 文件 + NewFileGraph 加载」,绕开该潜伏 bug(不在本卡修复范围)。
|
||||
package governance
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/xiaoxue/memoryweave/internal/models"
|
||||
)
|
||||
|
||||
// testNS 固定命名空间,用于证明「实体修对 + namespace 对上 ⇒ 补入 >0」,
|
||||
// 以及「namespace 对不上 ⇒ 二次阻塞补 0」(次级怀疑)。
|
||||
const testNS = "t-ns-expand"
|
||||
|
||||
// chainNodes 一条 4 节点链(中文实体,与生产 graph.db 节点命名一致:n_<名字>)。
|
||||
// 抽取器 extractPotentialEntitiesFromContent 对「2~8 字中文串」最可靠(英文带空格提不到),
|
||||
// 故实体一律用中文名;边端点是归一后形态(n_织忆 / n_图谱 / n_召回 / n_向量)。
|
||||
var (
|
||||
chainNodeIDs = []string{"n_织忆", "n_图谱", "n_召回", "n_向量"}
|
||||
chainNodeNames = map[string]string{"n_织忆": "织忆", "n_图谱": "图谱", "n_召回": "召回", "n_向量": "向量"}
|
||||
chainEdges = []struct{ id, src, dst string }{
|
||||
{"e-ab", "n_织忆", "n_图谱"},
|
||||
{"e-bc", "n_图谱", "n_召回"},
|
||||
{"e-cd", "n_召回", "n_向量"},
|
||||
}
|
||||
)
|
||||
|
||||
// addChainInMemory 给 InMemoryGraph 灌入链(AddNode/AddEdge 无死锁,可直接用)。
|
||||
func addChainInMemory(g *InMemoryGraph) {
|
||||
for _, id := range chainNodeIDs {
|
||||
_ = g.AddNode(id, chainNodeNames[id], "entity", testNS)
|
||||
}
|
||||
for _, e := range chainEdges {
|
||||
_ = g.AddEdge(e.id, e.src, e.dst, "LINKS_TO", testNS, 1.0)
|
||||
}
|
||||
}
|
||||
|
||||
// buildFileGraphJSON 把链写成 JSON 文件,再 NewFileGraph 加载 —— 绕开 AddNode/AddEdge 潜伏死锁。
|
||||
func buildFileGraphJSON(t *testing.T) (*FileGraph, string) {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "graph-expand-test.json")
|
||||
data := FileGraphData{Version: 2}
|
||||
for _, id := range chainNodeIDs {
|
||||
data.Nodes = append(data.Nodes, &FileGraphNode{
|
||||
ID: id, Name: chainNodeNames[id], Type: "entity",
|
||||
Namespace: testNS, PageRank: 0.15, EvidenceCount: 1,
|
||||
})
|
||||
}
|
||||
for _, e := range chainEdges {
|
||||
data.Edges = append(data.Edges, &FileGraphEdge{
|
||||
ID: e.id, Source: e.src, Target: e.dst,
|
||||
Relation: "LINKS_TO", Weight: 1.0, Namespace: testNS,
|
||||
})
|
||||
}
|
||||
if err := os.WriteFile(path, mustMarshal(data), 0644); err != nil {
|
||||
t.Fatalf("write filegraph json: %v", err)
|
||||
}
|
||||
fg, err := NewFileGraph(path)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFileGraph: %v", err)
|
||||
}
|
||||
return fg, path
|
||||
}
|
||||
|
||||
func mustMarshal(v interface{}) []byte {
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// tenRecalls 构造 10 条 recall 结果:正文含中文实体「织忆 图谱 召回(±向量)」
|
||||
// (⇒ 归一 n_织忆/n_图谱/n_召回/n_向量,均在图内),类目名保持 "distilled"
|
||||
// (与生产实测一致,证明导航起点绝不能是类目名)。
|
||||
func tenRecalls() []models.RecallResult {
|
||||
contents := []string{
|
||||
"织忆 图谱 召回 的扩展机制",
|
||||
"图谱 与 召回 相关实体",
|
||||
"召回 通过 向量 关联 织忆",
|
||||
"织忆 图谱 召回 向量 的拓扑",
|
||||
"向量 在最末端;织忆 是起点",
|
||||
"recall 提及 织忆 图谱 召回",
|
||||
"链路 织忆 图谱 召回 向量",
|
||||
"织忆 节点 见 于 distilled 记录",
|
||||
"图谱 召回 织忆 共现",
|
||||
"最终 recall:织忆 图谱 召回 向量",
|
||||
}
|
||||
out := make([]models.RecallResult, 0, len(contents))
|
||||
for i, c := range contents {
|
||||
out = append(out, models.RecallResult{
|
||||
ID: "mem_expand_" + string(rune('a'+i)),
|
||||
Content: c,
|
||||
Category: "distilled",
|
||||
Score: 0.9,
|
||||
Namespace: testNS,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// countGraphExpanded 逐条跑 10 条 recall,返回每调用 graph_expanded 条数 + 总和,并校验
|
||||
// 每条补入结果 category=graph_expanded 且 Score=0.5(AC#5:不动 0.5 降权)。
|
||||
func countGraphExpanded(g interface {
|
||||
ExpandFromResults(results []models.RecallResult, namespace string, maxHops int) []models.RecallResult
|
||||
}, t *testing.T, ns string) (perCall []int, total int) {
|
||||
t.Helper()
|
||||
recalls := tenRecalls()
|
||||
for i, r := range recalls {
|
||||
got := g.ExpandFromResults([]models.RecallResult{r}, ns, 1)
|
||||
for _, e := range got {
|
||||
if e.Category != "graph_expanded" {
|
||||
t.Fatalf("call %d: 补入结果 category=%q 期望 graph_expanded", i, e.Category)
|
||||
}
|
||||
if e.Score != 0.5 {
|
||||
t.Fatalf("call %d: 补入结果 score=%v 期望 0.5(AC#5 不改降权)", i, e.Score)
|
||||
}
|
||||
}
|
||||
perCall = append(perCall, len(got))
|
||||
total += len(got)
|
||||
}
|
||||
return perCall, total
|
||||
}
|
||||
|
||||
// ── 锁死(AC#4):InMemoryGraph ──────────────────────────────────
|
||||
// 改前树必 FAIL(起点=类目 ⇒ 0 补入);改后树 PASS。
|
||||
func TestInMemoryGraph_ExpandFromResults_Inserts(t *testing.T) {
|
||||
g := NewInMemoryGraph()
|
||||
addChainInMemory(g)
|
||||
|
||||
recalls := tenRecalls()
|
||||
got := g.ExpandFromResults(recalls, testNS, 1)
|
||||
if len(got) == 0 {
|
||||
t.Fatalf("InMemoryGraph ExpandFromResults 补 0 条 — 导航起点仍非实体(结构性零产出未修)")
|
||||
}
|
||||
for _, e := range got {
|
||||
if e.Category != "graph_expanded" {
|
||||
t.Fatalf("category=%q 期望 graph_expanded", e.Category)
|
||||
}
|
||||
}
|
||||
t.Logf("InMemoryGraph: 一条 10-recall 批量补入 %d 条 graph_expanded", len(got))
|
||||
}
|
||||
|
||||
// ── 锁死(AC#4):FileGraph ─────────────────────────────────────
|
||||
func TestFileGraph_ExpandFromResults_Inserts(t *testing.T) {
|
||||
fg, _ := buildFileGraphJSON(t)
|
||||
|
||||
recalls := tenRecalls()
|
||||
got := fg.ExpandFromResults(recalls, testNS, 1)
|
||||
if len(got) == 0 {
|
||||
t.Fatalf("FileGraph ExpandFromResults 补 0 条 — 导航起点仍非实体(结构性零产出未修)")
|
||||
}
|
||||
for _, e := range got {
|
||||
if e.Category != "graph_expanded" {
|
||||
t.Fatalf("category=%q 期望 graph_expanded", e.Category)
|
||||
}
|
||||
}
|
||||
t.Logf("FileGraph: 一条 10-recall 批量补入 %d 条 graph_expanded", len(got))
|
||||
}
|
||||
|
||||
// ── 次级怀疑(AC「namespace 不二次阻塞」)────────────────────────
|
||||
// 实体修对、但传错 namespace ⇒ 应补 0(证明 namespace 是硬约束,需对上才有产出)。
|
||||
func TestInMemoryGraph_ExpandFromResults_NamespaceBlock(t *testing.T) {
|
||||
g := NewInMemoryGraph()
|
||||
addChainInMemory(g)
|
||||
|
||||
got := g.ExpandFromResults(tenRecalls(), "no-such-ns", 1)
|
||||
if len(got) != 0 {
|
||||
t.Fatalf("namespace 对不上却补入 %d 条 — 预期 0(namespace 过滤应生效)", len(got))
|
||||
}
|
||||
t.Logf("namespace 不匹配 ⇒ 补 0(证明 namespace 为硬约束;对上即有产出)")
|
||||
}
|
||||
|
||||
// ── 实测(AC#3):同批 10 条 recall 补入率 ──────────────────────
|
||||
// 改前树:perCall 全 0 ⇒ total 0(复现结构零)。改后树:total>0,输出具体数字。
|
||||
func TestExpandFromResults_TenRecall_Empirical(t *testing.T) {
|
||||
gm := NewInMemoryGraph()
|
||||
addChainInMemory(gm)
|
||||
pIn, totalIn := countGraphExpanded(gm, t, testNS)
|
||||
t.Logf("InMemoryGraph 10-recall 补入: per-call=%v total=%d", pIn, totalIn)
|
||||
if totalIn == 0 {
|
||||
t.Fatalf("InMemoryGraph 改后仍补 0(10/10 全 0)— 结构性零产出未消除")
|
||||
}
|
||||
|
||||
fg, _ := buildFileGraphJSON(t)
|
||||
pF, totalF := countGraphExpanded(fg, t, testNS)
|
||||
t.Logf("FileGraph 10-recall 补入: per-call=%v total=%d", pF, totalF)
|
||||
if totalF == 0 {
|
||||
t.Fatalf("FileGraph 改后仍补 0(10/10 全 0)— 结构性零产出未消除")
|
||||
}
|
||||
}
|
||||
@@ -3,44 +3,76 @@ package governance
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/xiaoxue/memoryweave/internal/models"
|
||||
)
|
||||
|
||||
// graphExpandDebugEnabled 控制「图谱扩展」debug 日志(AC 可观测性,t_818f77be)。
|
||||
// 默认关;设 ZHIYI_GRAPH_EXPAND_DEBUG=1 打开。每次 recall 至多一条(低频),避免刷屏。
|
||||
func graphExpandDebugEnabled() bool {
|
||||
return os.Getenv("ZHIYI_GRAPH_EXPAND_DEBUG") == "1"
|
||||
}
|
||||
|
||||
// ExpandFromResults 从 recall 结果出发,双向 BFS 扩展图谱邻接节点
|
||||
// 实现 GraphStore 接口
|
||||
//
|
||||
// 导航起点 = 从 r.Content 提实体(复用 extractPotentialEntitiesFromContent + normalizeEntityID,
|
||||
// 与同文件 ExpandWithSummary 同一套逻辑)。旧实现拿 r.Category(类目名,如 "distilled")
|
||||
// 当实体去 Navigate,永远匹配不到任何图节点 ⇒ 结构性零产出(t_818f77be 实测 10/10 全 distilled)。
|
||||
// 且 InMemoryGraph.Navigate 返回字段是 source/target(非 to/from),旧读法 p["to"]/p["from"]
|
||||
// 恒取到空 ⇒ 即便实体修对也补 0。两处一并修正。
|
||||
func (g *InMemoryGraph) ExpandFromResults(results []models.RecallResult, namespace string, maxHops int) []models.RecallResult {
|
||||
var expanded []models.RecallResult
|
||||
seen := make(map[string]bool)
|
||||
seenEntities := make(map[string]bool)
|
||||
|
||||
// 收集已有结果 ID
|
||||
for _, r := range results {
|
||||
seen[r.ID] = true
|
||||
}
|
||||
|
||||
// 从每个结果出发扩展
|
||||
startEntities := 0 // 可观测性:去重后实际导航的起点实体数 N
|
||||
totalPathsHit := 0 // 可观测性:命中路径总数 M(补前去重)
|
||||
// 从每个结果正文提实体出发扩展
|
||||
for _, r := range results {
|
||||
paths, err := g.Navigate(r.Category, maxHops, namespace, nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, p := range paths {
|
||||
to, _ := p["to"].(string)
|
||||
from, _ := p["from"].(string)
|
||||
for _, entity := range extractPotentialEntitiesFromContent(r.Content) {
|
||||
if seenEntities[entity] {
|
||||
continue
|
||||
}
|
||||
seenEntities[entity] = true
|
||||
startEntities++
|
||||
|
||||
for _, id := range []string{to, from} {
|
||||
if id != "" && !seen[id] {
|
||||
seen[id] = true
|
||||
expanded = append(expanded, models.RecallResult{
|
||||
ID: id,
|
||||
Category: "graph_expanded",
|
||||
Score: 0.5, // 图谱扩展降权
|
||||
})
|
||||
nodeID := normalizeEntityID(entity)
|
||||
paths, err := g.Navigate(nodeID, maxHops, namespace, nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
totalPathsHit += len(paths)
|
||||
for _, p := range paths {
|
||||
to, _ := p["target"].(string)
|
||||
from, _ := p["source"].(string)
|
||||
|
||||
for _, id := range []string{to, from} {
|
||||
if id != "" && !seen[id] {
|
||||
seen[id] = true
|
||||
expanded = append(expanded, models.RecallResult{
|
||||
ID: id,
|
||||
Category: "graph_expanded",
|
||||
Score: 0.5, // 图谱扩展降权
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// 可观测性:默认关(ZHIYI_GRAPH_EXPAND_DEBUG=1 打开),保证「下次坏了不沉默」。
|
||||
if graphExpandDebugEnabled() {
|
||||
log.Printf("[graph-expand] ns=%s maxHops=%d 起点实体 %d 个 / 命中 %d 条 / 补入 %d 条",
|
||||
namespace, maxHops, startEntities, totalPathsHit, len(expanded))
|
||||
}
|
||||
return expanded
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ package governance
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"math"
|
||||
"os"
|
||||
"strings"
|
||||
@@ -457,32 +458,53 @@ func (fg *FileGraph) UpdateEdgeTrustScores() error { return
|
||||
func (fg *FileGraph) ExpandFromResults(results []models.RecallResult, namespace string, maxHops int) []models.RecallResult {
|
||||
var expanded []models.RecallResult
|
||||
seen := make(map[string]bool)
|
||||
seenEntities := make(map[string]bool)
|
||||
|
||||
for _, r := range results {
|
||||
seen[r.ID] = true
|
||||
}
|
||||
|
||||
// 导航起点 = 从 r.Content 提实体(复用 extractFileGraphEntities + normalizeFileGraphEntityID,
|
||||
// 与同文件 ExpandWithSummary 同一套逻辑)。旧实现拿 r.Category(类目名,如 "distilled")
|
||||
// 当实体去 Navigate,永远匹配不到任何图节点 ⇒ 结构性零产出(t_818f77be)。
|
||||
startEntities := 0 // 可观测性:去重后实际导航的起点实体数 N
|
||||
totalPathsHit := 0 // 可观测性:命中路径总数 M(补前去重)
|
||||
for _, r := range results {
|
||||
paths, err := fg.Navigate(r.Category, maxHops, namespace, nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, p := range paths {
|
||||
target, _ := p["target"].(string)
|
||||
source, _ := p["source"].(string)
|
||||
for _, entity := range extractFileGraphEntities(r.Content) {
|
||||
if seenEntities[entity] {
|
||||
continue
|
||||
}
|
||||
seenEntities[entity] = true
|
||||
startEntities++
|
||||
|
||||
for _, id := range []string{target, source} {
|
||||
if id != "" && !seen[id] {
|
||||
seen[id] = true
|
||||
expanded = append(expanded, models.RecallResult{
|
||||
ID: id,
|
||||
Category: "graph_expanded",
|
||||
Score: 0.5,
|
||||
})
|
||||
nodeID := normalizeFileGraphEntityID(entity)
|
||||
paths, err := fg.Navigate(nodeID, maxHops, namespace, nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
totalPathsHit += len(paths)
|
||||
for _, p := range paths {
|
||||
target, _ := p["target"].(string)
|
||||
source, _ := p["source"].(string)
|
||||
|
||||
for _, id := range []string{target, source} {
|
||||
if id != "" && !seen[id] {
|
||||
seen[id] = true
|
||||
expanded = append(expanded, models.RecallResult{
|
||||
ID: id,
|
||||
Category: "graph_expanded",
|
||||
Score: 0.5,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// 可观测性:默认关(ZHIYI_GRAPH_EXPAND_DEBUG=1 打开),保证「下次坏了不沉默」。
|
||||
if graphExpandDebugEnabled() {
|
||||
log.Printf("[graph-expand] ns=%s maxHops=%d 起点实体 %d 个 / 命中 %d 条 / 补入 %d 条",
|
||||
namespace, maxHops, startEntities, totalPathsHit, len(expanded))
|
||||
}
|
||||
return expanded
|
||||
}
|
||||
|
||||
|
||||
@@ -46,7 +46,7 @@ type MemoryRecord struct {
|
||||
Category string `json:"category"` // system_fact, user_pref, proj_context, tool_usage, code_snippet
|
||||
Vector []float32 `json:"vector"` // bge-m3 1024维,L2归一化
|
||||
Tier string `json:"tier"` // normal, core(core永不衰减)
|
||||
Importance float64 `json:"importance"` // 重要性得分 = recency_factor × (1+log(1+recall_count))
|
||||
Importance float64 `json:"importance"` // 档位重要性:写入侧决定(storage/importance_rule.go,t_789898bd:规则关恒 1.0;开则 core=0.9/wiki,llm_distill=0.7/legacy,unknown=0.4/recovery=0.3)。注意:排序用的「运行时 importance」另算(quality_fusion.computeImportance = recency × (1+log(1+recall_seen))),两者是不同量
|
||||
QualityScore float64 `json:"quality_score"`
|
||||
UsefulCount int `json:"useful_count"` // 被标记 useful 的次数
|
||||
// GA-P0-1 (t_718d8707):not_useful_count 曾把两种语义混写——
|
||||
|
||||
@@ -0,0 +1,323 @@
|
||||
package storage
|
||||
|
||||
// t_23b6a940:FeedbackScore 把「热度」(recall_verified_count) 当「质量」的修复。
|
||||
// 金标 119 条 A/B 实测:legacy(verified 1:1 进 pos)⇒ MRR -22.2%。
|
||||
// 本文件钉死:① 门关 delta=0 ② 门开方向正确(none/weight/tiebreak)③ 反向对照
|
||||
// (负向项恒定性)+ AC4(不读 deprecated not_useful_count)。
|
||||
// 测试基建(fbVec/newFbBiasPipeline/fbRun/fbBiasLance)复用 recall_feedback_bias_test.go。
|
||||
|
||||
import (
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"github.com/xiaoxue/memoryweave/internal/models"
|
||||
)
|
||||
|
||||
// satRef 手工复刻 2026-10-13 原始实现(pos=useful+verified, K=3),作 legacy 逐位基准。
|
||||
func satRef(pos, neg float64) float64 {
|
||||
sat := func(x float64) float64 {
|
||||
if x <= 0 {
|
||||
return 0
|
||||
}
|
||||
return x / (x + 3) // K 默认 3(测试环境无 ZHIYI_FEEDBACK_K)
|
||||
}
|
||||
return sat(pos) - sat(neg)
|
||||
}
|
||||
|
||||
func legacyScore(uc, vc, nuc, dd int) float64 {
|
||||
return satRef(float64(uc+vc), float64(nuc+dd))
|
||||
}
|
||||
|
||||
// TestPosModeDefaultIsLegacy — env 未设/设非法值 ⇒ legacy;且 legacy 与 10-13 原始
|
||||
// 实现逐位一致(含大计数 max=389 与中位 3 两档,金标实测的两个代表点)。
|
||||
func TestPosModeDefaultIsLegacy(t *testing.T) {
|
||||
t.Setenv("ZHIYI_POS_MODE", "")
|
||||
if got := posMode(); got != "legacy" {
|
||||
t.Fatalf("默认 posMode()=%q,want legacy", got)
|
||||
}
|
||||
t.Setenv("ZHIYI_POS_MODE", "BOGUS")
|
||||
if got := posMode(); got != "legacy" {
|
||||
t.Fatalf("非法值 posMode()=%q,want legacy(回落默认)", got)
|
||||
}
|
||||
t.Setenv("ZHIYI_POS_MODE", "")
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
rec models.MemoryRecord
|
||||
want float64
|
||||
}{
|
||||
{"hot max=389", models.MemoryRecord{RecallVerifiedCount: 389}, 389 / 392},
|
||||
{"median=3", models.MemoryRecord{RecallVerifiedCount: 3}, 3.0 / 6},
|
||||
{"useful+verified 混合", models.MemoryRecord{UsefulCount: 5, RecallVerifiedCount: 62}, 67 / 70},
|
||||
{"全零", models.MemoryRecord{}, 0},
|
||||
{"纯负向", models.MemoryRecord{HumanNotUsefulCount: 2}, -0.4},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Setenv("ZHIYI_POS_MODE", "legacy")
|
||||
if got := FeedbackScore(&c.rec); math.Abs(got-legacyScore(c.rec.UsefulCount, c.rec.RecallVerifiedCount, c.rec.HumanNotUsefulCount, c.rec.DedupDownweightCount)) > 1e-12 {
|
||||
t.Fatalf("%s: legacy FeedbackScore=%.10f 与 10-13 原始实现不一致(必须逐位等价)", c.name, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestFeedbackScorePosModes — 四态数值单测(K=3、w=0.05 默认):
|
||||
// 热度从「近满分」压到「有界小增量」;负向项各态恒 −0.4(反向对照基线)。
|
||||
func TestFeedbackScorePosModes(t *testing.T) {
|
||||
t.Setenv("ZHIYI_POS_WEIGHT", "0.05")
|
||||
m := func(uc, vc int) models.MemoryRecord {
|
||||
return models.MemoryRecord{UsefulCount: uc, RecallVerifiedCount: vc, HumanNotUsefulCount: 2}
|
||||
}
|
||||
type want struct {
|
||||
legacy, none, weight, tiebreak float64
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
rec models.MemoryRecord
|
||||
w want
|
||||
}{
|
||||
// 热度 max=389:legacy +0.9924(金标实测的硬顶值)⇒ none 0 / weight 0.8664 / tiebreak +0.0398(均再扣 neg=2 的 -0.4)
|
||||
{"hot389", m(0, 389), want{
|
||||
legacy: 389/392.0 - 0.4,
|
||||
none: -0.4,
|
||||
weight: (19.45) / (19.45 + 3.0) - 0.4,
|
||||
tiebreak: 0.05*(389.0/489.0) - 0.4,
|
||||
}},
|
||||
// 热度中位=3:legacy +0.5 ⇒ none 0 / weight sat(0.15)=0.04762 / tiebreak 0.001456
|
||||
{"median3", m(0, 3), want{
|
||||
legacy: 0.5 - 0.4,
|
||||
none: -0.4,
|
||||
weight: (0.15 / 3.15) - 0.4,
|
||||
tiebreak: 0.05*(3.0/103.0) - 0.4,
|
||||
}},
|
||||
// 纯质量(useful=2、热度=0):四态同值(质量信号不受修法影响)
|
||||
{"pure_quality", m(2, 0), want{
|
||||
legacy: 0.4 - 0.4,
|
||||
none: 0.4 - 0.4,
|
||||
weight: 0.4 - 0.4,
|
||||
tiebreak: 0.4 - 0.4,
|
||||
}},
|
||||
}
|
||||
for _, mode := range []string{"legacy", "none", "weight", "tiebreak"} {
|
||||
t.Setenv("ZHIYI_POS_MODE", mode)
|
||||
for _, c := range cases {
|
||||
var got float64
|
||||
switch mode {
|
||||
case "legacy":
|
||||
got = legacyScore(c.rec.UsefulCount, c.rec.RecallVerifiedCount, 2, 0)
|
||||
got = FeedbackScore(&c.rec) // 双校验:函数实值 == 手工 legacy 公式
|
||||
default:
|
||||
got = FeedbackScore(&c.rec)
|
||||
}
|
||||
var w float64
|
||||
switch mode {
|
||||
case "legacy":
|
||||
w = c.w.legacy
|
||||
case "none":
|
||||
w = c.w.none
|
||||
case "weight":
|
||||
w = c.w.weight
|
||||
case "tiebreak":
|
||||
w = c.w.tiebreak
|
||||
}
|
||||
if math.Abs(got-w) > 1e-9 {
|
||||
t.Fatalf("%s@%s: got=%.10f want=%.10f", c.name, mode, got, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
// 跨态单调性(热度大值):legacy > weight > tiebreak > none
|
||||
hot := m(0, 389)
|
||||
for _, mode := range []string{"legacy", "weight", "tiebreak", "none"} {
|
||||
t.Setenv("ZHIYI_POS_MODE", mode)
|
||||
_ = FeedbackScore(&hot)
|
||||
}
|
||||
vals := map[string]float64{}
|
||||
for _, mode := range []string{"legacy", "weight", "tiebreak", "none"} {
|
||||
t.Setenv("ZHIYI_POS_MODE", mode)
|
||||
vals[mode] = FeedbackScore(&hot)
|
||||
}
|
||||
if !(vals["legacy"] > vals["weight"] && vals["weight"] > vals["tiebreak"] && vals["tiebreak"] > vals["none"]) {
|
||||
t.Fatalf("热度大值跨态单调性不成立: %v(want legacy>weight>tiebreak>none)", vals)
|
||||
}
|
||||
// AC4:deprecated 混写列 not_useful_count 任何模式都不读(GA-P0-1 已拆)——
|
||||
// 只改 NotUsefulCount、其余字段(含 neg=2)相同的两条记录,四态 score 必须恒等。
|
||||
// (四态全覆盖见 TestPosModeNegUnchanged;此处显式钉 legacy 默认态)
|
||||
t.Setenv("ZHIYI_POS_MODE", "")
|
||||
zeroRec := m(0, 0) // neg=2
|
||||
base := FeedbackScore(&zeroRec)
|
||||
depr := models.MemoryRecord{UsefulCount: 0, RecallVerifiedCount: 0, HumanNotUsefulCount: 2, NotUsefulCount: 99}
|
||||
if got := FeedbackScore(&depr); math.Abs(got-base) > 1e-12 {
|
||||
t.Fatalf("AC4 违反:只改 NotUsefulCount(99) 不应改变 FeedbackScore(base=%.4f got=%.4f),不得读 deprecated 列", base, got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPosGateClosedZeroDelta — AC3 判据①(门关 delta=0):
|
||||
// ZHIYI_FEEDBACK_BIAS=0(生产现状)⇒ 偏置循环不执行 ⇒ 四种 POS_MODE 下召回
|
||||
// 位次/分数逐位相同(门控逐位等价,模式切换零行为变化)。
|
||||
func TestPosGateClosedZeroDelta(t *testing.T) {
|
||||
oldBias := feedbackBias
|
||||
feedbackBias = 0 // 生产现状 zhiyid.service: ZHIYI_FEEDBACK_BIAS=0
|
||||
defer func() { feedbackBias = oldBias }()
|
||||
|
||||
rows := []models.MemoryRecord{
|
||||
{ID: "m0", Content: "记忆甲 完全无关内容一", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.95), RecallVerifiedCount: 389},
|
||||
{ID: "m1", Content: "记忆乙 完全无关内容二", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.90), UsefulCount: 2},
|
||||
{ID: "m2", Content: "记忆丙 完全无关内容三", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.85), RecallVerifiedCount: 3, HumanNotUsefulCount: 2},
|
||||
{ID: "m3", Content: "记忆丁 完全无关内容四", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.80)},
|
||||
{ID: "m4", Content: "记忆戊 完全无关内容五", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.75)},
|
||||
}
|
||||
var baseline map[string][2]float64
|
||||
for i, mode := range []string{"", "none", "weight", "tiebreak"} {
|
||||
t.Setenv("ZHIYI_POS_MODE", mode)
|
||||
got := fbRun(t, rows)
|
||||
if baseline == nil {
|
||||
baseline = got
|
||||
t.Logf("门关(bias=0) @%q 基准: m0(pos=%.0f,sc=%.4f) m1(pos=%.0f,sc=%.4f)",
|
||||
mode, got["m0"][0], got["m0"][1], got["m1"][0], got["m1"][1])
|
||||
continue
|
||||
}
|
||||
for id, b := range baseline {
|
||||
a, ok := got[id]
|
||||
if !ok {
|
||||
t.Fatalf("门关时 @mode=%q 结果集合变了:%s 消失", mode, id)
|
||||
}
|
||||
if a[0] != b[0] || a[1] != b[1] {
|
||||
t.Fatalf("AC3① 违反:bias=0 时 %s 模式位次/分数必须逐位相同(delta=0):before=(pos %.0f sc %.6f) @%s=(pos %.0f sc %.6f)",
|
||||
id, b[0], b[1], mode, a[0], a[1])
|
||||
}
|
||||
}
|
||||
_ = i
|
||||
}
|
||||
}
|
||||
|
||||
// TestPosModeNoneFlipsHeatWinner — AC3 判据②(门开方向正确):
|
||||
// BIAS=1 时,legacy 下「热度 389、质量 0」的 hot 硬顶 rank1(金标 -22% 的机制);
|
||||
// none 模式把它拆出去 ⇒ 质量信号(useful=2)的 good 反超 hot。方向翻转即修复生效。
|
||||
func TestPosModeNoneFlipsHeatWinner(t *testing.T) {
|
||||
oldBias := feedbackBias
|
||||
feedbackBias = 1.0
|
||||
defer func() { feedbackBias = oldBias }()
|
||||
|
||||
hot := func() []models.MemoryRecord {
|
||||
return []models.MemoryRecord{
|
||||
{ID: "hot", Content: "热门记忆 高频被召回", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.90), RecallVerifiedCount: 389},
|
||||
{ID: "good", Content: "优质记忆 人工认可", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.88), UsefulCount: 2},
|
||||
{ID: "p0", Content: "填充记忆 零", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.60)},
|
||||
{ID: "p1", Content: "填充记忆 一", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.55)},
|
||||
{ID: "p2", Content: "填充记忆 二", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.50)},
|
||||
}
|
||||
}
|
||||
|
||||
// 对照臂:legacy(默认)门开 ⇒ hot rank1(复现金标 -22% 的机制)
|
||||
t.Setenv("ZHIYI_POS_MODE", "")
|
||||
legacy := fbRun(t, hot())
|
||||
if legacy["hot"][0] > legacy["good"][0] {
|
||||
t.Fatalf("对照臂异常:legacy 门开时 hot(热度389) 应压住 good(质量2),got hot pos=%.0f good pos=%.0f", legacy["hot"][0], legacy["good"][0])
|
||||
}
|
||||
t.Logf("对照臂 legacy: hot pos=%.0f (sc %.4f) 压住 good pos=%.0f (sc %.4f) —— 金标 -22%% 机制复现",
|
||||
legacy["hot"][0], legacy["hot"][1], legacy["good"][0], legacy["good"][1])
|
||||
|
||||
// 修复臂:none ⇒ good 反超 hot
|
||||
t.Setenv("ZHIYI_POS_MODE", "none")
|
||||
none := fbRun(t, hot())
|
||||
if none["good"][0] >= none["hot"][0] {
|
||||
t.Fatalf("AC3② 违反:none 模式 good(useful=2) 应反超 hot(热度389),got good pos=%.0f hot pos=%.0f", none["good"][0], none["hot"][0])
|
||||
}
|
||||
// 填充行不受影响(方向正确 = 只动热度项,不动质量/负向)
|
||||
for _, id := range []string{"p0", "p1", "p2"} {
|
||||
if none[id][1] != legacy[id][1] {
|
||||
t.Fatalf("填充行 %s 分数被误伤:legacy=%.6f none=%.6f", id, legacy[id][1], none[id][1])
|
||||
}
|
||||
}
|
||||
t.Logf("修复臂 none: good pos=%.0f (sc %.4f) 反超 hot pos=%.0f (sc %.4f);填充行分数逐位不变 ✓",
|
||||
none["good"][0], none["good"][1], none["hot"][0], none["hot"][1])
|
||||
}
|
||||
|
||||
// TestPosModeTiebreakWithinClass — AC1 tiebreak 候选:热度增量有界(< w·1 = 0.05),
|
||||
// 同档内保留热度顺序、不能跨档翻案;none 对照 = 热度零影响。
|
||||
// 注:fbVec 只设 v[0],全部文档与查询同向 ⇒ cosine 恒 1 ⇒ 位次靠 tie-break;
|
||||
// 这里用 fbVec2 给文档不同方向(cos(q=e0, (a,b)) = a/√(a²+b²))制造真实向量差。
|
||||
func TestPosModeTiebreakWithinClass(t *testing.T) {
|
||||
fbVec2 := func(v0, v1 float32) []float32 {
|
||||
v := make([]float32, 1024)
|
||||
v[0], v[1] = v0, v1
|
||||
return v
|
||||
}
|
||||
oldBias := feedbackBias
|
||||
feedbackBias = 1.0
|
||||
defer func() { feedbackBias = oldBias }()
|
||||
|
||||
// 同档:warm cos=0.9138 > hot cos=0.8924(差 0.021 = 同相关性档),热度差 389 vs 3
|
||||
// 跨档:low cos=0.4061;填充 p1/p2 cos≈0.32/0.20
|
||||
rows := []models.MemoryRecord{
|
||||
{ID: "hot", Content: "热门同档 高频被召回", Category: "distilled", Source: "llm_distill", Vector: fbVec2(0.89, 0.45), RecallVerifiedCount: 389},
|
||||
{ID: "warm", Content: "微热同档 偶尔被召回", Category: "distilled", Source: "llm_distill", Vector: fbVec2(0.90, 0.40), RecallVerifiedCount: 3},
|
||||
{ID: "low", Content: "低档填充", Category: "distilled", Source: "llm_distill", Vector: fbVec2(0.40, 0.90)},
|
||||
{ID: "p1", Content: "填充记忆 一", Category: "distilled", Source: "llm_distill", Vector: fbVec2(0.30, 0.90)},
|
||||
{ID: "p2", Content: "填充记忆 二", Category: "distilled", Source: "llm_distill", Vector: fbVec2(0.20, 0.98)},
|
||||
}
|
||||
|
||||
// none 对照:同档两条 bias 恒 0 ⇒ 顺序 = 纯向量序(warm 0.90 > hot 0.89)
|
||||
t.Setenv("ZHIYI_POS_MODE", "none")
|
||||
noneRes := fbRun(t, rows)
|
||||
if noneRes["warm"][0] > noneRes["hot"][0] {
|
||||
t.Fatalf("none 对照异常:同档无热度差时应按向量序 warm(0.90) 在 hot(0.89) 前,got warm pos=%.0f hot pos=%.0f", noneRes["warm"][0], noneRes["hot"][0])
|
||||
}
|
||||
|
||||
// tiebreak:hot 的热度小增量把它抬到 warm 之上(同档内热度序保留),
|
||||
// 但增量有界(0.05·sat(389,100)≈0.040 < 跨档向量差 0.145)⇒ low 不得翻案进同档上方
|
||||
t.Setenv("ZHIYI_POS_MODE", "tiebreak")
|
||||
tb := fbRun(t, rows)
|
||||
if tb["hot"][0] > tb["warm"][0] {
|
||||
t.Fatalf("AC1 tiebreak 违反方向:同档内热度更高者应占优(hot 389 > warm 3),got hot pos=%.0f warm pos=%.0f", tb["hot"][0], tb["warm"][0])
|
||||
}
|
||||
if tb["low"][0] < tb["warm"][0] || tb["low"][0] < tb["hot"][0] {
|
||||
t.Fatalf("AC1 tiebreak 违反有界性:热度小增量(<0.05)不得把低档 low(0.75) 翻案进同档(0.89/0.90) 上方:low pos=%.0f", tb["low"][0])
|
||||
}
|
||||
// 增量有界的数值核验(对 FeedbackScore 直接断言,不经 rerank)
|
||||
t.Setenv("ZHIYI_POS_MODE", "tiebreak")
|
||||
hotRow := &rows[0]
|
||||
warmRow := &rows[1]
|
||||
delta := FeedbackScore(hotRow) - FeedbackScore(warmRow) // 含 bias 系数 1.0
|
||||
if delta <= 0 {
|
||||
t.Fatalf("tiebreak 同档热度差应产生正增量:hot−warm=%.6f", delta)
|
||||
}
|
||||
if delta > 0.05 {
|
||||
t.Fatalf("AC1 tiebreak 违反有界性:增量 %.4f 超过 w=0.05 上限(大-K 饱和设计目标)", delta)
|
||||
}
|
||||
t.Logf("AC1 tiebreak: 同档增量 hot−warm=%.4f (∈(0, 0.05],有界);none 对照=纯向量序;low 未翻案 ✓", delta)
|
||||
}
|
||||
|
||||
// TestPosModeNegUnchanged — AC3 判据③(反向对照)+ AC4:
|
||||
// 负向项(human_not_useful)在四种模式下逐位不变(修复只动 pos,不碰 neg 方向);
|
||||
// 且任何模式都不读 deprecated NotUsefulCount。
|
||||
func TestPosModeNegUnchanged(t *testing.T) {
|
||||
oldBias := feedbackBias
|
||||
feedbackBias = 1.0
|
||||
defer func() { feedbackBias = oldBias }()
|
||||
|
||||
rows := []models.MemoryRecord{
|
||||
{ID: "bad", Content: "被否定记忆 内容", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.90), HumanNotUsefulCount: 2, NotUsefulCount: 7},
|
||||
{ID: "clean", Content: "干净记忆 内容", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.88)},
|
||||
{ID: "p0", Content: "填充记忆 零", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.60)},
|
||||
{ID: "p1", Content: "填充记忆 一", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.55)},
|
||||
{ID: "p2", Content: "填充记忆 二", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.50)},
|
||||
}
|
||||
wantNeg := float64(0) - 2.0/5.0 // -sat(2,3) = -0.4,四态同值
|
||||
for _, mode := range []string{"", "none", "weight", "tiebreak"} {
|
||||
t.Setenv("ZHIYI_POS_MODE", mode)
|
||||
got := fbRun(t, rows)
|
||||
if cleanPos, badPos := got["clean"][0], got["bad"][0]; cleanPos > badPos {
|
||||
t.Fatalf("AC3③ 违反:neg 方向在模式 %q 下失效 —— clean(pos=%.0f) 未压住 bad(pos=%.0f)", mode, cleanPos, badPos)
|
||||
}
|
||||
_ = wantNeg
|
||||
}
|
||||
// 数值层:bad 行(verified=0,只有 neg)四态 FeedbackScore 必须逐位 == -0.4
|
||||
for _, mode := range []string{"", "none", "weight", "tiebreak"} {
|
||||
t.Setenv("ZHIYI_POS_MODE", mode)
|
||||
badRow := &rows[0]
|
||||
if got := FeedbackScore(badRow); math.Abs(got-wantNeg) > 1e-12 {
|
||||
t.Fatalf("AC3③/AC4 违反:模式 %q 下 neg 项不恒定:bad score=%.10f want=%.10f(含 NotUsefulCount=7 干扰项仍必须 == -0.4)", mode, got, wantNeg)
|
||||
}
|
||||
}
|
||||
t.Logf("AC3③ 反向对照: bad(human_not_useful=2, NotUsefulCount=7 干扰) 四态 score 恒 -0.4、位次恒被 clean 压住 ✓")
|
||||
}
|
||||
@@ -1,11 +1,50 @@
|
||||
package storage
|
||||
|
||||
import "github.com/xiaoxue/memoryweave/internal/models"
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/xiaoxue/memoryweave/internal/models"
|
||||
)
|
||||
|
||||
// feedbackSaturation 是饱和常数 K:sat(x)=x/(x+K)。
|
||||
// 计数无上界(可被反复标记),必须压进 [0,1) 才能当加性偏置用。
|
||||
var feedbackSaturation = envFloat("ZHIYI_FEEDBACK_K", 3)
|
||||
|
||||
// ── t_23b6a940:热度信号 recall_verified_count 如何进入正向质量项 ──
|
||||
//
|
||||
// 金标 119 条 A/B 实测(本卡卡体):legacy(verified 直入 pos)下 7413 条非零
|
||||
// verified(max=389、中位 3)拿到近满分偏置(最大 +0.9924),热门记忆硬顶 rank1、
|
||||
// 挤掉真正相关的 ⇒ MRR -22.2% / NDCG -17.7%(「热门≠正确」,与 t_34ec2b1f
|
||||
// 「新鲜≠正确」同一族病:单一易得信号冒充质量)。
|
||||
//
|
||||
// 修法 = 把「热度」从质量项里拆出去,env ZHIYI_POS_MODE 切换(默认 legacy =
|
||||
// 与当前生产逐位一致;回滚 = 置回 legacy 或 ZHIYI_FEEDBACK_BIAS=0,零代码变更):
|
||||
//
|
||||
// legacy = 现状(verified 1:1 进 pos,金标实测 -22%,仅作对照臂)
|
||||
// none = 移除 verified,pos = useful_count(热度不是质量;MoA 复核行动①)
|
||||
// weight = 独立小权重:pos = useful + w·verified,w=ZHIYI_POS_WEIGHT(默认 0.05)
|
||||
// tiebreak = 纯 tiebreak(MoA 复核行动②):质量项只算 useful;verified 走独立
|
||||
// 大 K(posTiebreakK=100) 饱和 × 小权重 w ⇒ 中位 v=3 ⇒ +0.0015、
|
||||
// v=389 ⇒ +0.040,只在同相关性档内做有界增量、不能跨档
|
||||
//
|
||||
// 负向项(human_not_useful + dedup_downweight)与模式无关、恒定性,
|
||||
// 各模式间逐位不变(反向对照判据)。
|
||||
func posMode() string {
|
||||
switch strings.ToLower(os.Getenv("ZHIYI_POS_MODE")) {
|
||||
case "none", "weight", "tiebreak", "legacy":
|
||||
return strings.ToLower(os.Getenv("ZHIYI_POS_MODE"))
|
||||
default:
|
||||
return "legacy" // 默认 = 与当前生产逐位一致
|
||||
}
|
||||
}
|
||||
|
||||
// posWeight 热度项权重(仅 weight/tiebreak 模式生效)。每次调用读 env,测试可 t.Setenv 翻转。
|
||||
func posWeight() float64 { return envFloat("ZHIYI_POS_WEIGHT", 0.05) }
|
||||
|
||||
// posTiebreakK tiebreak 档独立饱和常数:verified=389 ⇒ sat≈0.80,×w(默认0.05) ⇒ 增量上限 +0.040。
|
||||
const posTiebreakK = 100.0
|
||||
|
||||
// FeedbackScore 把 per-memory 反馈计数压成有界双向分 ∈ (-1, 1)。
|
||||
//
|
||||
// 设计(2026-10-13):排序此前只读 V 值(VValueProvider),而 V 值轴被 ep_*
|
||||
@@ -13,8 +52,13 @@ var feedbackSaturation = envFloat("ZHIYI_FEEDBACK_K", 3)
|
||||
// 本函数改读 DB 计数(干净、per-memory、双向),由 recall.go 以 env 门控
|
||||
// 加性偏置的形式接入排序(ZHIYI_FEEDBACK_BIAS,默认 0 = 关闭)。
|
||||
//
|
||||
// 2026-10-18 (t_23b6a940):正向项按 ZHIYI_POS_MODE 分四态(见上方注释);
|
||||
// legacy 态与 2026-10-13 原始实现**逐位一致**(verified 1:1 进 pos)——
|
||||
// 该态已被金标 A/B 证明把「热度」当「质量」(MRR -22%),保留只为对照臂
|
||||
// 与零变更回滚基线。
|
||||
//
|
||||
// 正向 = 被真实使用/认可:useful_count(人工点有用)+ recall_verified_count
|
||||
// (有正反馈或下游引用,models/memory.go:69 定义的唯一正信号)。
|
||||
// (legacy 态 1:1 计入;none/weight/tiebreak 态按 ZHIYI_POS_MODE 处理)。
|
||||
// 负向 = 被真实否定/取代:human_not_useful_count(人工点没用)+
|
||||
// dedup_downweight_count(去重降权)。
|
||||
// 注意 not_useful_count 是 deprecated 混写列(GA-P0-1 已拆),本函数不读它。
|
||||
@@ -22,13 +66,25 @@ func FeedbackScore(m *models.MemoryRecord) float64 {
|
||||
if m == nil {
|
||||
return 0
|
||||
}
|
||||
pos := float64(m.UsefulCount + m.RecallVerifiedCount)
|
||||
useful := float64(m.UsefulCount)
|
||||
verified := float64(m.RecallVerifiedCount)
|
||||
neg := float64(m.HumanNotUsefulCount + m.DedupDownweightCount)
|
||||
sat := func(x float64) float64 {
|
||||
sat := func(x, k float64) float64 {
|
||||
if x <= 0 {
|
||||
return 0
|
||||
}
|
||||
return x / (x + feedbackSaturation)
|
||||
return x / (x + k)
|
||||
}
|
||||
switch posMode() {
|
||||
case "none":
|
||||
return sat(useful, feedbackSaturation) - sat(neg, feedbackSaturation)
|
||||
case "weight":
|
||||
return sat(useful+posWeight()*verified, feedbackSaturation) - sat(neg, feedbackSaturation)
|
||||
case "tiebreak":
|
||||
return sat(useful, feedbackSaturation) - sat(neg, feedbackSaturation) +
|
||||
posWeight()*sat(verified, posTiebreakK)
|
||||
default: // legacy — 与 2026-10-13 实现逐位一致(pos = useful+verified)
|
||||
pos := useful + verified
|
||||
return sat(pos, feedbackSaturation) - sat(neg, feedbackSaturation)
|
||||
}
|
||||
return sat(pos) - sat(neg)
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package storage
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// importance 规则(t_789898bd,2026-10-18)——「装温度计」
|
||||
//
|
||||
// 背景:写入侧 importance 恒硬编码 1.0(lancedb_ipc.go:159 episode / :200 memory),
|
||||
// 全库 46,903 行恒 1.0 ⇒ 排序侧没有非恒定信号,任何排序修复都测不了 NDCG 提升
|
||||
// (「修 importance = 先装温度计,才知道退烧药有没有用」)。本文件给写入侧接上
|
||||
// 规则化 importance + env 回滚开关。
|
||||
//
|
||||
// 标定依据(2026-10-18 21:00 全库只读统计,/var/lib/memoryweave memories 46,903 行,
|
||||
// 原始输出见交付 importance-fix-t_789898bd.md 的 stats_readonly 一节):
|
||||
// tier: core=1,926 (4.11%) / normal=44,977 (95.89%)
|
||||
// source: wiki=26,664 (56.85%) / llm_distill=6,197 (13.21%) /
|
||||
// legacy=12,099 (25.80%) / recovery=1,926 (4.11%,全部 tier=core) /
|
||||
// other=17 (0.04%)
|
||||
// ⇒ 回填后分布恰好 3 档,每档 ≥1%(AC2 验收判据),且无任何值等于 1.0
|
||||
// (「恒 1.0 行数 → 0」机械可判):
|
||||
// 0.9 = core(4.11%)
|
||||
// 0.7 = normal × wiki:*/llm_distill(70.06%)
|
||||
// 0.4 = normal × legacy:*/unknown(25.84%,unknown=17 行并入低置信档,
|
||||
// 0.04% 单拆会产生 <1% 档 ⇒ 违反 AC2,故并档)
|
||||
// 0.3 = 预留档:episode 原始日志 / 非 core 的 recovery 行(存量 0 行,
|
||||
// 只对未来写入生效,不参与存量直方图「每档 ≥1%」验收)
|
||||
// 未纳入的微调项(每项实测依据):
|
||||
// · 内容深度:长度 p50=32 字、78.1% <50 字 ⇒ 按深度切档必出 <1% 档
|
||||
// · 时间衰减:排序已有运行时 recency 项(quality_fusion.computeImportance),
|
||||
// 存进列会双衰减,故存量值取「静态信号」
|
||||
// · 图谱连接度:需要 graph.db,本卡红线「不碰图谱」
|
||||
//
|
||||
// env 开关(AC5 回滚开关,默认 off = 零行为变化):
|
||||
// ZHIYI_IMPORTANCE_RULE=on ⇒ 未显式传 importance 时走 ComputeImportance / EpisodeImportance
|
||||
// ZHIYI_IMPORTANCE_RULE=off / 未设 / 其他值 ⇒ 写入回落恒 1.0(与改动前逐位一致)
|
||||
// 每次调用读 env(测试可 t.Setenv 翻转,同 t_7c0538eb lowScoreDwGate 先例)。
|
||||
// 本卡**不部署生产二进制**(部署另行拍板);存量回填用 rust 侧
|
||||
// importance_backfill 工具(同 lancedb 0.15 crate,仅写 importance 一列,组更新 3 个版本)。
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/xiaoxue/memoryweave/internal/models"
|
||||
)
|
||||
|
||||
// legacyImportanceConstant 改动前恒 1.0 的字面量(规则关 / 调用方未传显式值时的回落)。
|
||||
const legacyImportanceConstant = 1.0
|
||||
|
||||
// EpisodeImportance episode(原始日志)写入档:无蒸馏信号,预留 0.3 档。
|
||||
const EpisodeImportance = 0.3
|
||||
|
||||
// importanceRuleEnabled env 开关:on/true/1 = 开;其余(含未设)= 关。
|
||||
func importanceRuleEnabled() bool {
|
||||
switch strings.ToLower(os.Getenv("ZHIYI_IMPORTANCE_RULE")) {
|
||||
case "on", "true", "1":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// SourceClass 把 source 归入分档类。未知/空 source ⇒ "other"(按低置信类处理,
|
||||
// 实测 0.04%,不独立成档 —— 拆档会产生 <1% 档,违反 AC2 验收)。
|
||||
func SourceClass(source string) string {
|
||||
if source == "llm_distill" {
|
||||
return "llm_distill"
|
||||
}
|
||||
switch {
|
||||
case strings.HasPrefix(source, "wiki:"):
|
||||
return "wiki"
|
||||
case strings.HasPrefix(source, "legacy:"):
|
||||
return "legacy"
|
||||
case strings.HasPrefix(source, "recovery:"):
|
||||
return "recovery"
|
||||
default:
|
||||
return "other"
|
||||
}
|
||||
}
|
||||
|
||||
// ComputeImportance 由 (tier, source 类) 算出档位 importance。
|
||||
// 返回值 ∈ {0.9, 0.7, 0.4, 0.3},**永不返回 1.0**(「恒 1.0 行数→0」机械可判的前提)。
|
||||
func ComputeImportance(rec models.MemoryRecord) float64 {
|
||||
if rec.Tier == "core" {
|
||||
return 0.9 // 覆盖 1,926 行 = 4.11%
|
||||
}
|
||||
switch SourceClass(rec.Source) {
|
||||
case "wiki", "llm_distill":
|
||||
return 0.7 // 覆盖 32,861 行 = 70.06%
|
||||
case "recovery":
|
||||
return EpisodeImportance // 非 core 的 recovery(存量 0 行,预留档)
|
||||
default: // legacy / other / 空
|
||||
return 0.4 // 覆盖 12,116 行 = 25.84%
|
||||
}
|
||||
}
|
||||
|
||||
// EffectiveMemoryImportance 写入侧(InsertMemory :200)应写库的实际值(AC1 接线点):
|
||||
// 1. m.Importance != 0 ⇒ 显式值优先(调用方声明即权威,AC1.1)
|
||||
// 2. ZHIYI_IMPORTANCE_RULE=on ⇒ ComputeImportance(AC2 规则)
|
||||
// 3. 否则 ⇒ 回落 1.0(保护存量调用方,零行为突变,AC1.2)
|
||||
func EffectiveMemoryImportance(m models.MemoryRecord) float64 {
|
||||
if m.Importance != 0 {
|
||||
return m.Importance
|
||||
}
|
||||
if importanceRuleEnabled() {
|
||||
return ComputeImportance(m)
|
||||
}
|
||||
return legacyImportanceConstant
|
||||
}
|
||||
|
||||
// EffectiveEpisodeImportance episode 写入侧(InsertEpisode :159)应写库的实际值。
|
||||
// InsertEpisode 接口签名(6 个实现 + 4 个调用点)不带 importance 参数,改签名超出
|
||||
// 本卡范围;episode 原始日志也没有 tier/source 分档信号 ⇒ 规则档恒 0.3。
|
||||
// 规则 off(默认)⇒ 回落 1.0,与改动前逐位一致。
|
||||
func EffectiveEpisodeImportance() float64 {
|
||||
if importanceRuleEnabled() {
|
||||
return EpisodeImportance
|
||||
}
|
||||
return legacyImportanceConstant
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/xiaoxue/memoryweave/internal/models"
|
||||
)
|
||||
|
||||
// t_789898bd AC2:importance 规则单测(档位来自全库实测标定,见 importance_rule.go 头注释)。
|
||||
|
||||
func TestComputeImportanceTiers(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
rec models.MemoryRecord
|
||||
want float64
|
||||
}{
|
||||
{"core 恒 0.9(source 无关)", models.MemoryRecord{Tier: "core", Source: "recovery:P0-shadow-log"}, 0.9},
|
||||
{"normal × wiki:*", models.MemoryRecord{Tier: "normal", Source: "wiki:worklog-note"}, 0.7},
|
||||
{"normal × llm_distill", models.MemoryRecord{Tier: "normal", Source: "llm_distill"}, 0.7},
|
||||
{"normal × legacy:*", models.MemoryRecord{Tier: "normal", Source: "legacy:pre0926"}, 0.4},
|
||||
{"normal × 未知 source", models.MemoryRecord{Tier: "normal", Source: "whatever"}, 0.4},
|
||||
{"normal × 空 source", models.MemoryRecord{Tier: "normal"}, 0.4},
|
||||
{"normal × recovery:*(预留 0.3 档)", models.MemoryRecord{Tier: "normal", Source: "recovery:X"}, 0.3},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := ComputeImportance(c.rec); got != c.want {
|
||||
t.Errorf("%s: ComputeImportance=%v want %v", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeImportanceNeverOne(t *testing.T) {
|
||||
// AC3 验收「恒 1.0 行数 → 0」依赖:规则输出空间不含 1.0。
|
||||
for _, tier := range []string{"core", "normal", "episode", ""} {
|
||||
for _, src := range []string{"wiki:a", "llm_distill", "legacy:b", "recovery:c", "other", ""} {
|
||||
if got := ComputeImportance(models.MemoryRecord{Tier: tier, Source: src}); got == legacyImportanceConstant {
|
||||
t.Fatalf("tier=%q source=%q 规则输出 == 1.0,违反「永不返回 1.0」", tier, src)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveMemoryImportance(t *testing.T) {
|
||||
// 默认(规则关):显式值优先;未传 ⇒ 恒 1.0(改动前行为)
|
||||
t.Setenv("ZHIYI_IMPORTANCE_RULE", "off")
|
||||
if got := EffectiveMemoryImportance(models.MemoryRecord{Importance: 0.8}); got != 0.8 {
|
||||
t.Errorf("显式值 0.8 应直通,got %v", got)
|
||||
}
|
||||
if got := EffectiveMemoryImportance(models.MemoryRecord{Tier: "core"}); got != 1.0 {
|
||||
t.Errorf("规则关 + 未传值应回落 1.0,got %v", got)
|
||||
}
|
||||
// 规则开:未传值 ⇒ 走规则档
|
||||
t.Setenv("ZHIYI_IMPORTANCE_RULE", "on")
|
||||
if got := EffectiveMemoryImportance(models.MemoryRecord{Tier: "core"}); got != 0.9 {
|
||||
t.Errorf("规则开 + core 应 0.9,got %v", got)
|
||||
}
|
||||
if got := EffectiveMemoryImportance(models.MemoryRecord{Tier: "normal", Source: "legacy:pre0926"}); got != 0.4 {
|
||||
t.Errorf("规则开 + normal/legacy 应 0.4,got %v", got)
|
||||
}
|
||||
// 显式值在规则开时仍优先
|
||||
if got := EffectiveMemoryImportance(models.MemoryRecord{Tier: "core", Importance: 0.5}); got != 0.5 {
|
||||
t.Errorf("显式值 0.5 应优先于规则,got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveEpisodeImportance(t *testing.T) {
|
||||
t.Setenv("ZHIYI_IMPORTANCE_RULE", "off")
|
||||
if got := EffectiveEpisodeImportance(); got != 1.0 {
|
||||
t.Errorf("规则关 episode 应回落 1.0(改动前行为),got %v", got)
|
||||
}
|
||||
t.Setenv("ZHIYI_IMPORTANCE_RULE", "on")
|
||||
if got := EffectiveEpisodeImportance(); got != EpisodeImportance {
|
||||
t.Errorf("规则开 episode 应 0.3,got %v", got)
|
||||
}
|
||||
// 未设 env ⇒ 关(测试环境无该变量时走 unset 分支)
|
||||
t.Setenv("ZHIYI_IMPORTANCE_RULE", "")
|
||||
if got := EffectiveEpisodeImportance(); got != 1.0 {
|
||||
t.Errorf("env 未设 episode 应回落 1.0,got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceClass(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"llm_distill": "llm_distill",
|
||||
"wiki:worklog-note": "wiki",
|
||||
"legacy:pre0926": "legacy",
|
||||
"recovery:P0-shadow-log": "recovery",
|
||||
"episode": "other",
|
||||
"": "other",
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := SourceClass(in); got != want {
|
||||
t.Errorf("SourceClass(%q)=%q want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -156,7 +156,7 @@ func (rc *RustLanceDBClient) InsertEpisode(agentID, namespace, content, category
|
||||
record := map[string]interface{}{
|
||||
"id": id, "agent_id": agentID, "namespace": namespace,
|
||||
"content": content, "category": category,
|
||||
"tier": "episode", "importance": 1.0, "quality_score": 0.0,
|
||||
"tier": "episode", "importance": EffectiveEpisodeImportance(), "quality_score": 0.0,
|
||||
"recall_count": 0, "useful_count": 0, "not_useful_count": 0,
|
||||
// GA-P0-2 (t_c6d41ac6):拆分字段(episode 无召回/反馈,恒 0)
|
||||
"recall_seen_count": 0, "recall_verified_count": 0,
|
||||
@@ -197,7 +197,7 @@ func (rc *RustLanceDBClient) InsertMemory(m models.MemoryRecord) error {
|
||||
record := map[string]interface{}{
|
||||
"id": m.ID, "agent_id": m.AgentID, "namespace": m.Namespace,
|
||||
"content": m.Content, "category": m.Category, "vector": m.Vector,
|
||||
"tier": m.Tier, "importance": 1.0, "quality_score": m.QualityScore,
|
||||
"tier": m.Tier, "importance": EffectiveMemoryImportance(m), "quality_score": m.QualityScore,
|
||||
"recall_count": m.RecallCount, "useful_count": 0, "not_useful_count": 0,
|
||||
// GA-P0-2 (t_c6d41ac6):拆分字段(新记忆按 model 携带,通常 0)
|
||||
"recall_seen_count": m.RecallSeenCount, "recall_verified_count": m.RecallVerifiedCount,
|
||||
@@ -228,12 +228,16 @@ func (rc *RustLanceDBClient) InsertMemory(m models.MemoryRecord) error {
|
||||
return err
|
||||
}
|
||||
|
||||
func (rc *RustLanceDBClient) Search(table string, vector []float32, topK int, namespaceFilter string) ([]models.MemoryRecord, error) {
|
||||
// SearchWithTruncation 同 Search,但额外返回「本次结果是否被 Rust 侧单次重读行数上限
|
||||
// 截断」(ipcResp.Truncated,t_576714de 引入)。t_3aa2a8c1:该标志此前只打日志不上抛
|
||||
// (recall 调用方观测不到「结果被截断」),现透传出去供 /api/v1/recall 暴露。
|
||||
// 本地兜底(IPC 错误 / 零结果走 localSearch)没有行数上限 ⇒ 恒 false。
|
||||
func (rc *RustLanceDBClient) SearchWithTruncation(table string, vector []float32, topK int, namespaceFilter string) ([]models.MemoryRecord, bool, error) {
|
||||
resp, err := rc.rpc(ipcReq{
|
||||
Type: "lancedb_search", Vector: vector, TopK: topK * 2, Namespace: namespaceFilter,
|
||||
})
|
||||
if err != nil {
|
||||
return rc.localSearch(vector, topK, namespaceFilter), nil
|
||||
return rc.localSearch(vector, topK, namespaceFilter), false, nil
|
||||
}
|
||||
|
||||
var raw []struct {
|
||||
@@ -251,6 +255,15 @@ func (rc *RustLanceDBClient) Search(table string, vector []float32, topK int, na
|
||||
// GA-P0-2 (t_c6d41ac6):拆分字段读侧接线
|
||||
RecallSeenCount int64 `json:"recall_seen_count"`
|
||||
RecallVerifiedCount int64 `json:"recall_verified_count"`
|
||||
// t_4c810e02(反馈回路打通卡):反馈计数三列读侧接线。
|
||||
// Rust Search 已回传(serde 全字段),此前本解包结构缺这三行 ⇒ 候选的
|
||||
// UsefulCount/HumanNotUsefulCount/DedupDownweightCount 恒 0 ⇒
|
||||
// FeedbackScore≡0 ⇒ recall.go 的 feedbackBias 循环整体空转(「出端断链」,
|
||||
// 与 2026-09-26 Source / 2026-10-08 Confidence 同族:加了列、改了公式,
|
||||
// 但 IPC 解包不接 ⇒ 静默 no-op,只有端到端活证能抓到)。
|
||||
UsefulCount int64 `json:"useful_count"`
|
||||
HumanNotUsefulCount int64 `json:"human_not_useful_count"`
|
||||
DedupDownweightCount int64 `json:"dedup_downweight_count"`
|
||||
VolatileFlag bool `json:"volatile_flag"`
|
||||
Tier string `json:"tier"`
|
||||
LastRecalledAt string `json:"last_recalled_at"`
|
||||
@@ -279,7 +292,7 @@ func (rc *RustLanceDBClient) Search(table string, vector []float32, topK int, na
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
log.Printf("[ipc] Search: resp.ReportJSON unmarshal gave 0 results, falling back to localSearch (Result=%q)", resp.Result)
|
||||
return rc.localSearch(vector, topK, namespaceFilter), nil
|
||||
return rc.localSearch(vector, topK, namespaceFilter), false, nil
|
||||
}
|
||||
log.Printf("[ipc] Search: got %d results from Rust IPC (Result=%q)", len(raw), resp.Result)
|
||||
|
||||
@@ -337,6 +350,10 @@ func (rc *RustLanceDBClient) Search(table string, vector []float32, topK int, na
|
||||
// GA-P0-2 (t_c6d41ac6):拆分字段读侧接线
|
||||
RecallSeenCount: int(r.RecallSeenCount),
|
||||
RecallVerifiedCount: int(r.RecallVerifiedCount),
|
||||
// t_4c810e02:反馈计数输出映射(缺 ⇒ FeedbackScore 读 0 ⇒ 反馈不进排序)
|
||||
UsefulCount: int(r.UsefulCount),
|
||||
HumanNotUsefulCount: int(r.HumanNotUsefulCount),
|
||||
DedupDownweightCount: int(r.DedupDownweightCount),
|
||||
Source: r.Source, // 2026-09-26:出处透出(缺此行 → recall.source 恒空)
|
||||
Vector: r.Vector, // 去重判据依赖,勿删
|
||||
// t_631ab13d:DB 原值直通 —— RecallResult 展示用它,排序用 QualityScore(融合值)
|
||||
@@ -355,7 +372,13 @@ func (rc *RustLanceDBClient) Search(table string, vector []float32, topK int, na
|
||||
break
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
return out, resp.Truncated, nil
|
||||
}
|
||||
|
||||
// Search 旧签名(不报告截断)。截断标志的透传走 SearchWithTruncation(t_3aa2a8c1)。
|
||||
func (rc *RustLanceDBClient) Search(table string, vector []float32, topK int, namespaceFilter string) ([]models.MemoryRecord, error) {
|
||||
recs, _, err := rc.SearchWithTruncation(table, vector, topK, namespaceFilter)
|
||||
return recs, err
|
||||
}
|
||||
|
||||
// Scan 全表扫描存活记忆(走 Rust IPC `lancedb_scan`:is_deleted=false + namespace 下推)。
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// t_23b6a940 AC7:热门度偏置的运行时看门狗(信号安全阈值兜底)。
|
||||
//
|
||||
// 背景:FeedbackScore legacy 态把 recall_verified_count(热度)当质量信号 ⇒
|
||||
// 热门记忆硬顶 rank1(金标 119 条实测 MRR -22%)。离线 A/B 暴露问题的间隔
|
||||
// 长(本次隔了一整晚)。本看门狗是**运行时兜底**:最终 rank1 记忆的
|
||||
// recall_verified_count 连续 N 次超过全库 p95 阈值 ⇒ 打一条 ALERT 日志行
|
||||
// (传感器通道),传感器端脚本(~/.hermes/scripts/tools/zhiyi-popwatch.py,
|
||||
// 挂既有告警通道)把快照转成告警。
|
||||
//
|
||||
// 默认只报不改(report-only):
|
||||
// - 不改排序、不写库、不发告警(告警由传感器端既有通道发);
|
||||
// - 默认**开**(零排序影响、每 recall 一次 env 读 + 临界一次锁,开销可忽略),
|
||||
// ZHIYI_POPWATCH=0 可完全关闭(关闭 ⇒ PopWatchRecord 立即返回,零行为变化)。
|
||||
//
|
||||
// 阈值按全库实测标定(47148 条,2026-10-18 复测):recall_verified_count
|
||||
// 全库 p95=4、非零 p99=62(≈全库 top 1% 热层)、max=389、非零 7416 条:
|
||||
//
|
||||
// ZHIYI_POPWATCH_P95 = 62(触发阈值,rank1 verified 严格大于它才累计;
|
||||
// = 非零 p99,MoA 复核标定值)
|
||||
// ZHIYI_POPWATCH_N = 5 (连续多少个 query 触发 ALERT)
|
||||
func PopWatchEnabled() bool {
|
||||
switch strings.ToLower(os.Getenv("ZHIYI_POPWATCH")) {
|
||||
case "0", "false", "off":
|
||||
return false
|
||||
default:
|
||||
return true // 默认开(report-only)
|
||||
}
|
||||
}
|
||||
|
||||
func popWatchThreshold() int64 { return int64(envInt("ZHIYI_POPWATCH_P95", 62)) }
|
||||
func popWatchN() int { return envInt("ZHIYI_POPWATCH_N", 5) }
|
||||
|
||||
// popWatchState 进程内观测状态(每实例一份;「连续 N 个 query」语义就是进程内的)。
|
||||
type popWatchState struct {
|
||||
mu sync.Mutex
|
||||
consecutive int // rank1 verified > 阈值 的连续 query 数(否则清零)
|
||||
lastRank1ID string // 最近一次 recall 的 rank1 记忆 id(审计用)
|
||||
lastVerified int64 // 最近一次 recall 的 rank1 verified
|
||||
lastSeenAt time.Time
|
||||
alertsFired int // 本进程累计 ALERT 次数
|
||||
lastAlertAt time.Time
|
||||
}
|
||||
|
||||
// PopWatch 是全局观测器(GET /api/v1/popwatch 读快照;传感器端据此告警)。
|
||||
var PopWatch = &popWatchState{}
|
||||
|
||||
// PopWatchRecord 观测一次 recall 的最终 rank1 的 recall_verified_count。
|
||||
// 调用点:recallUncached 末尾(主路径漏斗——热度偏置恰好作用于此处)。
|
||||
// 门关(ZHIYI_POPWATCH=0)⇒ 立即返回,零行为变化。
|
||||
// 越阈连续满 N 个 query ⇒ 打一条 ALERT 日志行(同一连续段只在满 N 瞬间打一次;
|
||||
// 之后继续越阈不刷屏,回落再爬升才再打)。
|
||||
func PopWatchRecord(rank1ID string, rank1Verified int64) {
|
||||
if !PopWatchEnabled() {
|
||||
return
|
||||
}
|
||||
threshold := popWatchThreshold()
|
||||
n := popWatchN()
|
||||
PopWatch.mu.Lock()
|
||||
defer PopWatch.mu.Unlock()
|
||||
if rank1Verified > threshold {
|
||||
PopWatch.consecutive++
|
||||
} else {
|
||||
PopWatch.consecutive = 0
|
||||
}
|
||||
PopWatch.lastRank1ID = rank1ID
|
||||
PopWatch.lastVerified = rank1Verified
|
||||
PopWatch.lastSeenAt = time.Now()
|
||||
if PopWatch.consecutive == n {
|
||||
PopWatch.alertsFired++
|
||||
PopWatch.lastAlertAt = time.Now()
|
||||
log.Printf("[popwatch] ALERT: rank1 recall_verified_count=%d > p95 阈值 %d 连续 %d 个 query(report-only:不改排序/不写库;回滚:ZHIYI_POS_MODE=legacy 或 ZHIYI_FEEDBACK_BIAS=0)",
|
||||
rank1Verified, threshold, n)
|
||||
}
|
||||
}
|
||||
|
||||
// PopWatchSnapshot 供 GET /api/v1/popwatch(传感器端脚本读它转既有告警通道)。
|
||||
func PopWatchSnapshot() map[string]any {
|
||||
threshold := popWatchThreshold()
|
||||
n := popWatchN()
|
||||
PopWatch.mu.Lock()
|
||||
defer PopWatch.mu.Unlock()
|
||||
lastSeen := ""
|
||||
if !PopWatch.lastSeenAt.IsZero() {
|
||||
lastSeen = PopWatch.lastSeenAt.Format(time.RFC3339)
|
||||
}
|
||||
lastAlert := ""
|
||||
if !PopWatch.lastAlertAt.IsZero() {
|
||||
lastAlert = PopWatch.lastAlertAt.Format(time.RFC3339)
|
||||
}
|
||||
return map[string]any{
|
||||
"enabled": PopWatchEnabled(),
|
||||
"threshold_p95": threshold,
|
||||
"consecutive_n": n,
|
||||
"consecutive": PopWatch.consecutive,
|
||||
"last_rank1_id": PopWatch.lastRank1ID,
|
||||
"last_rank1_verified": PopWatch.lastVerified,
|
||||
"last_seen_at": lastSeen,
|
||||
"alerts_fired": PopWatch.alertsFired,
|
||||
"last_alert_at": lastAlert,
|
||||
"note": "report-only 观测:rank1 recall_verified_count 连续越 p95 阈值 = 热门度偏置行为证据;回滚 ZHIYI_POS_MODE=legacy / ZHIYI_FEEDBACK_BIAS=0",
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package storage
|
||||
import (
|
||||
"math"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/xiaoxue/memoryweave/internal/models"
|
||||
)
|
||||
@@ -19,6 +20,47 @@ import (
|
||||
// 不新增缓存/Redis 依赖;部署走既有「停→构建→起」流程(本卡仅 sidecar A/B,不动生产)。
|
||||
var vFuseUseV = os.Getenv("ZHIYI_V_FUSE_ENABLE") != "1"
|
||||
|
||||
// t_7c0538eb(2026-10-18):低分(可信评分)记忆在检索排序里降权(可逆,替代「真丢弃」)。
|
||||
// ZHIYI_LOWSCORE_DOWNWEIGHT 取值语义(回滚开关,默认关 = 零行为变化):
|
||||
// - 未设 / "0" / 非 "1":lowScoreDownweight 恒返回 1.0 ⇒ fuseQuality 逐位等价现状(零风险);
|
||||
// - "1":对「可靠打分且低分」(0<qs<lowScoreDWThreshold, IsReliableQuality=true) 的记忆,
|
||||
// 把其 quality 贡献按比例 q/阈值 压到 [lowScoreDWFloor, 1.0)——低分记忆排得更低,**不删数据**。
|
||||
// qs<=0(未打分 / bug 残留)与 qs≈0.755(echo 常量)IsReliableQuality=false,本就走
|
||||
// 「仅 importance」分支,不受本降权影响(降权只作用于「可信低分」那一档)。
|
||||
// 每次调用读 env(测试可 t.Setenv 翻转,同 timeBiasTrustGate 模式);回滚 = 去掉/置 0 env 重启。
|
||||
func lowScoreDwGate() bool {
|
||||
switch strings.ToLower(os.Getenv("ZHIYI_LOWSCORE_DOWNWEIGHT")) {
|
||||
case "1", "true", "on":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// lowScoreDWThreshold 与 GA-P1 影子 drop 阈值(ga-p1-threshold-replay.py DROP_T=0.30)对齐。
|
||||
const (
|
||||
lowScoreDWThreshold = 0.30
|
||||
lowScoreDWFloor = 0.5 // 最低的可信低分仍保留 50% 质量贡献(不彻底清零,可逆)
|
||||
)
|
||||
|
||||
// lowScoreDownweight 返回质量贡献缩放因子 f ∈ [lowScoreDWFloor, 1.0]。
|
||||
// gate 关 / 不可信(0.0、0.755) / qs>=阈值 ⇒ 1.0(不降权,逐位等价现状)。
|
||||
func lowScoreDownweight(rawQuality float64) float64 {
|
||||
if !lowScoreDwGate() {
|
||||
return 1.0
|
||||
}
|
||||
if rawQuality <= 0 || !models.IsReliableQuality(rawQuality) {
|
||||
return 1.0
|
||||
}
|
||||
if rawQuality >= lowScoreDWThreshold {
|
||||
return 1.0
|
||||
}
|
||||
f := rawQuality / lowScoreDWThreshold
|
||||
if f < lowScoreDWFloor {
|
||||
f = lowScoreDWFloor
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// 召回排序「频率 + 时间衰减」融合公式 —— 从 lancedb_ipc.go Search() 内联代码抽出的纯函数。
|
||||
// 行为逐位等价于原内联实现(t_5028c14d),不改任何数值常量。
|
||||
// 调用链:recall.go Recall() → Search() → 本文件三函数 → QualityScore 供排序。
|
||||
@@ -52,10 +94,13 @@ func computeImportance(days float64, recallSeenCount int64, volatile bool) float
|
||||
// rawQuality 经 models.IsReliableQuality 判定可信度(0=未打分/历史常量 bug 视为不可信)。
|
||||
func fuseQuality(computedImportance, rawQuality, vValue float64) float64 {
|
||||
qsTrusted := models.IsReliableQuality(rawQuality)
|
||||
// t_7c0538eb:低分(可信)降权因子;gate 关时恒 1.0(逐位等价现状,零行为变化)。
|
||||
dw := lowScoreDownweight(rawQuality)
|
||||
var finalQuality float64
|
||||
if qsTrusted {
|
||||
// QualityScore from LLM (0-1) + computed importance, capped at 1.0
|
||||
finalQuality = math.Min(0.4*computedImportance+0.6*rawQuality, 1.0)
|
||||
// t_7c0538eb:quality 贡献 × dw(可信低分按比例压低、可逆;不删数据)
|
||||
finalQuality = math.Min(0.4*computedImportance+0.6*rawQuality*dw, 1.0)
|
||||
} else {
|
||||
// 质量分不可信 → 仅用 importance([qstrust-ipc] 日志留在调用处)
|
||||
finalQuality = math.Min(computedImportance, 1.0)
|
||||
@@ -65,8 +110,8 @@ func fuseQuality(computedImportance, rawQuality, vValue float64) float64 {
|
||||
// 等价于现状 vValue<=0 行为;0.5:0.5 分支的 V 半边移除后归一回到无 V 式)。
|
||||
if vFuseUseV && vValue > 0 {
|
||||
if qsTrusted {
|
||||
// 30% V值 + 30% importance + 40% LLM quality
|
||||
finalQuality = 0.3*vValue + 0.3*computedImportance + 0.4*rawQuality
|
||||
// 30% V值 + 30% importance + 40% LLM quality(t_7c0538eb:quality 半边 × dw)
|
||||
finalQuality = 0.3*vValue + 0.3*computedImportance + 0.4*rawQuality*dw
|
||||
} else {
|
||||
// 质量分不可信 → 去掉它的 40%,剩余 30:30 重新归一为 50:50。
|
||||
finalQuality = 0.5*vValue + 0.5*computedImportance
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"math"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// t_7c0538eb AC5:低分(可信评分)记忆在检索排序里降权(可逆,不删数据)+ 回滚开关。
|
||||
//
|
||||
// 召回排序的单一融合点 = fuseQuality(lancedb_ipc.go:333 调用;纯函数)。本测试用一组
|
||||
// 固定候选(computedImportance + rawQuality,vValue=0 以隔离基线分支)按 fuseQuality 降序
|
||||
// 排序,验证:
|
||||
// ① gate 关(默认/回滚)= 基线序;
|
||||
// ② gate 开 = 「可信低分但高重要性」的记忆被压到「普通分低重要性」记忆之后(reorder);
|
||||
// ③ gate 再关 = 逐位复原到基线序(可逆)。
|
||||
// 另验证 0.0(未打分 / bug 残留)与 0.755(echo 常量)走「不可信→仅 importance」分支,
|
||||
// 不受本降权影响(dw 只作用于「可信低分」一档)——数值在 gate 开/关下不变。
|
||||
func TestFuseQuality_LowScoreDownweight_Ac5(t *testing.T) {
|
||||
// 固定候选:imp、rawQuality、id。vValue=0。
|
||||
type mem struct{
|
||||
id string
|
||||
imp float64
|
||||
qs float64
|
||||
}
|
||||
cands := []mem{
|
||||
{"L_low_hiimp", 1.0, 0.25}, // 可信低分 + 高重要性 → 被降权应下沉
|
||||
{"M_norm_loimp", 0.8, 0.35}, // 普通分(>=0.30 不降权)+ 较低重要性
|
||||
{"H_high", 0.5, 0.85}, // 高质量
|
||||
{"U_unscored", 1.0, 0.0}, // 0.0 未打分(bug 残留,不可信)
|
||||
{"E_echo", 0.6, 0.755}, // 0.755 echo 常量(不可信)
|
||||
}
|
||||
|
||||
// rank(order 用):按 fuseQuality(imp, qs, 0) 降序返回 id 列表 + 各 id 的 finalQuality。
|
||||
rank := func() (order []string, scores map[string]float64) {
|
||||
scores = map[string]float64{}
|
||||
type sc struct {
|
||||
m mem
|
||||
f float64
|
||||
}
|
||||
sclice := make([]sc, 0, len(cands))
|
||||
for _, m := range cands {
|
||||
sclice = append(sclice, sc{m, fuseQuality(m.imp, m.qs, 0)})
|
||||
}
|
||||
for i := 1; i < len(sclice); i++ {
|
||||
for j := i; j > 0 && sclice[j].f > sclice[j-1].f; j-- {
|
||||
sclice[j], sclice[j-1] = sclice[j-1], sclice[j]
|
||||
}
|
||||
}
|
||||
order = make([]string, 0, len(sclice))
|
||||
for _, s := range sclice {
|
||||
order = append(order, s.m.id)
|
||||
scores[s.m.id] = s.f
|
||||
}
|
||||
return order, scores
|
||||
}
|
||||
|
||||
approxEq := func(a, b float64) bool { return math.Abs(a-b) < 1e-9 }
|
||||
|
||||
// ── ① 基线(gate 关)─────────────────────────────────────────────
|
||||
t.Setenv("ZHIYI_LOWSCORE_DOWNWEIGHT", "0")
|
||||
baseOrder, baseScores := rank()
|
||||
t.Logf("基线(gate关) 序: %v 分: L=%.4f M=%.4f H=%.4f U=%.4f E=%.4f",
|
||||
baseOrder, baseScores["L_low_hiimp"], baseScores["M_norm_loimp"], baseScores["H_high"],
|
||||
baseScores["U_unscored"], baseScores["E_echo"])
|
||||
|
||||
// ── ② gate 开(降权生效)──────────────────────────────────────────
|
||||
t.Setenv("ZHIYI_LOWSCORE_DOWNWEIGHT", "1")
|
||||
onOrder, onScores := rank()
|
||||
t.Logf("gate开 序: %v 分: L=%.4f M=%.4f H=%.4f U=%.4f E=%.4f",
|
||||
onOrder, onScores["L_low_hiimp"], onScores["M_norm_loimp"], onScores["H_high"],
|
||||
onScores["U_unscored"], onScores["E_echo"])
|
||||
|
||||
// 断言①:可信低分 L 被压低(分数下降),普通分 M / 高质量 H 不变。
|
||||
if !(onScores["L_low_hiimp"] < baseScores["L_low_hiimp"]) {
|
||||
t.Fatalf("AC5 降权未生效:L 开gate后(%.4f) 未低于 基线(%.4f)",
|
||||
onScores["L_low_hiimp"], baseScores["L_low_hiimp"])
|
||||
}
|
||||
if !approxEq(onScores["M_norm_loimp"], baseScores["M_norm_loimp"]) {
|
||||
t.Fatalf("AC5 误伤普通分:M(q=0.35>=阈值) 开gate后(%.4f) ≠ 基线(%.4f),本应不变",
|
||||
onScores["M_norm_loimp"], baseScores["M_norm_loimp"])
|
||||
}
|
||||
if !approxEq(onScores["H_high"], baseScores["H_high"]) {
|
||||
t.Fatalf("AC5 误伤高质量:H 开gate后(%.4f) ≠ 基线(%.4f)", onScores["H_high"], baseScores["H_high"])
|
||||
}
|
||||
// 断言②:0.0 / 0.755 不可信行不受降权影响(数值在开/关下逐位不变)。
|
||||
if !approxEq(onScores["U_unscored"], baseScores["U_unscored"]) {
|
||||
t.Fatalf("AC5 误伤 0.0 未打分行:U 开gate后(%.4f) ≠ 基线(%.4f),本应不变(不可信分支)",
|
||||
onScores["U_unscored"], baseScores["U_unscored"])
|
||||
}
|
||||
if !approxEq(onScores["E_echo"], baseScores["E_echo"]) {
|
||||
t.Fatalf("AC5 误伤 0.755 echo 行:E 开gate后(%.4f) ≠ 基线(%.4f)", onScores["E_echo"], baseScores["E_echo"])
|
||||
}
|
||||
// 断言③:出现 reorder —— 基线里 L 排在 M 前,开 gate 后 M 反超 L。
|
||||
if !(_idx(baseOrder, "L_low_hiimp") < _idx(baseOrder, "M_norm_loimp")) {
|
||||
t.Fatalf("前置不成立:基线序 L 应在 M 前,得 %v", baseOrder)
|
||||
}
|
||||
if !(_idx(onOrder, "M_norm_loimp") < _idx(onOrder, "L_low_hiimp")) {
|
||||
t.Fatalf("AC5 未 reorder:开gate后 M 应反超低分 L,得 %v", onOrder)
|
||||
}
|
||||
|
||||
// ── ③ 回滚(gate 再关)= 逐位复原基线 ───────────────────────────
|
||||
t.Setenv("ZHIYI_LOWSCORE_DOWNWEIGHT", "0")
|
||||
backOrder, backScores := rank()
|
||||
if !equalStrs(backOrder, baseOrder) {
|
||||
t.Fatalf("AC5 回滚未复原:gate 再关序 %v ≠ 基线 %v", backOrder, baseOrder)
|
||||
}
|
||||
for _, id := range []string{"L_low_hiimp", "M_norm_loimp", "H_high", "U_unscored", "E_echo"} {
|
||||
if !approxEq(backScores[id], baseScores[id]) {
|
||||
t.Fatalf("AC5 回滚未复原 %s:再关(%.6f) ≠ 基线(%.6f)", id, backScores[id], baseScores[id])
|
||||
}
|
||||
}
|
||||
t.Logf("回滚(gate再关) 序: %v —— 与基线逐位一致,低分 L 复原到 %.4f", backOrder, backScores["L_low_hiimp"])
|
||||
}
|
||||
|
||||
func _idx(order []string, id string) int {
|
||||
for i, x := range order {
|
||||
if x == id {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func equalStrs(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -319,6 +319,19 @@ var qstrustVerbose = os.Getenv("ZHIYI_QSTRUST_VERBOSE") == "1"
|
||||
type RecallMeta struct {
|
||||
CandidatesScanned int
|
||||
CacheHit bool
|
||||
// Truncated(t_3aa2a8c1):本次召回的粗检索单次重读是否被 Rust 侧行数上限截断
|
||||
// (lancedb_search 单次重读 cap,默认 20000,env ZHIYI_HEAVY_READ_MAX_ROWS 可调)。
|
||||
// true = 部分结果:candidates_scanned/count 反映截断后状态,调用方不得当作「全量」。
|
||||
// 缓存命中路径 / 纯 Go 后端(无行数上限)⇒ false。
|
||||
Truncated bool
|
||||
}
|
||||
|
||||
// SearchTruncationReporter 可选扩展接口(t_3aa2a8c1):能报告「粗检索单次重读是否被
|
||||
// Rust 侧行数上限截断」的存储后端。RustLanceDBClient(IPC 路径)实现之;
|
||||
// 纯 Go 后端 / 测试 fake 不实现 ⇒ 按「恒未截断」处理(其 Search 没有行数上限,
|
||||
// 零值语义,行为不变)。
|
||||
type SearchTruncationReporter interface {
|
||||
SearchWithTruncation(table string, vector []float32, topK int, namespaceFilter string) ([]models.MemoryRecord, bool, error)
|
||||
}
|
||||
|
||||
// RecallWithMeta 同 Recall,但额外返回召回元数据(候选池真实大小 + 是否命中缓存)。
|
||||
@@ -400,8 +413,23 @@ func (p *RecallPipeline) recallUncached(query, namespace string, topK int, diver
|
||||
// ⇒ 默认召回只能看到极小一片(全表 10172 条,默认调用实测仅 11 个候选)。
|
||||
// 显式传 namespace / agent_id 时保持原作用域行为,不改变调用方语义。
|
||||
var candidates []models.MemoryRecord
|
||||
// t_3aa2a8c1:Rust 侧单次重读行数上限的 truncated 上抛到 recall 响应。
|
||||
// 只有实现了可选扩展接口 SearchTruncationReporter 的后端(RustLanceDBClient)
|
||||
// 会上报截断;未实现的(纯 Go / 测试 fake)保持旧 Search ⇒ 恒未截断。
|
||||
truncated := false
|
||||
reporter, _ := p.lancedb.(SearchTruncationReporter)
|
||||
searchCoarse := func(topK int, ns string) ([]models.MemoryRecord, error) {
|
||||
if reporter != nil {
|
||||
recs, t, err := reporter.SearchWithTruncation("memories", queryVec, topK, ns)
|
||||
if err == nil {
|
||||
truncated = truncated || t
|
||||
}
|
||||
return recs, err
|
||||
}
|
||||
return p.lancedb.Search("memories", queryVec, topK, ns)
|
||||
}
|
||||
if namespace == "" {
|
||||
all, err := p.lancedb.Search("memories", queryVec, 200, "") // 空 ns = 不过滤 = 全库
|
||||
all, err := searchCoarse(200, "") // 空 ns = 不过滤 = 全库
|
||||
if err == nil {
|
||||
candidates = append(candidates, all...)
|
||||
}
|
||||
@@ -415,7 +443,7 @@ func (p *RecallPipeline) recallUncached(query, namespace string, topK int, diver
|
||||
if recallV2Enabled() {
|
||||
pool = 200 // 实测相关记忆可能排在向量 #126 ⇒ top-50 池根本够不着
|
||||
}
|
||||
nsResults, err := p.lancedb.Search("memories", queryVec, pool, ns)
|
||||
nsResults, err := searchCoarse(pool, ns)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
@@ -591,7 +619,7 @@ func (p *RecallPipeline) recallUncached(query, namespace string, topK int, diver
|
||||
p.recordMiss(query, namespace)
|
||||
}
|
||||
if len(ftsTail) == 0 {
|
||||
return []models.RecallResult{}, RecallMeta{CandidatesScanned: poolSize}, nil
|
||||
return []models.RecallResult{}, RecallMeta{CandidatesScanned: poolSize, Truncated: truncated}, nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -842,6 +870,14 @@ func (p *RecallPipeline) recallUncached(query, namespace string, topK int, diver
|
||||
nAppend = len(fresh)
|
||||
}
|
||||
if len(results) >= topK && nAppend > 0 {
|
||||
// t_bd718662 P1:让位数不得超过现有条数。原实现在 len(results)==topK 时
|
||||
// 硬编码 budget=3,topK∈{1,2} 且 fresh 够多时 cut=len(results)-nAppend<0
|
||||
// ⇒ results[:cut] 负索引 panic(线上 trace: `slice bounds out of range [:-2]`
|
||||
// @ recall.go:846)。修后 cut≥0 恒成立;全仓唯一裁剪点仍在此(AC-D:
|
||||
// 任何路径不返回 > topK 条 —— 让位只做等量替换,不做追加扩张)。
|
||||
if nAppend > len(results) {
|
||||
nAppend = len(results)
|
||||
}
|
||||
cut := len(results) - nAppend
|
||||
results = results[:cut] // 让出尾部 nAppend 位(头部至多动 3 位)
|
||||
}
|
||||
@@ -984,7 +1020,21 @@ func (p *RecallPipeline) recallUncached(query, namespace string, topK int, diver
|
||||
}
|
||||
}()
|
||||
|
||||
return results, RecallMeta{CandidatesScanned: poolSize}, nil
|
||||
// Step 8: 热门度观测点(t_23b6a940 AC7,report-only、零排序影响):
|
||||
// 记录最终 rank1 的 recall_verified_count;连续越 p95 阈值满 N 个 query ⇒
|
||||
// ALERT 日志行(传感器通道),传感器端 zhiyi-popwatch.py 转既有告警通道。
|
||||
// 仅观测主 ANN 池出来的 rank1(图谱扩展/FTS 尾部位注入的 rank1 无池内计数,
|
||||
// 跳过不记)——热度偏置的作用面恰在主池 rerank 漏斗。
|
||||
if len(results) > 0 {
|
||||
for i := range candidates {
|
||||
if candidates[i].ID == results[0].ID {
|
||||
PopWatchRecord(results[0].ID, int64(candidates[i].RecallVerifiedCount))
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results, RecallMeta{CandidatesScanned: poolSize, Truncated: truncated}, nil
|
||||
}
|
||||
|
||||
// recallV2 开关(2026-10-14)。默认关 ⇒ 行为零变化。
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"github.com/xiaoxue/memoryweave/internal/models"
|
||||
)
|
||||
|
||||
// fbBiasLance 复用 nsRecordingLance 的 Search/Update 桩(行由外部注入),
|
||||
// 额外实现 IncrementHumanNotUseful —— 让测试走**与生产路由同一条 storage 接口方法**
|
||||
// (feedback.go MarkNotUseful → LanceDB.IncrementHumanNotUseful),而不是手工改字段。
|
||||
type fbBiasLance struct {
|
||||
nsRecordingLance
|
||||
}
|
||||
|
||||
// IncrementHumanNotUseful 对注入行做 +1(镜像 lancedb_ipc.go 的 $inc 语义:
|
||||
// 生产是原子 UPDATE ... SET human_not_useful_count = human_not_useful_count + 1)。
|
||||
func (f *fbBiasLance) IncrementHumanNotUseful(id string) error {
|
||||
for i := range f.rows {
|
||||
if f.rows[i].ID == id {
|
||||
f.rows[i].HumanNotUsefulCount++
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("fbBiasLance: row not found: %s", id)
|
||||
}
|
||||
|
||||
func fbVec(v0 float32) []float32 {
|
||||
v := make([]float32, 1024)
|
||||
v[0] = v0
|
||||
return v
|
||||
}
|
||||
|
||||
// newFbBiasPipeline 与 newSrcTierPipeline 同构:隔离进程级搜索缓存 + 预置 query 向量缓存,
|
||||
// 注入可自定义行/向量的 LanceDB 桩。
|
||||
func newFbBiasPipeline(t *testing.T, rows []models.MemoryRecord, queryVec []float32) *RecallPipeline {
|
||||
t.Helper()
|
||||
saved := SearchCacheInstance
|
||||
SearchCacheInstance = nil
|
||||
t.Cleanup(func() { SearchCacheInstance = saved })
|
||||
|
||||
emb := &Embedder{endpoint: "http://127.0.0.1:1/never", dim: 1024}
|
||||
emb.cache.Store("fbBiasQuery", queryVec)
|
||||
fake := &fbBiasLance{nsRecordingLance{rows: rows}}
|
||||
p := NewRecallPipeline(emb, fake, &Reranker{})
|
||||
_ = fake
|
||||
return p
|
||||
}
|
||||
|
||||
// fbRun 跑一次召回并返回 [id → (位次, 分数)];位次 0-based。
|
||||
func fbRun(t *testing.T, rows []models.MemoryRecord) map[string][2]float64 {
|
||||
t.Helper()
|
||||
p := newFbBiasPipeline(t, rows, fbVec(1.0))
|
||||
res, _, err := p.RecallWithMeta("fbBiasQuery", "hermes-main", 5, 0.0)
|
||||
if err != nil {
|
||||
t.Fatalf("RecallWithMeta 出错: %v", err)
|
||||
}
|
||||
out := make(map[string][2]float64, len(res))
|
||||
for i, r := range res {
|
||||
out[r.ID] = [2]float64{float64(i), r.Score}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TestFeedbackBiasZeroDiffWhenOff — t_7ba5e479 AC2(门关判据):
|
||||
// 生产现状 ZHIYI_FEEDBACK_BIAS=0(zhiyid.service 实测)⇒ recall.go:714 的偏置循环不执行 ⇒
|
||||
// 标记 not_useful 前后召回结果**逐位相同**(id 序列 + 分数全同,delta=0)。
|
||||
// 这正是 AC2「若没下降」的机械解释:信号写进库了,但排序主链在 env 门处断开。
|
||||
func TestFeedbackBiasZeroDiffWhenOff(t *testing.T) {
|
||||
oldBias := feedbackBias
|
||||
feedbackBias = 0 // 生产现状(zhiyid.service: ZHIYI_FEEDBACK_BIAS=0)
|
||||
defer func() { feedbackBias = oldBias }()
|
||||
|
||||
rows := []models.MemoryRecord{
|
||||
{ID: "m0", Content: "记忆甲 完全无关内容一", Category: "distilled", Source: "llm_distill", Vector: fbVec(1.0)},
|
||||
{ID: "m1", Content: "记忆乙 完全无关内容二", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.9)},
|
||||
{ID: "m2", Content: "记忆丙 完全无关内容三", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.8)},
|
||||
{ID: "m3", Content: "记忆丁 完全无关内容四", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.7)},
|
||||
{ID: "m4", Content: "记忆戊 完全无关内容五", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.6)},
|
||||
}
|
||||
|
||||
before := fbRun(t, rows) // m2 未标记(count=0)
|
||||
|
||||
// 走生产同一条 storage 方法标记 m2(等价路由 POST /feedback/not-useful 的写库动作)
|
||||
mlc := &fbBiasLance{nsRecordingLance{rows: rows}}
|
||||
if err := mlc.IncrementHumanNotUseful("m2"); err != nil {
|
||||
t.Fatalf("IncrementHumanNotUseful err: %v", err)
|
||||
}
|
||||
if rows[2].HumanNotUsefulCount != 1 {
|
||||
t.Fatalf("human_not_useful_count 应从 0→1,got %d", rows[2].HumanNotUsefulCount)
|
||||
}
|
||||
after := fbRun(t, rows) // 同查询重跑
|
||||
|
||||
// 判据:逐位相同(门关 ⇒ 零变化)
|
||||
for id, b := range before {
|
||||
a, ok := after[id]
|
||||
if !ok {
|
||||
t.Fatalf("门关时结果集合变了:m2 标记后 %s 消失", id)
|
||||
}
|
||||
if a[0] != b[0] || a[1] != b[1] {
|
||||
t.Fatalf("ZHIYI_FEEDBACK_BIAS=0 时标记前后位次/分数必须逐位相同:%s before=(pos %.0f score %.4f) after=(pos %.0f score %.4f)",
|
||||
id, b[0], b[1], a[0], a[1])
|
||||
}
|
||||
}
|
||||
t.Logf("bias=0(生产现状): 5 条结果标记前后逐位相同 ✓ m2 pos=%.0f score=%.4f(delta=0)",
|
||||
before["m2"][0], before["m2"][1])
|
||||
}
|
||||
|
||||
// TestFeedbackBiasLowersNotUseful — t_7ba5e479 AC2(门开判据):
|
||||
// ZHIYI_FEEDBACK_BIAS=1.0 ⇒ 偏置循环生效,FeedbackScore(m2) = sat(0) - sat(1) = -1/4 = -0.25
|
||||
// (K=ZHIYI_FEEDBACK_K 默认 3)⇒ m2 分数下降 0.25、位次下滑;
|
||||
// 未标记行 FeedbackScore=0 ⇒ 偏置恒 0,分数/相对位次不变(不误伤)。
|
||||
func TestFeedbackBiasLowersNotUseful(t *testing.T) {
|
||||
oldBias := feedbackBias
|
||||
feedbackBias = 1.0 // 门开(等价 env ZHIYI_FEEDBACK_BIAS=1.0)
|
||||
defer func() { feedbackBias = oldBias }()
|
||||
|
||||
rows := []models.MemoryRecord{
|
||||
{ID: "m0", Content: "记忆甲 完全无关内容一", Category: "distilled", Source: "llm_distill", Vector: fbVec(1.0)},
|
||||
{ID: "m1", Content: "记忆乙 完全无关内容二", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.9)},
|
||||
{ID: "m2", Content: "记忆丙 完全无关内容三", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.8)},
|
||||
{ID: "m3", Content: "记忆丁 完全无关内容四", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.7)},
|
||||
{ID: "m4", Content: "记忆戊 完全无关内容五", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.6)},
|
||||
}
|
||||
|
||||
before := fbRun(t, rows) // m2 count=0(标记前)
|
||||
mlc := &fbBiasLance{nsRecordingLance{rows: rows}}
|
||||
if err := mlc.IncrementHumanNotUseful("m2"); err != nil {
|
||||
t.Fatalf("IncrementHumanNotUseful err: %v", err)
|
||||
}
|
||||
after := fbRun(t, rows) // m2 count=1(标记后),同查询重跑
|
||||
|
||||
posBefore, scoreBefore := before["m2"][0], before["m2"][1]
|
||||
posAfter, scoreAfter := after["m2"][0], after["m2"][1]
|
||||
if !(posAfter > posBefore) {
|
||||
t.Fatalf("门开后 m2 位次未下滑:before pos=%.0f / after pos=%.0f(AC2 要求下降)", posBefore, posAfter)
|
||||
}
|
||||
if !(scoreAfter < scoreBefore) {
|
||||
t.Fatalf("门开后 m2 分数未下降:before=%.4f / after=%.4f", scoreBefore, scoreAfter)
|
||||
}
|
||||
// 预期降幅 = -feedbackBias * sat(1) = -1.0 * 1/(1+3) = -0.25(K 默认 3);
|
||||
// 若 rerank 重打分路径导致非精确值,退化为「严格下降 + 其它行逐位不动」双断言。
|
||||
want := -feedbackBias * (1.0 / (1.0 + float64(3)))
|
||||
delta := scoreAfter - scoreBefore
|
||||
if math.Abs(delta-want) > 1e-9 {
|
||||
t.Logf("note: 分数 delta=%.4f(rerank 重打分,非精确 %.4f),其它行未动则判据仍成立", delta, want)
|
||||
}
|
||||
// 不误伤:其它 4 条分数不变
|
||||
for _, id := range []string{"m0", "m1", "m3", "m4"} {
|
||||
if after[id][1] != before[id][1] {
|
||||
t.Fatalf("未标记行 %s 分数被误伤:before=%.4f after=%.4f", id, before[id][1], after[id][1])
|
||||
}
|
||||
}
|
||||
t.Logf("bias=1.0: m2 pos %.0f→%.0f, score %.4f→%.4f (delta=%.4f, 预期=%.4f) ✓ 未标记 4 行分数逐位不变 ✓",
|
||||
posBefore, posAfter, scoreBefore, scoreAfter, delta, want)
|
||||
}
|
||||
|
||||
// TestFeedbackBiasLiftsUseful — 正向对照(同门控的另一侧):
|
||||
// UsefulCount=1 ⇒ FeedbackScore = +sat(1) = +0.25 ⇒ 门开时 m2 分数上升、位次上移。
|
||||
func TestFeedbackBiasLiftsUseful(t *testing.T) {
|
||||
oldBias := feedbackBias
|
||||
feedbackBias = 1.0
|
||||
defer func() { feedbackBias = oldBias }()
|
||||
|
||||
rows := []models.MemoryRecord{
|
||||
{ID: "m0", Content: "记忆甲 完全无关内容一", Category: "distilled", Source: "llm_distill", Vector: fbVec(1.0)},
|
||||
{ID: "m1", Content: "记忆乙 完全无关内容二", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.9)},
|
||||
{ID: "m2", Content: "记忆丙 完全无关内容三", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.8)},
|
||||
{ID: "m3", Content: "记忆丁 完全无关内容四", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.7)},
|
||||
{ID: "m4", Content: "记忆戊 完全无关内容五", Category: "distilled", Source: "llm_distill", Vector: fbVec(0.6)},
|
||||
}
|
||||
|
||||
before := fbRun(t, rows)
|
||||
// 正向标记(同一 storage 接口的 MarkUseful 写法):+1 useful_count
|
||||
for i := range rows {
|
||||
if rows[i].ID == "m2" {
|
||||
rows[i].UsefulCount++
|
||||
}
|
||||
}
|
||||
after := fbRun(t, rows)
|
||||
if !(after["m2"][0] < before["m2"][0] || after["m2"][1] > before["m2"][1]) {
|
||||
t.Fatalf("门开后 useful 标记应使 m2 上移/加分:before=(pos %.0f, score %.4f) after=(pos %.0f, score %.4f)",
|
||||
before["m2"][0], before["m2"][1], after["m2"][0], after["m2"][1])
|
||||
}
|
||||
t.Logf("bias=1.0 正向对照: m2 pos %.0f→%.0f, score %.4f→%.4f ✓",
|
||||
before["m2"][0], after["m2"][0], before["m2"][1], after["m2"][1])
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
package storage
|
||||
|
||||
// t_bd718662【织忆·P1】/recall 线上 panic:recallUncached 让位截断出负索引。
|
||||
// 根因:results 已满(len==topK)时 budget 硬编码 3,nAppend=min(3,len(fresh));
|
||||
// topK∈{1,2} 且 fresh 足够多时 cut = len(results)-nAppend < 0 ⇒ results[:cut] 负索引 panic
|
||||
// (生产 trace:`runtime error: slice bounds out of range [:-2]` @ recall.go:846)。
|
||||
//
|
||||
// 本文件三层验证(对应卡 AC + 附录 D 增补):
|
||||
// 1. TestFTS_SmallTopKTailMatrix —— 表驱动:topK∈{1,2,3} × len(results)∈{0,1,2,topK}
|
||||
// × len(fresh)∈{0,1,2,3,5},断言:不 panic + len ≤ topK + 无重复 ID + 尾部 FTS 集合精确。
|
||||
// 修前:topK=1/F≥2 与 topK=2/F≥3 的满池格 panic(slice bounds out of range);
|
||||
// 修后:全部格通过。
|
||||
// 2. TestFTS_SmallTopKGolden —— topK∈{3,4,5} 的逐字输出(ID+Score+QualityScore+FtsBoosted),
|
||||
// 修前/修后各跑一次 diff,证明正常路径零回归(AC2 逐字一致)。
|
||||
// 3. 不变量(AC4/AC-D):任何路径 len(结果) ≤ topK —— 让位只做替换(截断+追加等量),
|
||||
// 全仓唯一裁剪点 recall.go:846,修后仍只此一处理截断。
|
||||
//
|
||||
// 全部纯内存 fake(向量池 fake + FTS Ldb fake),不触网、不碰 LanceDB/生产数据。
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/xiaoxue/memoryweave/internal/models"
|
||||
)
|
||||
|
||||
// matrixQuery 固定 5 个造词 ASCII 稀有词 + 1 个 CJK bigram(库里无 ⇒ 不命中)。
|
||||
// F 个 FTS 文档各含 Zetaflux1..F 之一 ⇒ FtsSearch 命中恰 F 条,且全在 ASCII 档(≥0.55)。
|
||||
const matrixQuery = "Zetaflux1 Zetaflux2 Zetaflux3 Zetaflux4 Zetaflux5 去重"
|
||||
|
||||
// poolOnly 向量池行:P 条纯内容文档(不含任何 Zetaflux 词)。
|
||||
// 关键一:每条**不同向量**(第 i 维 =1,其余 0)⇒ 两两 cosine=0,MMR 不做去重合并。
|
||||
// 关键二:每条**不同内容** ⇒ 躲开 Step 6「相同 content 只保留最高分」去重
|
||||
// (同内容池内文档会被塌成 1 条,P≥topK 满池场景造不出来)。
|
||||
var poolThemes = []string{"图谱", "嵌入", "蒸馏", "遗忘", "整合"}
|
||||
|
||||
func poolOnly(P int) []models.MemoryRecord {
|
||||
rows := make([]models.MemoryRecord, 0, P)
|
||||
for i := 1; i <= P; i++ {
|
||||
vec := make([]float32, 8)
|
||||
vec[i-1] = 1.0 // 第 i 条独占第 i 维
|
||||
rows = append(rows, models.MemoryRecord{
|
||||
ID: fmt.Sprintf("pdoc%d", i), Namespace: "hermes-main", Category: "distilled",
|
||||
Content: fmt.Sprintf("池内文档第%d篇讲织忆%s主题。", i, poolThemes[i-1]),
|
||||
Vector: vec,
|
||||
})
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
// ftsDocs FTS 侧行:F 条各含不同 Zetaflux 词的 shared 区文档(池外 ⇒ 只能靠词面通道进结果)。
|
||||
func ftsDocs(F int) []models.MemoryRecord {
|
||||
vec := make([]float32, 8)
|
||||
for i := range vec {
|
||||
vec[i] = 0.5
|
||||
}
|
||||
rows := make([]models.MemoryRecord, 0, F)
|
||||
for i := 1; i <= F; i++ {
|
||||
rows = append(rows, models.MemoryRecord{
|
||||
ID: fmt.Sprintf("zdoc%d", i), Namespace: "shared", Category: "distilled",
|
||||
Content: fmt.Sprintf("共享区稀有词 Zetaflux%d 出现处。", i), Vector: vec,
|
||||
})
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
// smallKPool:向量搜索恒返回固定池行(忽略请求 topK,模拟「池内 P 条、池外 F 条」),
|
||||
// FTS 池由 FtsProvider.FtsLdb 注入(池行 + FTS 行全量,索引层自己剔类别/软删)。
|
||||
type smallKPool struct {
|
||||
LanceDB // 未覆写的方法(AppendHitEvent 等)走接口缺省 —— 同 ftsSearchOnly 范式
|
||||
poolRows []models.MemoryRecord
|
||||
dim int
|
||||
}
|
||||
|
||||
func (f *smallKPool) Search(table string, vector []float32, topK int, namespaceFilter string) ([]models.MemoryRecord, error) {
|
||||
return f.poolRows, nil
|
||||
}
|
||||
func (f *smallKPool) Update(table, id string, fields map[string]any) error { return nil }
|
||||
|
||||
// runMatrixCell 单格:P 条池内 + F 条池外,topK 召回。
|
||||
func runMatrixCell(t *testing.T, topK, P, F int) []models.RecallResult {
|
||||
t.Helper()
|
||||
|
||||
savedCache := SearchCacheInstance
|
||||
SearchCacheInstance = nil
|
||||
t.Cleanup(func() { SearchCacheInstance = savedCache })
|
||||
|
||||
savedFts := FtsProvider.FtsLdb
|
||||
all := make([]models.MemoryRecord, 0, P+F)
|
||||
all = append(all, poolOnly(P)...)
|
||||
all = append(all, ftsDocs(F)...)
|
||||
FtsProvider.FtsLdb = func() ([]models.MemoryRecord, error) { return all, nil }
|
||||
t.Cleanup(func() { FtsProvider.FtsLdb = savedFts })
|
||||
|
||||
resetFTSIndex()
|
||||
|
||||
vec := make([]float32, 8)
|
||||
for i := range vec {
|
||||
vec[i] = 0.5
|
||||
}
|
||||
emb := &Embedder{endpoint: "http://127.0.0.1:1/never", dim: 8}
|
||||
emb.cache.Store(matrixQuery, vec)
|
||||
p := NewRecallPipeline(emb, &smallKPool{poolRows: poolOnly(P), dim: 8}, &Reranker{})
|
||||
|
||||
res, err := p.Recall(matrixQuery, "hermes-main", topK, 0.5)
|
||||
if err != nil {
|
||||
t.Fatalf("topK=%d P=%d F=%d recall 报错: %v", topK, P, F, err)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// 期望值推导(修后语义 = 让位只做等量替换):
|
||||
//
|
||||
// P < topK:head = P(满 P 条池内),nAppend = min(topK-P, F),total = P + nAppend ≤ topK
|
||||
// P ≥ topK:head = topK,nAppend = min(3, F, topK),截掉 nAppend 条再补 nAppend 条,total = topK
|
||||
// 尾部 FTS 集合 = zdoc1..zdoc_nAppend(各命中词 idf 相等 ⇒ 同分按 ID 升序)
|
||||
func expectedYield(topK, P, F int) (total, nAppend int) {
|
||||
if P < topK {
|
||||
nAppend = topK - P
|
||||
if nAppend > F {
|
||||
nAppend = F
|
||||
}
|
||||
total = P + nAppend
|
||||
} else {
|
||||
nAppend = 3
|
||||
if nAppend > F {
|
||||
nAppend = F
|
||||
}
|
||||
if nAppend > topK { // t_bd718662 修后:让位上限 = min(3,F,现有条数)
|
||||
nAppend = topK
|
||||
}
|
||||
total = topK
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// 表驱动(附录 D):topK∈{1,2,3} × len(results)∈{0,1,2,topK} × len(fresh)∈{0,1,2,3,5}。
|
||||
// P(池内条数)决定 len(results)=min(P,topK):topK=1/2 时 P∈{0,1,2}、topK=3 时 P∈{0,1,2,3}。
|
||||
func TestFTS_SmallTopKTailMatrix(t *testing.T) {
|
||||
t.Setenv("ZHIYI_RECALL_FTS", "1")
|
||||
|
||||
type cell struct {
|
||||
topK, P, F int
|
||||
}
|
||||
cells := []cell{}
|
||||
for _, tk := range []int{1, 2, 3} {
|
||||
ps := []int{0, 1, 2}
|
||||
if tk >= 3 {
|
||||
ps = append(ps, 3)
|
||||
}
|
||||
for _, P := range ps {
|
||||
for _, F := range []int{0, 1, 2, 3, 5} {
|
||||
cells = append(cells, cell{tk, P, F})
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, c := range cells {
|
||||
c := c
|
||||
t.Run(fmt.Sprintf("topK%d_P%d_F%d", c.topK, c.P, c.F), func(t *testing.T) {
|
||||
res := runMatrixCell(t, c.topK, c.P, c.F)
|
||||
|
||||
// AC1/AC4/AC-D:不超 topK
|
||||
if len(res) > c.topK {
|
||||
t.Fatalf("topK=%d P=%d F=%d 返回 %d 条 > topK(让位逻辑做了追加扩张): %v",
|
||||
c.topK, c.P, c.F, len(res), idsRes(res))
|
||||
}
|
||||
// 无重复 ID
|
||||
seen := map[string]int{}
|
||||
for _, r := range res {
|
||||
seen[r.ID]++
|
||||
}
|
||||
for id, n := range seen {
|
||||
if n > 1 {
|
||||
t.Fatalf("topK=%d P=%d F=%d id %s 出现 %d 次 ⇒ 去重被破坏", c.topK, c.P, c.F, id, n)
|
||||
}
|
||||
}
|
||||
// 精确集合断言:头部 = 池内 pdoc 前 min(P,topK) 条(同向量下 MMR 稳定序),
|
||||
// 尾部 = zdoc1..zdoc_nAppend(FTS 同分按 ID 升序)
|
||||
wantTotal, wantN := expectedYield(c.topK, c.P, c.F)
|
||||
if len(res) != wantTotal {
|
||||
t.Fatalf("topK=%d P=%d F=%d 长度 %d ≠ 期望 %d(结果=%v)",
|
||||
c.topK, c.P, c.F, len(res), wantTotal, idsRes(res))
|
||||
}
|
||||
var gotFts []string
|
||||
for _, r := range res {
|
||||
if r.FtsBoosted {
|
||||
gotFts = append(gotFts, r.ID)
|
||||
}
|
||||
}
|
||||
if len(gotFts) != wantN {
|
||||
t.Fatalf("topK=%d P=%d F=%d FTS 条数 %d ≠ 期望 %d(结果=%v)",
|
||||
c.topK, c.P, c.F, len(gotFts), wantN, idsRes(res))
|
||||
}
|
||||
for i := 0; i < wantN; i++ {
|
||||
if gotFts[i] != fmt.Sprintf("zdoc%d", i+1) {
|
||||
t.Fatalf("topK=%d P=%d F=%d 尾部第 %d 位 = %s ≠ 期望 zdoc%d(FTS 分降序+ID 升序被破坏: %v)",
|
||||
c.topK, c.P, c.F, i+1, gotFts[i], i+1, gotFts)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// golden:topK∈{3,4,5}(正常路径)+ F=0(无 FTS 命中)四格,逐字输出。
|
||||
// 修前修后各跑一次:go test -run TestFTS_SmallTopKGolden ./internal/storage/ -v 2>&1 | grep GOLDEN
|
||||
// 两份输出 diff 必须为空(AC2 零回归判据)。
|
||||
func TestFTS_SmallTopKGolden(t *testing.T) {
|
||||
t.Setenv("ZHIYI_RECALL_FTS", "1")
|
||||
for _, tc := range []struct {
|
||||
topK, P, F int
|
||||
}{
|
||||
{3, 5, 3}, {4, 5, 3}, {5, 5, 3}, {5, 5, 0},
|
||||
} {
|
||||
res := runMatrixCell(t, tc.topK, tc.P, tc.F)
|
||||
line := fmt.Sprintf("GOLDEN topK=%d P=%d F=%d: [", tc.topK, tc.P, tc.F)
|
||||
for i, r := range res {
|
||||
if i > 0 {
|
||||
line += " "
|
||||
}
|
||||
line += fmt.Sprintf("{%s score=%.6f qs=%.6f fts=%v}", r.ID, r.Score, r.QualityScore, r.FtsBoosted)
|
||||
}
|
||||
line += "]"
|
||||
t.Logf("%s", line)
|
||||
}
|
||||
}
|
||||
@@ -45,6 +45,12 @@ type BatchRecallUpdater interface {
|
||||
const (
|
||||
defaultRecallFlushInterval = 300 * time.Second
|
||||
defaultRecallFlushMaxItems = 256
|
||||
// 优雅停止窗口(t_fa797edd):Stop 内的「等后台协程退出 + 最终落盘」都必须有界。
|
||||
// 否则 Rust/LanceDB 批量更新在活 IPC 流量下阻塞时,Stop 永远不返回 → 信号协程
|
||||
// 到不了 srv.Shutdown → 进程跑满 systemd 90s TimeoutStopSec 被 SIGKILL
|
||||
// (丢写风险 + 每次重启都变强杀 + 40s IPC 中断)。
|
||||
defaultRecallStopLoopWait = 5 * time.Second // 等后台 flush 协程退出的上限
|
||||
defaultRecallStopFlushWait = 8 * time.Second // 最终落盘的上限(召回统计,丢失可接受)
|
||||
)
|
||||
|
||||
type pendingRecall struct {
|
||||
@@ -249,6 +255,10 @@ func (b *RecallWriteBuffer) Flush() (int, error) {
|
||||
}
|
||||
|
||||
// Stop 停止后台 flush 协程,并把最后一个窗口落盘(进程优雅退出时调用)。
|
||||
// 全程有界(t_fa797edd):等后台协程 ≤ defaultRecallStopLoopWait,最终落盘
|
||||
// ≤ defaultRecallStopFlushWait。Rust/LanceDB 批量更新在活 IPC 流量下可能阻塞
|
||||
// 数十秒;召回统计丢了可接受(源码注释「进程崩溃会丢最后一个窗口」),
|
||||
// 因此超时即放弃落盘、放行退出,保证进程总在有限秒内到 srv.Shutdown。
|
||||
func (b *RecallWriteBuffer) Stop() {
|
||||
b.mu.Lock()
|
||||
if b.stopped {
|
||||
@@ -261,10 +271,23 @@ func (b *RecallWriteBuffer) Stop() {
|
||||
close(b.stopCh)
|
||||
select {
|
||||
case <-b.doneCh:
|
||||
case <-time.After(10 * time.Second):
|
||||
log.Printf("[recall-buffer] 停止超时,直接落盘剩余窗口")
|
||||
case <-time.After(defaultRecallStopLoopWait):
|
||||
log.Printf("[recall-buffer] 后台 flush 协程停止超时(%s),直接进入最终落盘", defaultRecallStopLoopWait)
|
||||
}
|
||||
|
||||
// 最终落盘有界:放到 goroutine 跑,到点不再等。若后台 ticker/阈值 flush
|
||||
// 正持锁阻塞在 Rust IPC 上,本 goroutine 的 Flush 会先等 b.mu —— 同样被
|
||||
// defaultRecallStopFlushWait 兜住,超时即放行进程退出。
|
||||
flushDone := make(chan struct{})
|
||||
go func() {
|
||||
b.Flush()
|
||||
close(flushDone)
|
||||
}()
|
||||
select {
|
||||
case <-flushDone:
|
||||
case <-time.After(defaultRecallStopFlushWait):
|
||||
log.Printf("[recall-buffer] 最终落盘超时(%s),放弃剩余窗口(仅丢召回统计,进程照常退出)", defaultRecallStopFlushWait)
|
||||
}
|
||||
b.Flush()
|
||||
}
|
||||
|
||||
// Stats 观测用(flush 频率 / 合并率 / 版本写入行数)。
|
||||
|
||||
@@ -240,3 +240,31 @@ func TestRecallWriteBufferStopFlushes(t *testing.T) {
|
||||
t.Fatalf("Stop 应落盘最后一个窗口,batchCalls=%d", fake.batchCalls)
|
||||
}
|
||||
}
|
||||
|
||||
// hangingLDB:UpdateRecallBatch 永不返回(模拟 Rust/LanceDB 批量更新在活 IPC
|
||||
// 流量下被卡死,t_fa797edd 生产实测 80s+)。用于验证 Stop 有界:不会因 flush
|
||||
// 挂起而无限阻塞进程退出。
|
||||
type hangingLDB struct{ fakeLDB }
|
||||
|
||||
func (h *hangingLDB) UpdateRecallBatch(table string, items []RecallWriteItem, lastRecalledAt string) (int64, error) {
|
||||
select {} // 永不返回
|
||||
}
|
||||
|
||||
// t_fa797edd 反向用例:最终落盘挂起时,Stop 必须在有界时间内返回(不许无限阻塞)。
|
||||
// 原实现 b.Flush() 同步调用 → Rust 卡死则 Stop 永不返回 → 进程到不了 srv.Shutdown
|
||||
// → 跑满 systemd 90s 被 SIGKILL。修复后 Stop 在 defaultRecallStopFlushWait 内放行。
|
||||
func TestRecallWriteBufferStopBoundedWhenFlushHangs(t *testing.T) {
|
||||
h := &hangingLDB{}
|
||||
b := NewRecallWriteBuffer(h, 0, 1000)
|
||||
b.Record([]string{"mem_hang"})
|
||||
|
||||
start := time.Now()
|
||||
b.Stop()
|
||||
elapsed := time.Since(start)
|
||||
|
||||
// 有界上限:后台协程等待(5s) + 最终落盘等待(8s) + 裕量。50s 远小于 systemd 90s 超时。
|
||||
if elapsed > 50*time.Second {
|
||||
t.Fatalf("Stop 应是有界的,实际阻塞了 %s(>50s,会触发 systemd 90s SIGKILL)", elapsed)
|
||||
}
|
||||
t.Logf("Stop 在 %s 内放行(flush 挂起时也不阻塞进程退出)", elapsed)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
package storage
|
||||
|
||||
// t_3aa2a8c1:`truncated` 在 IPC 层被 Go 客户端丢掉 —— Search 只打日志不上抛,
|
||||
// 本测试用假 sidecar(unix socket 实现真实 IPC 协议)验证 SearchWithTruncation 透传:
|
||||
// ① 假 sidecar 回 truncated=true ⇒ 标志上抛(不再静默)
|
||||
// ② 回 truncated=false / 旧 sidecar 无该字段 ⇒ false(向后兼容)
|
||||
// ③ 本地兜底(IPC 不可用 / 零结果)⇒ false(兜底路径没有行数上限)
|
||||
// ④ 旧签名 Search 行为回归(委托 SearchWithTruncation,records 不变)
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// startFakeTruncSidecar 起一个只回一帧的假 sidecar:读满请求(长度前缀 + body),
|
||||
// 按 truncated 值回 ConsolidateResponse 形状(report_json = 1 条空壳记录,非零结果,
|
||||
// 避免触发本地兜底分支)。
|
||||
func startFakeTruncSidecar(t *testing.T, truncated bool) (string, func()) {
|
||||
t.Helper()
|
||||
tmp, err := os.MkdirTemp(os.TempDir(), "zytrunc")
|
||||
if err != nil {
|
||||
t.Fatalf("mkdtemp: %v", err)
|
||||
}
|
||||
sock := filepath.Join(tmp, "tr.sock")
|
||||
l, err := net.Listen("unix", sock)
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
go func() {
|
||||
for {
|
||||
c, aerr := l.Accept()
|
||||
if aerr != nil {
|
||||
return
|
||||
}
|
||||
go func(conn net.Conn) {
|
||||
var lenBuf [4]byte
|
||||
if _, rerr := io.ReadFull(conn, lenBuf[:]); rerr != nil {
|
||||
return
|
||||
}
|
||||
n := binary.BigEndian.Uint32(lenBuf[:])
|
||||
buf := make([]byte, n)
|
||||
if _, rerr := io.ReadFull(conn, buf); rerr != nil {
|
||||
return
|
||||
}
|
||||
report := `[{"id":"mem_fake"}]`
|
||||
resp, _ := json.Marshal(map[string]interface{}{
|
||||
"status": "ok", "report_json": report, "truncated": truncated,
|
||||
})
|
||||
binary.BigEndian.PutUint32(lenBuf[:], uint32(len(resp)))
|
||||
if _, werr := conn.Write(lenBuf[:]); werr != nil {
|
||||
return
|
||||
}
|
||||
if _, werr := conn.Write(resp); werr != nil {
|
||||
return
|
||||
}
|
||||
}(c)
|
||||
}
|
||||
}()
|
||||
cleanup := func() {
|
||||
l.Close()
|
||||
os.RemoveAll(tmp)
|
||||
}
|
||||
return sock, cleanup
|
||||
}
|
||||
|
||||
// ① Rust 侧被行数上限截断 ⇒ Go 客户端必须上抛(t_3aa2a8c1 核心)
|
||||
func TestSearchWithTruncation_FlagPropagated(t *testing.T) {
|
||||
sock, cleanup := startFakeTruncSidecar(t, true)
|
||||
defer cleanup()
|
||||
rc := NewRustLanceDBClient(sock, nil)
|
||||
|
||||
recs, trunc, err := rc.SearchWithTruncation("memories", make([]float32, 1024), 3, "")
|
||||
if err != nil {
|
||||
t.Fatalf("rpc 失败: %v", err)
|
||||
}
|
||||
if !trunc {
|
||||
t.Fatalf("假 sidecar 已回 truncated=true,SearchWithTruncation 却未上抛(仍静默)")
|
||||
}
|
||||
if len(recs) != 1 || recs[0].ID != "mem_fake" {
|
||||
t.Fatalf("records 回归异常: %+v", recs)
|
||||
}
|
||||
}
|
||||
|
||||
// ② 未截断(新 sidecar 显式 false)与旧 sidecar(字段缺失 ⇒ 缺省 false)⇒ 恒 false
|
||||
func TestSearchWithTruncation_NotTruncated(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
truncated bool
|
||||
}{
|
||||
{"new sidecar false", false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
sock, cleanup := startFakeTruncSidecar(t, tc.truncated)
|
||||
defer cleanup()
|
||||
rc := NewRustLanceDBClient(sock, nil)
|
||||
_, trunc, err := rc.SearchWithTruncation("memories", make([]float32, 1024), 3, "")
|
||||
if err != nil {
|
||||
t.Fatalf("rpc 失败: %v", err)
|
||||
}
|
||||
if trunc {
|
||||
t.Fatalf("未截断场景却上报 truncated=true(假阳性)")
|
||||
}
|
||||
})
|
||||
}
|
||||
// 旧 sidecar 形状:响应无 truncated 字段 ⇒ serde/JSON 缺省 false(向后兼容回归)
|
||||
tmp, _ := os.MkdirTemp(os.TempDir(), "zyold")
|
||||
defer os.RemoveAll(tmp)
|
||||
sock := filepath.Join(tmp, "old.sock")
|
||||
l, _ := net.Listen("unix", sock)
|
||||
go func() {
|
||||
for {
|
||||
c, aerr := l.Accept()
|
||||
if aerr != nil {
|
||||
return
|
||||
}
|
||||
go func(conn net.Conn) {
|
||||
var lenBuf [4]byte
|
||||
if _, rerr := io.ReadFull(conn, lenBuf[:]); rerr != nil {
|
||||
return
|
||||
}
|
||||
n := binary.BigEndian.Uint32(lenBuf[:])
|
||||
buf := make([]byte, n)
|
||||
if _, rerr := io.ReadFull(conn, buf); rerr != nil {
|
||||
return
|
||||
}
|
||||
// 无 truncated 字段(旧 sidecar 形状)
|
||||
resp, _ := json.Marshal(map[string]interface{}{
|
||||
"status": "ok", "report_json": `[{"id":"mem_old"}]`,
|
||||
})
|
||||
binary.BigEndian.PutUint32(lenBuf[:], uint32(len(resp)))
|
||||
_, _ = conn.Write(lenBuf[:])
|
||||
_, _ = conn.Write(resp)
|
||||
}(c)
|
||||
}
|
||||
}()
|
||||
defer l.Close()
|
||||
rc := NewRustLanceDBClient(sock, nil)
|
||||
_, trunc, err := rc.SearchWithTruncation("memories", make([]float32, 1024), 3, "")
|
||||
if err != nil {
|
||||
t.Fatalf("旧 sidecar 形状 rpc 失败: %v", err)
|
||||
}
|
||||
if trunc {
|
||||
t.Fatalf("旧 sidecar(无 truncated 字段)必须向后兼容为 false,实为 true")
|
||||
}
|
||||
}
|
||||
|
||||
// ③ 本地兜底(socket 不存在 ⇒ dial 失败 ⇒ localSearch)⇒ false,不报错(降级语义不变)
|
||||
func TestSearchWithTruncation_LocalFallbackFalse(t *testing.T) {
|
||||
rc := NewRustLanceDBClient("/tmp/zhiyi-trunc-does-not-exist.sock", nil)
|
||||
_, trunc, err := rc.SearchWithTruncation("memories", make([]float32, 1024), 3, "")
|
||||
if err != nil {
|
||||
t.Fatalf("本地兜底应降级成功(旧 Search 语义:返回 nil err),实际: %v", err)
|
||||
}
|
||||
if trunc {
|
||||
t.Fatalf("本地兜底路径没有行数上限,必须 false")
|
||||
}
|
||||
}
|
||||
|
||||
// ④ 回归:旧签名 Search 走委托,records 行为不变(拿到结果、nil 错误)
|
||||
func TestSearch_DelegatedRegression(t *testing.T) {
|
||||
sock, cleanup := startFakeTruncSidecar(t, true)
|
||||
defer cleanup()
|
||||
rc := NewRustLanceDBClient(sock, nil)
|
||||
recs, err := rc.Search("memories", make([]float32, 1024), 3, "")
|
||||
if err != nil {
|
||||
t.Fatalf("旧签名 Search 回归破坏: %v", err)
|
||||
}
|
||||
if len(recs) != 1 || recs[0].ID != "mem_fake" {
|
||||
t.Fatalf("旧签名 Search records 回归异常: %+v", recs)
|
||||
}
|
||||
}
|
||||
@@ -68,6 +68,34 @@ var (
|
||||
usageTokenTopN = int(envFloat("ZHIYI_USAGE_TOKEN_TOP_N", 8))
|
||||
)
|
||||
|
||||
// 退避参数(t_20bfbd9a 治本①):命中事件重试的跨 commit 指数退避。
|
||||
// 与 IPC 层 in-call 退避(lancedb_ipc.go 50/100/200ms)正交:这里的退避是**跨 commit 层面**,
|
||||
// 防的是「一条持久性失败的坏行在每次 commit 都被热重扫」→ 风暴叠风暴。首次失败退避 1s,
|
||||
// 每多失败一次翻倍,封顶 10min。成功消费后 retry_count 归零(usage_track.go 消费分支)。
|
||||
var (
|
||||
usageBackoffBase = envFloat("ZHIYI_USAGE_BACKOFF_BASE_S", 1)
|
||||
usageBackoffCap = envFloat("ZHIYI_USAGE_BACKOFF_CAP_S", 600)
|
||||
)
|
||||
|
||||
// usageBackoff 第 n 次失败的退避时长:min(2^(n-1) * base, cap)。
|
||||
// n>=1;n=1 → base,n=2 → 2*base,… 封顶 cap。
|
||||
func usageBackoff(n int64) time.Duration {
|
||||
s := usageBackoffBase
|
||||
for i := int64(1); i < n; i++ {
|
||||
next := s * 2
|
||||
if next >= usageBackoffCap {
|
||||
// 下一步将超过 cap:直接封顶返回,不再累计
|
||||
return time.Duration(usageBackoffCap * float64(time.Second))
|
||||
}
|
||||
s = next
|
||||
}
|
||||
return time.Duration(s * float64(time.Second))
|
||||
}
|
||||
|
||||
// envFloat 读取环境浮点配置(默认值兜底;0/负值视为默认)。
|
||||
// ⚠️ recall.go 已有同签名 envFloat(t_da61c470,语义:解析失败回落默认、≤0 原样返回),
|
||||
// 这里不再重声明,直接复用;消费方需自备 >0 守卫(backlogMax/backoff 均已带)。
|
||||
|
||||
// ─── 库文件(织忆自己的库,非 Hermes state.db)─────────────────────
|
||||
// var 而非 const:测试可换到临时路径,避免污染生产 usage_track.db。
|
||||
var usageTrackDBPath = "/var/lib/memoryweave/usage_track.db"
|
||||
@@ -108,10 +136,22 @@ func usageOpen() *sql.DB {
|
||||
agent TEXT,
|
||||
recalled_at TEXT NOT NULL,
|
||||
consumed INTEGER NOT NULL DEFAULT 0,
|
||||
consumed_at TEXT
|
||||
consumed_at TEXT,
|
||||
retry_count INTEGER NOT NULL DEFAULT 0,
|
||||
next_retry_at TEXT
|
||||
)`); err != nil {
|
||||
logUsage("create table: %v", err)
|
||||
}
|
||||
// 存量库迁移(t_20bfbd9a):旧表无 retry_count/next_retry_at 列,逐列补(幂等)。
|
||||
for _, stmt := range []string{
|
||||
"ALTER TABLE usage_recall ADD COLUMN retry_count INTEGER NOT NULL DEFAULT 0",
|
||||
"ALTER TABLE usage_recall ADD COLUMN next_retry_at TEXT",
|
||||
} {
|
||||
if _, err := db.Exec(stmt); err != nil && !strings.Contains(err.Error(), "duplicate column name") {
|
||||
// 非「已存在」的错(表不存在等)也忽略:CREATE IF NOT EXISTS 已兜表级,
|
||||
// 这里只负责加列,列已存在即静默通过。
|
||||
}
|
||||
}
|
||||
_, _ = db.Exec(`CREATE INDEX IF NOT EXISTS idx_usage_mem ON usage_recall(memory_id)`)
|
||||
_, _ = db.Exec(`CREATE INDEX IF NOT EXISTS idx_usage_time ON usage_recall(recalled_at)`)
|
||||
usageDB = db
|
||||
@@ -169,10 +209,26 @@ func CheckCommittedForUse(ldb LanceDB, agent, namespace, content string) []Usage
|
||||
if db == nil {
|
||||
return nil
|
||||
}
|
||||
// 积压上限量化(t_20bfbd9a 治本②):未消费待重试行超过上限 ⇒ 本轮跳过扫描,
|
||||
// 防止「一条永远失败的坏行 + 高 commit 频率」在每次 commit 都全量重扫、叠成风暴。
|
||||
// 上限 env ZHIYI_USAGE_BACKLOG_MAX,默认 500。
|
||||
backlogMax := int(envFloat("ZHIYI_USAGE_BACKLOG_MAX", 500))
|
||||
if backlogMax > 0 {
|
||||
var pending int
|
||||
if err := db.QueryRow(`SELECT COUNT(*) FROM usage_recall WHERE consumed=0`).Scan(&pending); err == nil && pending > backlogMax {
|
||||
logUsage("backlog %d > cap %d: skip scan this commit", pending, backlogMax)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
cutoff := time.Now().Add(-time.Duration(usageWindowHours * float64(time.Hour))).Format(time.RFC3339)
|
||||
scanNow := time.Now()
|
||||
// 退避过滤(t_20bfbd9a 治本①):next_retry_at 为 NULL(首次待处理)或已到重试时间,
|
||||
// 才进本轮重扫。持久性失败的坏行被指数退避挡在窗口外,不再每 commit 热重试。
|
||||
rows, err := db.Query(
|
||||
`SELECT id, memory_id, content, tokens, namespace, agent
|
||||
FROM usage_recall WHERE consumed=0 AND recalled_at>=?`, cutoff)
|
||||
`SELECT id, memory_id, content, tokens, namespace, agent, retry_count
|
||||
FROM usage_recall WHERE consumed=0 AND recalled_at>=?
|
||||
AND (next_retry_at IS NULL OR next_retry_at<=?)`,
|
||||
cutoff, scanNow.Format(time.RFC3339))
|
||||
if err != nil {
|
||||
logUsage("query recall log: %v", err)
|
||||
return nil
|
||||
@@ -183,7 +239,8 @@ func CheckCommittedForUse(ldb LanceDB, agent, namespace, content string) []Usage
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
var memID, memContent, tokensJSON, ns, ag string
|
||||
if err := rows.Scan(&id, &memID, &memContent, &tokensJSON, &ns, &ag); err != nil {
|
||||
var retryCount int64
|
||||
if err := rows.Scan(&id, &memID, &memContent, &tokensJSON, &ns, &ag, &retryCount); err != nil {
|
||||
continue
|
||||
}
|
||||
// 归属过滤:同一 agent,或同一 namespace,或共享/全局。跨域不强求。
|
||||
@@ -207,6 +264,13 @@ func CheckCommittedForUse(ldb LanceDB, agent, namespace, content string) []Usage
|
||||
// zhiyi_hit_event_append_failed_total),禁静默。
|
||||
if err := ldb.AppendHitEvent(memID, HitOutcomePos, HitReasonDownstreamUse); err != nil {
|
||||
logUsage("hit event append FAILED (id=%s, NOT consumed, will retry next commit) : %v", memID, err)
|
||||
// t_20bfbd9a 治本①:持久性失败(如坏行 memID 已从 LanceDB 删除)若不加退避,
|
||||
// 会在每次 commit 都热重试,commit 风暴时叠成重扫风暴。加指数退避:
|
||||
// 第 n 次失败后 next_retry_at = now + min(2^(n-1) * base, cap),退避期外不再重扫。
|
||||
// 成功消费时把 retry_count 归零;每轮只推进 retry_count 一次(不指数级暴涨)。
|
||||
_, _ = db.Exec(
|
||||
`UPDATE usage_recall SET retry_count=retry_count+1, next_retry_at=? WHERE id=?`,
|
||||
time.Now().Add(usageBackoff(retryCount+1)).Format(time.RFC3339), id)
|
||||
continue
|
||||
}
|
||||
// t_28aa278e 方案①:更新成功后才消费 usage_recall 行。
|
||||
@@ -218,9 +282,12 @@ func CheckCommittedForUse(ldb LanceDB, agent, namespace, content string) []Usage
|
||||
"recall_verified_count": map[string]string{"$inc": "1"},
|
||||
}); err != nil {
|
||||
logUsage("recall_verified_count update FAILED (id=%s, NOT consumed, will retry next commit) : %v", memID, err)
|
||||
_, _ = db.Exec(
|
||||
`UPDATE usage_recall SET retry_count=retry_count+1, next_retry_at=? WHERE id=?`,
|
||||
time.Now().Add(usageBackoff(retryCount+1)).Format(time.RFC3339), id)
|
||||
continue
|
||||
}
|
||||
_, _ = db.Exec(`UPDATE usage_recall SET consumed=1, consumed_at=? WHERE id=?`, now, id)
|
||||
_, _ = db.Exec(`UPDATE usage_recall SET consumed=1, consumed_at=?, retry_count=0, next_retry_at=NULL WHERE id=?`, now, id)
|
||||
flags = append(flags, UsageFlag{
|
||||
MemoryID: memID, Content: memContent, Criterion: "carry_forward",
|
||||
MatchedTokens: toks, HitCount: hits, RunesMatched: runes, ConsumedAt: now,
|
||||
|
||||
@@ -3,6 +3,7 @@ package storage
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// setupUsageTest 把追踪库指到临时路径 + 显式打开门控,测完复位。
|
||||
@@ -123,4 +124,60 @@ func TestUsageTracking_NoFalsePositiveOnUnrelated(t *testing.T) {
|
||||
if f := CheckCommittedForUse(mlc, "a06", "hermes", "今天天气很好,出去散步了"); len(f) != 0 {
|
||||
t.Fatalf("无关内容不应误判, got %v", f)
|
||||
}
|
||||
}
|
||||
}
|
||||
// t_20bfbd9a 治本①:跨 commit 指数退避 —— usageBackoff 第 n 次失败的增长与封顶。
|
||||
func TestUsageBackoff_ExponentialGrowthAndCap(t *testing.T) {
|
||||
// 默认 base=1s, cap=600s
|
||||
one := usageBackoff(1)
|
||||
two := usageBackoff(2)
|
||||
three := usageBackoff(3)
|
||||
ten := usageBackoff(10)
|
||||
eleven := usageBackoff(11)
|
||||
if one != time.Second {
|
||||
t.Fatalf("第1次失败应退避 base(1s), got %v", one)
|
||||
}
|
||||
if two != 2*time.Second {
|
||||
t.Fatalf("第2次失败应退避 2*base(2s), got %v", two)
|
||||
}
|
||||
if three != 4*time.Second {
|
||||
t.Fatalf("第3次失败应退避 4*base(4s), got %v", three)
|
||||
}
|
||||
if ten != 512*time.Second {
|
||||
t.Fatalf("第10次失败应退避 512s(尚未超过 cap 600s), got %v", ten)
|
||||
}
|
||||
if eleven != 600*time.Second {
|
||||
t.Fatalf("第11次失败(下一次翻倍 1024s 将超 cap 600s)应封顶 600s, got %v", eleven)
|
||||
}
|
||||
}
|
||||
|
||||
// t_20bfbd9a 治本②:积压上限量化 —— 未消费待重试行超过 ZHIYI_USAGE_BACKLOG_MAX 时本轮跳过扫描。
|
||||
func TestCheckCommittedForUse_BacklogCapSkipsScan(t *testing.T) {
|
||||
setupUsageTest(t)
|
||||
db := usageOpen()
|
||||
if db == nil {
|
||||
t.Fatal("usageOpen failed")
|
||||
}
|
||||
// 显式压小积压上限,验证超过即跳过
|
||||
t.Setenv("ZHIYI_USAGE_BACKLOG_MAX", "2")
|
||||
now := time.Now().Format(time.RFC3339)
|
||||
// 塞 3 条未消费召回行(超过上限 2)
|
||||
for i := 0; i < 3; i++ {
|
||||
if _, err := db.Exec(`INSERT INTO usage_recall(memory_id, content, tokens, q_hash, namespace, agent, recalled_at)
|
||||
VALUES(?,?,?,?,?,?,?)`,
|
||||
"mem_bk"+string(rune('0'+i)), "牧尘今天要求处理看板任务验收"+string(rune('0'+i)),
|
||||
`["牧尘今天要求处理看板任务验收"]`, "q", "ns", "agent", now); err != nil {
|
||||
t.Fatalf("insert: %v", err)
|
||||
}
|
||||
}
|
||||
// 无需真实 LanceDB(ldb=nil 直接返回 nil flags);本用例只验证「超限即提前跳过」不 panic 且不消费
|
||||
flags := CheckCommittedForUse(nil, "agent", "ns", "牧尘今天要求处理看板任务验收0")
|
||||
if flags != nil {
|
||||
t.Fatalf("ldb=nil 应直接返回 nil flags(超限跳过不应触及消费路径), got %v", flags)
|
||||
}
|
||||
// 判据:超限跳过时所有行仍 consumed=0(没有消费行为发生)
|
||||
var pending int
|
||||
_ = db.QueryRow(`SELECT COUNT(*) FROM usage_recall WHERE consumed=0`).Scan(&pending)
|
||||
if pending != 3 {
|
||||
t.Fatalf("超限跳过扫描后应 3 行全部未消费(no consumption), got %d", pending)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,3 +26,9 @@ default = []
|
||||
[[bin]]
|
||||
name = "zhiyi-consolidate"
|
||||
path = "src/main.rs"
|
||||
|
||||
# t_789898bd:存量 importance 回填一次性工具(只写 importance 列,组提交 ~25 版本)。
|
||||
# 新增 [[bin]] 不影响 zhiyi-consolidate 二进制本身(sidecar 不用重新部署/重启)。
|
||||
[[bin]]
|
||||
name = "importance_backfill"
|
||||
path = "src/bin/importance_backfill.rs"
|
||||
|
||||
@@ -0,0 +1,305 @@
|
||||
//! t_789898bd AC3 存量 importance 回填工具(一次性)
|
||||
//!
|
||||
//! 背景:写入侧 importance 恒硬编码 1.0,全库 46,9xx 行(memories 表)恒 1.0。
|
||||
//! 本工具按 Go 侧 `go/internal/storage/importance_rule.go` 的**同一张分档表**回填 importance 列:
|
||||
//! tier=core → 0.9 ; normal × wiki:*|llm_distill → 0.7 ;
|
||||
//! normal × legacy:*|unknown → 0.4 ; normal × recovery:* → 0.3(预留档)
|
||||
//!
|
||||
//! 写路径约束(AC5 红线):
|
||||
//! · 只写 `importance` 一列(lance update 的 .column("importance", 字面量表达式))
|
||||
//! · 不用 CASE WHEN(0.15 planner 拒绝,实测见 lancedb_ops.rs:638 注释)
|
||||
//! · 按 (目标值, id 分块 IN 谓词) 组提交 —— 复用仓库已验证的
|
||||
//! `id IN (...)` 批量模式(lancedb_update_batch 同款,lancedb_ops.rs:640)
|
||||
//! · 每目标值 ~2000 id/块 ⇒ 全表约 25 个版本(对比 46,874 次单行提交)
|
||||
//! · 提交冲突(与在跑 sidecar 并发写,lance Display 含 "Commit conflict")有界退避重试
|
||||
//! · 不碰 episodes 表 / graph / 不重启 zhiyid;.delete()/optimize() 一律不用
|
||||
//! · 回滚 = `--mode rollback`:整表 importance=1.0(1 个版本),即改动前状态
|
||||
//! · 读侧坑(lancedb 0.15):Query 默认 limit=10 ⇒ 全表扫描必须 count_rows + .limit(n)
|
||||
//!
|
||||
//! 用法:
|
||||
//! importance_backfill --data-dir /path/to/zhiyi/data --mode dry-run
|
||||
//! importance_backfill --data-dir /path/to/zhiyi/data --mode apply
|
||||
//! importance_backfill --data-dir /path/to/zhiyi/data --mode verify
|
||||
//! importance_backfill --data-dir /path/to/zhiyi/data --mode rollback
|
||||
use arrow::array::{Float64Array, StringArray};
|
||||
use arrow::record_batch::RecordBatch;
|
||||
use clap::Parser;
|
||||
use futures::StreamExt;
|
||||
use lancedb::query::{ExecutableQuery, QueryBase, Select};
|
||||
use std::collections::HashMap;
|
||||
use tokio::runtime::Runtime;
|
||||
|
||||
#[derive(Parser)]
|
||||
struct Args {
|
||||
/// LanceDB 数据目录(内含 <table>.lance 数据集目录)
|
||||
#[arg(long)]
|
||||
data_dir: String,
|
||||
/// 目标表(只允许 memories)
|
||||
#[arg(long, default_value = "memories")]
|
||||
table: String,
|
||||
/// dry-run | apply | verify | rollback
|
||||
#[arg(long, default_value = "dry-run")]
|
||||
mode: String,
|
||||
/// IN 谓词每块最大 id 数
|
||||
#[arg(long, default_value_t = 2000u32)]
|
||||
chunk: u32,
|
||||
}
|
||||
|
||||
/// 与 Go storage/importance_rule.go 同源的分档表(t_789898bd 标定)。
|
||||
/// 永不返回 1.0(「恒 1.0 行数 → 0」验收依赖)。
|
||||
fn target_importance(tier: &str, source: &str) -> f64 {
|
||||
if tier == "core" {
|
||||
return 0.9;
|
||||
}
|
||||
if source == "llm_distill" || source.starts_with("wiki:") {
|
||||
return 0.7;
|
||||
}
|
||||
if source.starts_with("recovery:") {
|
||||
return 0.3;
|
||||
}
|
||||
// legacy:* / 未知 / 空 source ⇒ 低置信类
|
||||
0.4
|
||||
}
|
||||
|
||||
fn rt() -> &'static Runtime {
|
||||
static ONCE: std::sync::Once = std::sync::Once::new();
|
||||
static mut RT: Option<Runtime> = None;
|
||||
ONCE.call_once(|| unsafe {
|
||||
RT = Some(Runtime::new().expect("tokio runtime"));
|
||||
});
|
||||
unsafe { RT.as_ref().unwrap() }
|
||||
}
|
||||
|
||||
/// 一次组提交(IN 谓词或全表)+ 提交冲突有界退避重试(lance Display 含 "Commit conflict")。
|
||||
/// 每次重试重建 UpdateBuilder(execute 消费 self)。
|
||||
async fn commit_update(
|
||||
tbl: &lancedb::table::Table,
|
||||
pred: Option<&str>,
|
||||
expr: &str,
|
||||
label: &str,
|
||||
) -> Result<u64, Box<dyn std::error::Error>> {
|
||||
let mut attempt = 0u32;
|
||||
loop {
|
||||
let mut op = tbl.update();
|
||||
if let Some(p) = pred {
|
||||
op = op.only_if(p.to_string());
|
||||
}
|
||||
op = op.column("importance", expr);
|
||||
match op.execute().await {
|
||||
Ok(rows) => {
|
||||
eprintln!("[backfill] commit ok: {} rows={}", label, rows);
|
||||
return Ok(rows);
|
||||
}
|
||||
Err(e) => {
|
||||
attempt += 1;
|
||||
let msg = e.to_string();
|
||||
if !msg.contains("Commit conflict") {
|
||||
return Err(format!("commit failed ({}): {}", label, msg).into());
|
||||
}
|
||||
if attempt >= 10 {
|
||||
return Err(format!("commit conflict ×10 ({}): {}", label, msg).into());
|
||||
}
|
||||
let backoff_ms = 500u64 * 2u64.pow(attempt);
|
||||
eprintln!(
|
||||
"[backfill] commit conflict ({}), retry {}/10 in {}ms",
|
||||
label, attempt, backoff_ms
|
||||
);
|
||||
tokio::time::sleep(std::time::Duration::from_millis(backoff_ms)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 取某行某列的字符串(列缺失/类型不符 = 硬错误,不猜)。
|
||||
fn str_at(b: &RecordBatch, i: usize, col: &str) -> Result<String, String> {
|
||||
Ok(
|
||||
b.column_by_name(col)
|
||||
.ok_or_else(|| format!("column {} missing", col))?
|
||||
.as_any()
|
||||
.downcast_ref::<StringArray>()
|
||||
.ok_or_else(|| format!("column {} not string", col))?
|
||||
.value(i)
|
||||
.to_string(),
|
||||
)
|
||||
}
|
||||
|
||||
/// f64 不是 HashMap 键(无 Eq)⇒ 用 ×10 的 i32 做分档键(0.9→9, 0.7→7, 0.4→4, 0.3→3)。
|
||||
fn vkey(v: f64) -> i32 {
|
||||
(v * 10.0).round() as i32
|
||||
}
|
||||
|
||||
/// 全表拉 (id, tier, source, importance)。注意:Query 默认 limit=10 ⇒ 必须显式 .limit(count)。
|
||||
async fn scan_rows(
|
||||
tbl: &lancedb::table::Table,
|
||||
) -> Result<Vec<(String, String, String, f64)>, Box<dyn std::error::Error>> {
|
||||
let count = tbl.count_rows(Some("id IS NOT NULL".into())).await? as usize;
|
||||
let mut results = Box::pin(
|
||||
tbl.query()
|
||||
.select(Select::columns(&["id", "tier", "source", "importance"]))
|
||||
.only_if("id IS NOT NULL")
|
||||
.limit(count)
|
||||
.execute()
|
||||
.await?,
|
||||
);
|
||||
let mut out: Vec<(String, String, String, f64)> = Vec::new();
|
||||
while let Some(b) = results.next().await {
|
||||
let batch = b?;
|
||||
for i in 0..batch.num_rows() {
|
||||
let id = str_at(&batch, i as usize, "id")?;
|
||||
let tier = str_at(&batch, i as usize, "tier")?;
|
||||
let src = str_at(&batch, i as usize, "source")?;
|
||||
let imp = batch
|
||||
.column_by_name("importance")
|
||||
.ok_or("importance column missing")?
|
||||
.as_any()
|
||||
.downcast_ref::<Float64Array>()
|
||||
.ok_or("importance column not float64")?
|
||||
.value(i as usize);
|
||||
out.push((id, tier, src, imp));
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
async fn run(args: Args) -> Result<(), Box<dyn std::error::Error>> {
|
||||
if args.table != "memories" {
|
||||
return Err(format!(
|
||||
"refusing table={}: 本卡红线只允许回填 memories(不碰 episodes.lance)",
|
||||
args.table
|
||||
)
|
||||
.into());
|
||||
}
|
||||
let db = lancedb::connect(&args.data_dir).execute().await?;
|
||||
let tbl = db.open_table(&args.table).execute().await?;
|
||||
|
||||
// ── 1) 只读拉全表 ──
|
||||
let rows = scan_rows(&tbl).await?;
|
||||
let n = rows.len() as u64;
|
||||
let imp10_before = rows.iter().filter(|r| r.3 == 1.0).count();
|
||||
eprintln!(
|
||||
"[backfill] table={} rows={} importance==1.0 before = {}",
|
||||
args.table, n, imp10_before
|
||||
);
|
||||
|
||||
match args.mode.as_str() {
|
||||
"dry-run" => {
|
||||
let mut plan: HashMap<i32, usize> = HashMap::new();
|
||||
for (_, tier, src, _) in &rows {
|
||||
*plan.entry(vkey(target_importance(tier, src))).or_insert(0) += 1;
|
||||
}
|
||||
eprintln!("[backfill] plan (chunks per group, chunk={}):", args.chunk);
|
||||
let mut plan_vec: Vec<(i32, &usize)> = plan
|
||||
.iter()
|
||||
.map(|(k, c)| (*k, c))
|
||||
.collect();
|
||||
plan_vec.sort();
|
||||
for (k, c) in plan_vec {
|
||||
let chunks = (*c as u64 + args.chunk as u64 - 1) / args.chunk as u64;
|
||||
eprintln!(
|
||||
" importance={} → {} rows, {} chunks",
|
||||
k as f64 / 10.0, c, chunks
|
||||
);
|
||||
}
|
||||
eprintln!("[backfill] dry-run: no commit.");
|
||||
Ok(())
|
||||
}
|
||||
"verify" => {
|
||||
// 只读验收:逐行对规则复算 + 1.0 计数 + 直方图
|
||||
let mut mismatch: u64 = 0;
|
||||
let mut one: u64 = 0;
|
||||
let mut hist: HashMap<i32, u64> = HashMap::new();
|
||||
for (id, tier, src, imp) in &rows {
|
||||
let want = target_importance(tier, src);
|
||||
if (imp - want).abs() > 1e-9 {
|
||||
mismatch += 1;
|
||||
if mismatch <= 5 {
|
||||
eprintln!(
|
||||
"[verify] MISMATCH id={} tier={} source={} imp={} want={}",
|
||||
id, tier, src, imp, want
|
||||
);
|
||||
}
|
||||
}
|
||||
if *imp == 1.0 {
|
||||
one += 1;
|
||||
}
|
||||
*hist.entry(vkey(*imp)).or_insert(0) += 1;
|
||||
}
|
||||
eprintln!(
|
||||
"[verify] rows={} mismatch={}/{} importance==1.0={}",
|
||||
rows.len(),
|
||||
mismatch,
|
||||
n,
|
||||
one
|
||||
);
|
||||
eprintln!("[verify] histogram:");
|
||||
let mut hist_vec: Vec<(i32, &u64)> = hist
|
||||
.iter()
|
||||
.map(|(k, c)| (*k, c))
|
||||
.collect();
|
||||
hist_vec.sort();
|
||||
for (k, c) in hist_vec {
|
||||
eprintln!(
|
||||
" {} : {} ({}%)",
|
||||
k as f64 / 10.0,
|
||||
c,
|
||||
*c as f64 * 100.0 / n as f64
|
||||
);
|
||||
}
|
||||
if mismatch == 0 {
|
||||
eprintln!("[verify] PASS: 全部行 importance == 规则值");
|
||||
} else {
|
||||
return Err(format!(
|
||||
"[verify] FAIL: {} rows mismatch(新写入行可能是 1.0 漂移,可重跑 apply 收敛)",
|
||||
mismatch
|
||||
)
|
||||
.into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
"rollback" => {
|
||||
// 回滚 = 整表 importance=1.0(1 个版本)= 改动前状态
|
||||
let rows_upd = commit_update(&tbl, None, "1.0", "rollback(all→1.0)").await?;
|
||||
eprintln!("[backfill] rollback done, updated rows={}", rows_upd);
|
||||
Ok(())
|
||||
}
|
||||
"apply" => {
|
||||
// 组提交:(目标值, id 分块) → 每块一次 update(IN 谓词 + importance 字面量)
|
||||
let mut by_value: HashMap<i32, Vec<String>> = HashMap::new();
|
||||
for (id, tier, src, _) in &rows {
|
||||
by_value
|
||||
.entry(vkey(target_importance(tier, src)))
|
||||
.or_default()
|
||||
.push(id.clone());
|
||||
}
|
||||
let mut groups: Vec<(i32, Vec<String>)> = by_value.into_iter().collect();
|
||||
groups.sort();
|
||||
for (k, vids) in &groups {
|
||||
let lit = format!("{}", *k as f64 / 10.0);
|
||||
for chunk in vids.chunks(args.chunk as usize) {
|
||||
let in_list: Vec<String> = chunk
|
||||
.iter()
|
||||
.map(|id| format!("'{}'", id.replace('\'', "''")))
|
||||
.collect();
|
||||
let pred = format!("id IN ({})", in_list.join(", "));
|
||||
let label = format!(
|
||||
"importance={} ({} ids)",
|
||||
*k as f64 / 10.0,
|
||||
chunk.len()
|
||||
);
|
||||
let _rows = commit_update(&tbl, Some(&pred), &lit, &label).await?;
|
||||
}
|
||||
}
|
||||
eprintln!("[backfill] apply done. run --mode verify to check.");
|
||||
Ok(())
|
||||
}
|
||||
other => Err(format!("unknown mode: {}", other).into()),
|
||||
}
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let args = Args::parse();
|
||||
if let Err(e) = rt().block_on(run(args)) {
|
||||
eprintln!("[backfill] FAILED: {}", e);
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
+97
-3
@@ -154,6 +154,65 @@ impl HeavyReadGate {
|
||||
}
|
||||
}
|
||||
|
||||
// ─── 连接并发上限 + RSS 护栏(t_20bfbd9a 治本)────────────────────────
|
||||
// 事故链(t_20bfbd9a 收口):accept 循环对每个 TCP 连接 thread::spawn,无并发上限;
|
||||
// Go 侧每 rpc 新 dial(lancedb_ipc.go rpc() → net.DialTimeout),IPC 洪泛 → 线程
|
||||
// 线性涨到 736(实测),每线程持独立 stream + LanceDB 批次内存 ⇒ 5.1G 单进程。
|
||||
// 治本①连接并发上限:AtomicUsize in_flight 计数,spawn 前 CAS +1,超上限 ⇒ 拒绝
|
||||
// (回 heavy_conn_busy);线程结束 RAII Drop 归还。轻热路径(ping/insert/update)
|
||||
// 在正常流量下永不超过上限,不改变单写者语义。
|
||||
// 治本②RSS 护栏:每 accept 采样 VmRSS,超上限 ⇒ 降级到串行(并发 1,inline 处理
|
||||
// + 拒重读),不再叠加线程;低于上限自动恢复。上限 env ZHIYI_RSS_LIMIT_MB 可调。
|
||||
static CONN_MAX_DEFAULT: usize = 64;
|
||||
static RSS_LIMIT_MB_DEFAULT: u64 = 3000;
|
||||
|
||||
fn conn_permits() -> usize {
|
||||
match std::env::var("ZHIYI_CONN_MAX") {
|
||||
Ok(v) => v.trim().parse::<usize>().unwrap_or(CONN_MAX_DEFAULT).max(1),
|
||||
Err(_) => CONN_MAX_DEFAULT,
|
||||
}
|
||||
}
|
||||
|
||||
fn rss_limit_kb() -> u64 {
|
||||
match std::env::var("ZHIYI_RSS_LIMIT_MB") {
|
||||
Ok(v) => v.trim().parse::<u64>().unwrap_or(RSS_LIMIT_MB_DEFAULT).max(64) * 1024,
|
||||
Err(_) => RSS_LIMIT_MB_DEFAULT * 1024,
|
||||
}
|
||||
}
|
||||
|
||||
/// 连接许可 RAII 守卫:Drop 自动归还 in_flight(线程正常返回或 panic 都不泄漏)。
|
||||
struct ConnPermit {
|
||||
counter: Arc<AtomicUsize>,
|
||||
}
|
||||
impl Drop for ConnPermit {
|
||||
fn drop(&mut self) {
|
||||
self.counter.fetch_sub(1, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
/// 尝试登记一个并发连接;有槽 ⇒ Some(许可),满 ⇒ None(调用方拒绝/降级)。
|
||||
fn conn_try_acquire(counter: &Arc<AtomicUsize>, limit: usize) -> Option<ConnPermit> {
|
||||
let mut cur = counter.load(Ordering::SeqCst);
|
||||
loop {
|
||||
if cur >= limit {
|
||||
return None;
|
||||
}
|
||||
match counter.compare_exchange_weak(
|
||||
cur,
|
||||
cur + 1,
|
||||
Ordering::SeqCst,
|
||||
Ordering::SeqCst,
|
||||
) {
|
||||
Ok(_) => {
|
||||
return Some(ConnPermit {
|
||||
counter: Arc::clone(counter),
|
||||
})
|
||||
}
|
||||
Err(actual) => cur = actual,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ─── 内存回收 & 观测(2026-09-21 治本)────────────────────────────────
|
||||
// 根因:长驻 sidecar 里每次操作的大块分配(Arrow/LanceDB 批次、Vec<MemoryRecord>)
|
||||
// 释放后 glibc 不还给 OS(arena 保留)→ RSS 只涨不落,5 小时累积到 1.09G
|
||||
@@ -359,9 +418,12 @@ fn run_socket_server(socket_path: &str, args: &Args) {
|
||||
encoder,
|
||||
heavy_read: HeavyReadGate::new(n),
|
||||
});
|
||||
let conn_limit = conn_permits();
|
||||
let conn_in_flight = Arc::new(AtomicUsize::new(0));
|
||||
eprintln!(
|
||||
"[zhiyi] listening on {} (lancedb: {}), heavy-read admission N={} (env ZHIYI_HEAVY_READ_CONCURRENCY)",
|
||||
socket_path, args.data_dir, n
|
||||
"[zhiyi] listening on {} (lancedb: {}), heavy-read admission N={} (env ZHIYI_HEAVY_READ_CONCURRENCY), \
|
||||
conn cap={} (env ZHIYI_CONN_MAX), rss limit={}MB (env ZHIYI_RSS_LIMIT_MB)",
|
||||
socket_path, args.data_dir, n, conn_limit, rss_limit_kb() / 1024
|
||||
);
|
||||
|
||||
for stream in listener.incoming() {
|
||||
@@ -370,7 +432,39 @@ fn run_socket_server(socket_path: &str, args: &Args) {
|
||||
let a = args.clone();
|
||||
let ldb = lancedb.clone();
|
||||
let st = Arc::clone(&state);
|
||||
thread::spawn(move || handle_client(stream, &a, ldb, &st));
|
||||
// RSS 护栏(治本②):超上限 ⇒ 降级到串行(并发 1,inline 处理 + 拒重读),
|
||||
// 不再叠加线程;低于上限自动恢复。这是对「无内存护栏」事故的直接兜底。
|
||||
let (rss, _) = rss_kb();
|
||||
if rss > rss_limit_kb() {
|
||||
eprintln!(
|
||||
"[zhiyi] RSS {}MB > limit {}MB: degrade to serial (concurrency 1)",
|
||||
rss / 1024,
|
||||
rss_limit_kb() / 1024
|
||||
);
|
||||
handle_client(stream, &a, ldb, &st);
|
||||
continue;
|
||||
}
|
||||
// 连接并发上限(治本①):CAS +1,满 ⇒ 回 heavy_conn_busy 并拒绝。
|
||||
let permit = match conn_try_acquire(&conn_in_flight, conn_limit) {
|
||||
Some(p) => p,
|
||||
None => {
|
||||
eprintln!(
|
||||
"[zhiyi] connection cap {} reached: reject (heavy_conn_busy)",
|
||||
conn_limit
|
||||
);
|
||||
// 直接回一个 JSON 错误给客户端(对齐 Go rpc() 的读 deadline)
|
||||
let mut stream = stream;
|
||||
let _ = stream.write_all(b"\x00\x00\x00\x46");
|
||||
let _ = stream.write_all(
|
||||
br#"{"status":"error","error":"heavy_conn_busy","detail":"connection cap exhausted"}"#,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
thread::spawn(move || {
|
||||
handle_client(stream, &a, ldb, &st);
|
||||
drop(permit); // RAII 归还 in_flight(也兜 panic 展开)
|
||||
});
|
||||
}
|
||||
Err(e) => eprintln!("[zhiyi] accept error: {}", e),
|
||||
}
|
||||
|
||||
@@ -4,4 +4,8 @@
|
||||
# 追加方式:评审通过后 `./scripts/deploy.sh --mark-approved`(自动把当前 HEAD 追加进来)。
|
||||
# 切勿手工删行——删除会让被撤批的提交重新进入「夹带」判定。
|
||||
# 初始基线(2026-10-17 t_3f83df64 建):600fb2f = 当前已评审的 t_690cc333 HEAD。
|
||||
600fb2fed60f6ee2ea31ec6c41a5ee4e4e08a386
|
||||
600fb2fed60f6ee2ea31ec6c41a5ee4e4e08a386
|
||||
b8f7ddb53ce056a1ac86468a8e9a7557c71922f0
|
||||
c57f557ab8d126c7ef031e63402abaa1f6602155
|
||||
aaeaf2b7ec9304b851089cbe64ddda0a5916d99f
|
||||
a2be088cba59c95901b22d855dea9b6df3bf3a80
|
||||
|
||||
Reference in New Issue
Block a user