diff --git a/go/internal/storage/usage_cursor_test.go b/go/internal/storage/usage_cursor_test.go new file mode 100644 index 0000000..5f167af --- /dev/null +++ b/go/internal/storage/usage_cursor_test.go @@ -0,0 +1,215 @@ +package storage + +// t_9143c968 消费侧熔断修复:游标调度测试。 +// 覆盖 AC5(窗口外 1000 pending + 窗口内 1 行必须被处理)、tail/sweep 游标、 +// 失败熔断(替代旧「积压>cap⇒skip」)、retry 通道、expire 标记与 TTL。 + +import ( + "database/sql" + "encoding/json" + "errors" + "strconv" + "testing" + "time" +) + +// flakyLance 包装任意 LanceDB:failAppend=true ⇒ AppendHitEvent 恒失败(模拟持久性坏行)。 +type flakyLance struct { + LanceDB + failAppend bool +} + +func (f *flakyLance) AppendHitEvent(id, outcome, reason string) error { + if f.failAppend { + return errors.New("simulated commit conflict") + } + return f.LanceDB.AppendHitEvent(id, outcome, reason) +} + +// insertRecallRowAt 直插一条召回行(recalled_at 可控,绕开 RecordRecalled 的 now)。 +func insertRecallRowAt(t *testing.T, db *sql.DB, memID, content, recalledAt string) { + t.Helper() + toks, _ := json.Marshal(DistinctiveTokens(content, usageTokenTopN)) + if _, err := db.Exec(`INSERT INTO usage_recall(memory_id,content,tokens,q_hash,namespace,agent,recalled_at) + VALUES(?,?,?,?,?,?,?)`, memID, content, string(toks), "q", "ns", "agent", recalledAt); err != nil { + t.Fatalf("insert recall row: %v", err) + } +} + +const usageTestMem = "家庭服务器IP固定为192.168.123.11,星空组网走VPN 192.168.188.10" +const usageTestCommit = "家庭服务器IP是192.168.123.11,星空组网用VPN 192.168.188.10 访问" + +func usagePendingBy(t *testing.T, db *sql.DB, where string, args ...interface{}) int { + t.Helper() + var n int + if err := db.QueryRow(`SELECT COUNT(*) FROM usage_recall WHERE `+where, args...).Scan(&n); err != nil { + t.Fatalf("count(%s): %v", where, err) + } + return n +} + +// AC5:窗口外 1000 行 pending + 窗口内 1 行 ⇒ 窗口内那 1 行必须被消费。 +// (旧实现在此场景恒 skip:pending 1001 > cap 500 ⇒ 每 commit「backlog > cap」——正是本次熔断本身。) +func TestUsageCursor_A5OldBacklogMustNotBlockNewRow(t *testing.T) { + setupUsageTest(t) + db := usageOpen() + if db == nil { + t.Fatal("usageOpen failed") + } + mlc := NewMemLanceClient(nil) + now := time.Now() + old := now.Add(-72 * time.Hour).Format(time.RFC3339) + fresh := now.Format(time.RFC3339) + for i := 0; i < 1000; i++ { + insertRecallRowAt(t, db, "mem_old"+strconv.Itoa(i), "旧内容占位记忆"+strconv.Itoa(i), old) + } + insertRecallRowAt(t, db, "mem_new", usageTestMem, fresh) + + start := time.Now() + flags := CheckCommittedForUse(mlc, "agent", "ns", usageTestCommit) + t.Logf("1001 行积压下 单轮耗时 %v", time.Since(start)) + if len(flags) != 1 || flags[0].MemoryID != "mem_new" { + t.Fatalf("窗口内 1 行必须被消费(旧积压闸已删),got %+v", flags) + } + // 窗口外 1000 行:同轮被 expire 通道标 consumed=2(skipped_expired,留证据不物理删) + if c2 := usagePendingBy(t, db, "consumed=2"); c2 != 1000 { + t.Fatalf("窗口外 pending 应全标 consumed=2, actual %d", c2) + } + if p := usagePendingBy(t, db, "consumed=0"); p != 0 { + t.Fatalf("不应再有 consumed=0 残留, actual %d", p) + } + // 行仍在库里(证据保留):总量 = 1001 + if total := usagePendingBy(t, db, "1=1"); total != 1001 { + t.Fatalf("expire 只标记不删除, total=%d", total) + } +} + +// tail 只吃游标之上的有界批次(LAG),落后行由 sweep 逐轮补齐;全部行最终各消费一次。 +func TestUsageCursor_TailBoundedSweepCovers(t *testing.T) { + setupUsageTest(t) + t.Setenv("ZHIYI_USAGE_BATCH", "3") + t.Setenv("ZHIYI_USAGE_SWEEP_BATCH", "2") + db := usageOpen() + mlc := NewMemLanceClient(nil) + now := time.Now().Format(time.RFC3339) + for i := 1; i <= 10; i++ { + insertRecallRowAt(t, db, "mem_c"+strconv.Itoa(i), usageTestMem, now) + } + // call1:tail 取最新 3 行(ids 10,9,8);sweep 从头取 2 行(ids 1,2)⇒ 5 flags + if f := CheckCommittedForUse(mlc, "agent", "ns", usageTestCommit); len(f) != 5 { + t.Fatalf("call1 期望 5 flags (tail3+sweep2), got %d", len(f)) + } + if c := usagePendingBy(t, db, "consumed=1"); c != 5 { + t.Fatalf("call1 后应消费 5 行, actual %d", c) + } + // 继续调用直至全部消费;每次调用都是新 commit,行只能消费一次 + for round := 2; round <= 10; round++ { + CheckCommittedForUse(mlc, "agent", "ns", usageTestCommit) + } + if c := usagePendingBy(t, db, "consumed=1"); c != 10 { + t.Fatalf("多轮后应全部 10 行消费, actual %d", c) + } + if p := usagePendingBy(t, db, "consumed=0"); p != 0 { + t.Fatalf("不应有 pending 残留, actual %d", p) + } + // 每行只 1 条 hit event(无重复累加) + for i := 1; i <= 10; i++ { + ev, _ := mlc.GetHitEvents("mem_c"+strconv.Itoa(i), 10) + if len(ev) != 1 { + t.Fatalf("mem_c%d 应恰 1 条 hit event, got %d", i, len(ev)) + } + } +} + +// 失败熔断(ZHIYI_USAGE_BACKLOG_MAX 新语义):单轮失败 ≥cap ⇒ 停,不再全表跳过。 +// 反证旧行为:pending 6 > cap 2 在旧实现 = 「backlog > cap skip」0 行处理; +// 新实现必须实际处理(失败行记退避,坏行不再每 commit 热重试,也不再拖死好行)。 +func TestUsageCursor_FailureBreakerReplacesBacklogSkip(t *testing.T) { + setupUsageTest(t) + t.Setenv("ZHIYI_USAGE_BACKLOG_MAX", "2") + db := usageOpen() + mlc := NewMemLanceClient(nil) + flaky := &flakyLance{LanceDB: mlc, failAppend: true} + now := time.Now().Format(time.RFC3339) + for i := 1; i <= 6; i++ { + insertRecallRowAt(t, db, "mem_bk"+strconv.Itoa(i), usageTestMem, now) + } + flags := CheckCommittedForUse(flaky, "agent", "ns", usageTestCommit) + if len(flags) != 0 { + t.Fatalf("恒失败不应有 flag, got %d", len(flags)) + } + // tail DESC 先吃 ids 6,5 → 2 次失败即熔断;sweep 被单次调用熔断跳过 ⇒ 仅 2 行记退避 + if n := usagePendingBy(t, db, "retry_count=1"); n != 2 { + t.Fatalf("熔断应只记 2 行退避(单轮 cap=2), actual %d", n) + } + if n := usagePendingBy(t, db, "retry_count=0 AND consumed=0"); n != 4 { + t.Fatalf("熔断后其余 4 行应未被触碰, actual %d", n) + } + if n := usagePendingBy(t, db, "consumed=1"); n != 0 { + t.Fatalf("失败行不得消费, actual %d", n) + } + // 关键反证:旧「pending(6)>cap(2)⇒skip」在此场景 0 行处理;新实现推进了 2 行状态。 +} + +// retry 通道:游标推过头的失败行,退避到期后仍能被重试消费(不依赖 tail 位置)。 +func TestUsageCursor_RetryPassAfterBackoff(t *testing.T) { + setupUsageTest(t) + db := usageOpen() + mlc := NewMemLanceClient(nil) + flaky := &flakyLance{LanceDB: mlc, failAppend: true} + now := time.Now().Format(time.RFC3339) + insertRecallRowAt(t, db, "mem_retry", usageTestMem, now) + + if f := CheckCommittedForUse(flaky, "agent", "ns", usageTestCommit); len(f) != 0 { + t.Fatalf("第一次应失败, got %d", len(f)) + } + if c := usagePendingBy(t, db, "consumed=1"); c != 0 { + t.Fatalf("失败不应消费, actual %d", c) + } + // 手动把 next_retry_at 拨到过去(模拟退避到期;包级 backoff 参数在 init 时已定) + if _, err := db.Exec(`UPDATE usage_recall SET next_retry_at='2000-01-01T00:00:00Z' WHERE memory_id='mem_retry'`); err != nil { + t.Fatalf("backdate retry: %v", err) + } + flaky.failAppend = false // 故障恢复 + f := CheckCommittedForUse(mlc, "agent", "ns", usageTestCommit) + if len(f) != 1 || f[0].MemoryID != "mem_retry" { + t.Fatalf("退避到期后 retry 通道必须消费该行, got %+v", f) + } + if c := usagePendingBy(t, db, "consumed=1 AND retry_count=0"); c != 1 { + t.Fatalf("成功消费应把 retry_count 归零, actual %d", c) + } +} + +// expire + TTL:出窗 pending 标 2(72h,未到 7d 不删);consumed>=1 且 >7d 删除;新鲜行不动。 +func TestUsageCursor_ExpireMarkAndTTL(t *testing.T) { + setupUsageTest(t) + db := usageOpen() + mlc := NewMemLanceClient(nil) + now := time.Now() + insertRecallRowAt(t, db, "mem_expired_pending", "旧内容占位记忆甲", now.Add(-72*time.Hour).Format(time.RFC3339)) + insertRecallRowAt(t, db, "mem_stale_consumed", "旧内容占位记忆乙", now.Add(-8*24*time.Hour).Format(time.RFC3339)) + if _, err := db.Exec(`UPDATE usage_recall SET consumed=1 WHERE memory_id IN ('mem_stale_consumed')`); err != nil { + t.Fatalf("mark consumed: %v", err) + } + insertRecallRowAt(t, db, "mem_fresh_consumed", "旧内容占位记忆丙", now.Add(-1*time.Hour).Format(time.RFC3339)) + if _, err := db.Exec(`UPDATE usage_recall SET consumed=1 WHERE memory_id='mem_fresh_consumed'`); err != nil { + t.Fatalf("mark consumed: %v", err) + } + insertRecallRowAt(t, db, "mem_fresh_pending", "旧内容占位记忆丁", now.Add(-1*time.Hour).Format(time.RFC3339)) + + // 不相关内容 ⇒ 无判定,但 expire/TTL 维护通道照常运行 + CheckCommittedForUse(mlc, "agent", "ns", "今天天气很好出去散步") + + if n := usagePendingBy(t, db, "memory_id='mem_expired_pending' AND consumed=2"); n != 1 { + t.Fatalf("出窗 pending 应标 consumed=2(且 72h<7d 不得删除), rows=%d", n) + } + if n := usagePendingBy(t, db, "memory_id='mem_stale_consumed'"); n != 0 { + t.Fatalf("consumed>=1 且 recalled_at>7d 应被 TTL 删除, rows=%d", n) + } + if n := usagePendingBy(t, db, "memory_id='mem_fresh_consumed' AND consumed=1"); n != 1 { + t.Fatalf("新鲜 consumed 行不得删, rows=%d", n) + } + if n := usagePendingBy(t, db, "memory_id='mem_fresh_pending' AND consumed=0"); n != 1 { + t.Fatalf("窗内 pending 不得被动, rows=%d", n) + } +} diff --git a/go/internal/storage/usage_track.go b/go/internal/storage/usage_track.go index ef60f65..67c2020 100644 --- a/go/internal/storage/usage_track.go +++ b/go/internal/storage/usage_track.go @@ -29,6 +29,7 @@ import ( "log" "os" "sort" + "strconv" "strings" "sync" "time" @@ -154,6 +155,20 @@ func usageOpen() *sql.DB { } _, _ = db.Exec(`CREATE INDEX IF NOT EXISTS idx_usage_mem ON usage_recall(memory_id)`) _, _ = db.Exec(`CREATE INDEX IF NOT EXISTS idx_usage_time ON usage_recall(recalled_at)`) + // 消费游标(t_9143c968):单行表,id 恒为 1。tail_pos=尾部已消费位点,sweep_pos=轮巡位点。 + if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS usage_cursor( + id INTEGER PRIMARY KEY CHECK(id=1), + tail_pos INTEGER NOT NULL DEFAULT 0, + sweep_pos INTEGER NOT NULL DEFAULT 0, + updated_at TEXT)`); err != nil { + logUsage("create cursor table: %v", err) + } + // 部分索引(t_9143c968):让消费端单轮扫描代价 O(batch),与表总量解耦。 + // ⚠️ 生产 5M+ 行库的这三条索引须在停机窗口预建(sqlite3 CLI), + // 否则首次 usageOpen 会在请求路径里建索引卡几秒;IF NOT EXISTS 双保险。 + _, _ = db.Exec(`CREATE INDEX IF NOT EXISTS idx_usage_pending ON usage_recall(id, recalled_at) WHERE consumed=0`) + _, _ = db.Exec(`CREATE INDEX IF NOT EXISTS idx_usage_pending_time ON usage_recall(recalled_at) WHERE consumed=0`) + _, _ = db.Exec(`CREATE INDEX IF NOT EXISTS idx_usage_retry ON usage_recall(id) WHERE next_retry_at IS NOT NULL`) usageDB = db }) return usageDB @@ -198,44 +213,72 @@ func RecordRecalled(recs []RecalledMemory) { } } -// CheckCommittedForUse commit 写侧钩子:检查新写入的内容是否携带了 -// 最近被召回记忆的独特 token;命中则判定「被用上」并落 outcome(门控)。 -// 返回本次判定出的「被用上」记忆(供上报/取证)。 -func CheckCommittedForUse(ldb LanceDB, agent, namespace, content string) []UsageFlag { - if !usageTrackEnabled || ldb == nil || content == "" { - return nil - } - db := usageOpen() - if db == nil { - return nil - } - // 积压上限量化(t_20bfbd9a 治本②):未消费待重试行超过上限 ⇒ 本轮跳过扫描, - // 防止「一条永远失败的坏行 + 高 commit 频率」在每次 commit 都全量重扫、叠成风暴。 - // 上限 env ZHIYI_USAGE_BACKLOG_MAX,默认 500。 - backlogMax := int(envFloat("ZHIYI_USAGE_BACKLOG_MAX", 500)) - if backlogMax > 0 { - var pending int - if err := db.QueryRow(`SELECT COUNT(*) FROM usage_recall WHERE consumed=0`).Scan(&pending); err == nil && pending > backlogMax { - logUsage("backlog %d > cap %d: skip scan this commit", pending, backlogMax) - return nil +// ─── 消费游标与批量消费(t_9143c968 熔断修复,2026-10-30)──────────── +// +// 根因(卡 body + comment #1757/#1758 两次修正):旧实现扫描前用无窗 count(consumed=0) +// 作积压熔断(cap 500,t_20bfbd9a)。写入量涨到 40~200 万行/日后 pending 恒百万级 +// ⇒ 每次 commit 都命中「backlog > cap ⇒ skip」——消费侧永久短路(10-19 起 consumed=0)。 +// cap 是安全阀不是 bug;真 bug 是「消费无增量位点,只能每次 commit 全量扫窗」。 +// +// 新方案(卡推荐 A:增量游标;单轮代价 O(batch),与表总量解耦): +// - expire 过期标记:consumed=0 且已出窗(recalled_at=1(含 2)且 recalled_at 0 { + return n } } - cutoff := time.Now().Add(-time.Duration(usageWindowHours * float64(time.Hour))).Format(time.RFC3339) - scanNow := time.Now() - // 退避过滤(t_20bfbd9a 治本①):next_retry_at 为 NULL(首次待处理)或已到重试时间, - // 才进本轮重扫。持久性失败的坏行被指数退避挡在窗口外,不再每 commit 热重试。 - rows, err := db.Query( - `SELECT id, memory_id, content, tokens, namespace, agent, retry_count - FROM usage_recall WHERE consumed=0 AND recalled_at>=? - AND (next_retry_at IS NULL OR next_retry_at<=?)`, - cutoff, scanNow.Format(time.RFC3339)) + return def +} + +// usageScanBatch 执行一条批次查询并逐行判定,返回(判定样本, 失败行数, 本批最大 id)。 +// query 须选出 id,memory_id,content,tokens,namespace,agent,retry_count 七列。 +// 单行失败(AppendHitEvent/Update 终态失败)⇒ 不消费 + 指数退避(t_20bfbd9a 语义不变); +// 失败累计 ≥ ZHIYI_USAGE_BACKLOG_MAX(>0 时)⇒ 熔断:本轮提前结束。 +func usageScanBatch(db *sql.DB, ldb LanceDB, agent, namespace string, contentSet map[string]bool, + query string, args ...interface{}) (flags []UsageFlag, failed int, maxID int64) { + rows, err := db.Query(query, args...) if err != nil { - logUsage("query recall log: %v", err) - return nil + logUsage("query recall batch: %v", err) + return } defer rows.Close() - contentSet := tokenSet(content) - flags := make([]UsageFlag, 0) + breakerMax := int(envFloat("ZHIYI_USAGE_BACKLOG_MAX", 500)) + now := time.Now().Format(time.RFC3339) for rows.Next() { var id int64 var memID, memContent, tokensJSON, ns, ag string @@ -243,6 +286,9 @@ func CheckCommittedForUse(ldb LanceDB, agent, namespace, content string) []Usage if err := rows.Scan(&id, &memID, &memContent, &tokensJSON, &ns, &ag, &retryCount); err != nil { continue } + if id > maxID { + maxID = id + } // 归属过滤:同一 agent,或同一 namespace,或共享/全局。跨域不强求。 if ag != "" && agent != "" && ag != agent && ns != "" && namespace != "" && ns != namespace { continue @@ -256,43 +302,159 @@ func CheckCommittedForUse(ldb LanceDB, agent, namespace, content string) []Usage continue } // 判定「被用上」:落 outcome + 累加 recall_verified_count + 消费该 recall 行。 - now := time.Now().Format(time.RFC3339) - // t_7793a484:outcome 事件失败不再静默吞(修复前 `_ =` 丢错 + 单次 rpc 无重试 ⇒ - // 一次冲突即终态丢,10-16 实测 42 条丢失走此路径)。现:AppendHitEvent 内置 - // 有界重试(同 Update 规格);终态失败(耗尽/非可重试)⇒ 不消费(consumed 保持 0) - // ⇒ 下一轮 commit 重扫重试,事件不静默丢。失败可查(logUsage 计数 + 计数器 - // zhiyi_hit_event_append_failed_total),禁静默。 + // t_7793a484:outcome 终态失败 ⇒ 不消费 + 指数退避(t_20bfbd9a),下一轮重扫 + // 重试,事件不静默丢。t_28aa278e 方案①:Update 成功后才消费,计数增量不丢。 if err := ldb.AppendHitEvent(memID, HitOutcomePos, HitReasonDownstreamUse); err != nil { - logUsage("hit event append FAILED (id=%s, NOT consumed, will retry next commit) : %v", memID, err) - // t_20bfbd9a 治本①:持久性失败(如坏行 memID 已从 LanceDB 删除)若不加退避, - // 会在每次 commit 都热重试,commit 风暴时叠成重扫风暴。加指数退避: - // 第 n 次失败后 next_retry_at = now + min(2^(n-1) * base, cap),退避期外不再重扫。 - // 成功消费时把 retry_count 归零;每轮只推进 retry_count 一次(不指数级暴涨)。 - _, _ = db.Exec( - `UPDATE usage_recall SET retry_count=retry_count+1, next_retry_at=? WHERE id=?`, + logUsage("hit event append FAILED (id=%s, NOT consumed, will retry) : %v", memID, err) + _, _ = db.Exec(`UPDATE usage_recall SET retry_count=retry_count+1, next_retry_at=? WHERE id=?`, time.Now().Add(usageBackoff(retryCount+1)).Format(time.RFC3339), id) + failed++ + if breakerMax > 0 && failed >= breakerMax { + logUsage("failure breaker: %d failures >= cap %d, abort this call", failed, breakerMax) + return + } continue } - // t_28aa278e 方案①:更新成功后才消费 usage_recall 行。 - // 旧实现无条件消费 ⇒ Update 冲突耗尽时该 recall_verified_count 增量被永久丢弃 - // (下轮不再重扫)。现改为:Update 返回非 nil(冲突重试耗尽/非可重试错误) - // ⇒ 不消费(consumed 保持 0)⇒ 下一轮 commit 重扫该窗口会重试,计数增量不静默丢失。 - // 失败数可查(logUsage 计数),禁静默。 if err := ldb.Update("memories", memID, map[string]any{ "recall_verified_count": map[string]string{"$inc": "1"}, }); err != nil { - logUsage("recall_verified_count update FAILED (id=%s, NOT consumed, will retry next commit) : %v", memID, err) - _, _ = db.Exec( - `UPDATE usage_recall SET retry_count=retry_count+1, next_retry_at=? WHERE id=?`, + logUsage("recall_verified_count update FAILED (id=%s, NOT consumed, will retry) : %v", memID, err) + _, _ = db.Exec(`UPDATE usage_recall SET retry_count=retry_count+1, next_retry_at=? WHERE id=?`, time.Now().Add(usageBackoff(retryCount+1)).Format(time.RFC3339), id) + failed++ + if breakerMax > 0 && failed >= breakerMax { + logUsage("failure breaker: %d failures >= cap %d, abort this call", failed, breakerMax) + return + } + continue + } + if _, err := db.Exec(`UPDATE usage_recall SET consumed=1, consumed_at=?, retry_count=0, next_retry_at=NULL WHERE id=?`, now, id); err != nil { + logUsage("consume row id=%d failed: %v", id, err) continue } - _, _ = db.Exec(`UPDATE usage_recall SET consumed=1, consumed_at=?, retry_count=0, next_retry_at=NULL WHERE id=?`, now, id) flags = append(flags, UsageFlag{ MemoryID: memID, Content: memContent, Criterion: "carry_forward", MatchedTokens: toks, HitCount: hits, RunesMatched: runes, ConsumedAt: now, }) } + return +} + +// CheckCommittedForUse commit 写侧钩子:检查新写入的内容是否携带了 +// 最近被召回记忆的独特 token;命中则判定「被用上」并落 outcome(门控)。 +// 返回本次判定出的「被用上」记忆(供上报/取证)。 +// 消费调度(t_9143c968):expire→retry→tail→sweep→ttl,全程 O(batch),无全表计数。 +func CheckCommittedForUse(ldb LanceDB, agent, namespace, content string) []UsageFlag { + if !usageTrackEnabled || ldb == nil || content == "" { + return nil + } + db := usageOpen() + if db == nil { + return nil + } + usageMu.Lock() + defer usageMu.Unlock() + + batch := usageEnvInt("ZHIYI_USAGE_BATCH", 200) + sweepBatch := usageEnvInt("ZHIYI_USAGE_SWEEP_BATCH", 500) // 0=关闭轮巡 + breakerMax := int(envFloat("ZHIYI_USAGE_BACKLOG_MAX", 500)) + cutoff := time.Now().Add(-time.Duration(usageWindowHours * float64(time.Hour))).Format(time.RFC3339) + scanNow := time.Now().Format(time.RFC3339) + contentSet := tokenSet(content) + var scanned int + + // ── expire:出窗未消费行标 consumed=2(skipped_expired,留证据不物理删)── + if chunk := usageEnvInt("ZHIYI_USAGE_EXPIRE_CHUNK", 5000); chunk > 0 { + if res, err := db.Exec(`UPDATE usage_recall SET consumed=2 + WHERE id IN (SELECT id FROM usage_recall WHERE consumed=0 AND recalled_at 0 { + logUsage("expire mark: %d rows -> consumed=2 (skipped_expired)", n) + } + } + + // ── retry:退避到期的失败行优先(t_20bfbd9a 退避语义不变)── + rf, rFailed, _ := usageScanBatch(db, ldb, agent, namespace, contentSet, + `SELECT id, memory_id, content, tokens, namespace, agent, retry_count + FROM usage_recall + WHERE consumed=0 AND next_retry_at IS NOT NULL AND next_retry_at<=? AND recalled_at>=? + ORDER BY id LIMIT ?`, scanNow, cutoff, batch) + scanned += len(rf) + rFailed + + // ── tail:游标向上 ≤batch 行(DESC 先吃最新;与本次 commit 因果对齐的主信号面)── + var top int64 + _ = db.QueryRow(`SELECT COALESCE(MAX(id),0) FROM usage_recall`).Scan(&top) + tail, sweep := usageCursorGet(db) + tf, tFailed, _ := usageScanBatch(db, ldb, agent, namespace, contentSet, + `SELECT id, memory_id, content, tokens, namespace, agent, retry_count + FROM usage_recall + WHERE consumed=0 AND id>? AND id<=? AND recalled_at>=? AND (next_retry_at IS NULL OR next_retry_at<=?) + ORDER BY id DESC LIMIT ?`, tail, top, cutoff, scanNow, batch) + scanned += len(tf) + tFailed + newTail := top // 游标推进到表尾;本批未触达的落后行留给 sweep/expire(代价有界) + if newTail < tail { + newTail = tail // 库被外部重置等异常:游标不回退 + } + + // ── sweep:轮巡复查未消费行;游标后无行 ⇒ 游标归 0 并【本轮立即从 0 补扫一次】 + // (「一次召回、后续多个 commit 逐个命中」依赖这条;至多两次有界查询,不整窗扫)── + var sf []UsageFlag + sFailed, sMax := 0, int64(0) + newSweep := sweep + if sweepBatch > 0 && breakerMax > 0 && rFailed+tFailed >= breakerMax { + // 单次调用失败熔断(ZHIYI_USAGE_BACKLOG_MAX 收窄后的新语义): + // 前面已积累 ≥cap 失败 ⇒ 本轮不再开新失败面(旧「全表积压跳过」已删除)。 + logUsage("sweep skipped by failure breaker (%d failures >= cap %d)", rFailed+tFailed, breakerMax) + } else if sweepBatch > 0 { + sf, sFailed, sMax = usageScanBatch(db, ldb, agent, namespace, contentSet, + `SELECT id, memory_id, content, tokens, namespace, agent, retry_count + FROM usage_recall + WHERE consumed=0 AND id>? AND recalled_at>=? AND (next_retry_at IS NULL OR next_retry_at<=?) + ORDER BY id LIMIT ?`, sweep, cutoff, scanNow, sweepBatch) + if sMax == 0 && sweep > 0 { // 到表尾:回绕 0,本轮补扫 + var f2 []UsageFlag + var x2 int + f2, x2, sMax = usageScanBatch(db, ldb, agent, namespace, contentSet, + `SELECT id, memory_id, content, tokens, namespace, agent, retry_count + FROM usage_recall + WHERE consumed=0 AND id>0 AND recalled_at>=? AND (next_retry_at IS NULL OR next_retry_at<=?) + ORDER BY id LIMIT ?`, cutoff, scanNow, sweepBatch) + sf = append(sf, f2...) + sFailed += x2 + } + scanned += len(sf) + sFailed + if sMax > 0 { + newSweep = sMax + } else { + newSweep = 0 // 表尾到达:回绕 + } + } + + usageCursorSet(db, newTail, newSweep) + + // ── TTL:consumed>=1(含 2)且 recalled_at<7d 删除,chunk 化防长事务 ── + if chunk := usageEnvInt("ZHIYI_USAGE_TTL_CHUNK", 5000); chunk > 0 { + ttlCutoff := time.Now().Add(-7 * 24 * time.Hour).Format(time.RFC3339) + if res, err := db.Exec(`DELETE FROM usage_recall + WHERE id IN (SELECT id FROM usage_recall WHERE consumed>=1 AND recalled_at 0 { + logUsage("ttl purged %d rows (consumed>=1, recalled_at<7d)", n) + // 定期截断 WAL,防删除型事务把 -wal 撑大 + _, _ = db.Exec(`PRAGMA wal_checkpoint(TRUNCATE)`) + } + } + + flags := make([]UsageFlag, 0, len(rf)+len(tf)+len(sf)) + flags = append(flags, rf...) + flags = append(flags, tf...) + flags = append(flags, sf...) + if scanned > 0 || len(flags) > 0 { + logUsage("usage scan: scanned=%d flagged=%d failed=%d tail=%d->%d sweep=%d->%d", + scanned, len(flags), rFailed+tFailed+sFailed, tail, newTail, sweep, newSweep) + } return flags } diff --git a/go/internal/storage/usage_track_test.go b/go/internal/storage/usage_track_test.go index 2dfc2ac..45d607a 100644 --- a/go/internal/storage/usage_track_test.go +++ b/go/internal/storage/usage_track_test.go @@ -150,18 +150,22 @@ func TestUsageBackoff_ExponentialGrowthAndCap(t *testing.T) { } } -// t_20bfbd9a 治本②:积压上限量化 —— 未消费待重试行超过 ZHIYI_USAGE_BACKLOG_MAX 时本轮跳过扫描。 +// t_20bfbd9a 治本②(旧):积压超 ZHIYI_USAGE_BACKLOG_MAX ⇒ 本轮 skip。 +// t_9143c968(2026-10-30)作废该语义:写入量涨到 40~200 万行/日后「无窗 pending>cap」 +// 恒成立 ⇒ 每 commit skip ⇒ 消费侧永久熔断(10-19 起 consumed=0)。cap 收窄为 +// 「单次调用失败熔断」(见 TestUsageCursor_FailureBreakerReplacesBacklogSkip)。 +// 本用例改为验证:积压再大也不再 skip —— 窗口内有可消费行就必须消费, +// 窗口外 pending 不再阻塞(被 expire 标 2)。这正是卡体 AC1 的必要条件。 func TestCheckCommittedForUse_BacklogCapSkipsScan(t *testing.T) { setupUsageTest(t) db := usageOpen() if db == nil { t.Fatal("usageOpen failed") } - // 显式压小积压上限,验证超过即跳过 + // 压小 cap 到 2,且制造远超 cap 的 pending —— 旧实现此处必 skip。 t.Setenv("ZHIYI_USAGE_BACKLOG_MAX", "2") now := time.Now().Format(time.RFC3339) - // 塞 3 条未消费召回行(超过上限 2) - for i := 0; i < 3; i++ { + for i := 0; i < 5; i++ { if _, err := db.Exec(`INSERT INTO usage_recall(memory_id, content, tokens, q_hash, namespace, agent, recalled_at) VALUES(?,?,?,?,?,?,?)`, "mem_bk"+string(rune('0'+i)), "牧尘今天要求处理看板任务验收"+string(rune('0'+i)), @@ -169,15 +173,15 @@ func TestCheckCommittedForUse_BacklogCapSkipsScan(t *testing.T) { t.Fatalf("insert: %v", err) } } - // 无需真实 LanceDB(ldb=nil 直接返回 nil flags);本用例只验证「超限即提前跳过」不 panic 且不消费 - flags := CheckCommittedForUse(nil, "agent", "ns", "牧尘今天要求处理看板任务验收0") - if flags != nil { - t.Fatalf("ldb=nil 应直接返回 nil flags(超限跳过不应触及消费路径), got %v", flags) + // 窗口内有可消费行 ⇒ 即使 pending(5)≫cap(2) 也必须消费它(不再 skip)。 + mlc := NewMemLanceClient(nil) + flags := CheckCommittedForUse(mlc, "agent", "ns", "牧尘今天要求处理看板任务验收") + if len(flags) == 0 { + t.Fatalf("积压 > cap 不再触发 skip:窗口内可消费行必须被判定, got 0 flags") } - // 判据:超限跳过时所有行仍 consumed=0(没有消费行为发生) - var pending int - _ = db.QueryRow(`SELECT COUNT(*) FROM usage_recall WHERE consumed=0`).Scan(&pending) - if pending != 3 { - t.Fatalf("超限跳过扫描后应 3 行全部未消费(no consumption), got %d", pending) + // 被消费的行 consumed=1;旧实现这里会 5 行全 consumed=0。 + if c := usagePendingBy(t, db, "consumed=1"); c == 0 { + t.Fatalf("修复后应至少消费 1 行(旧 skip 语义已废), actual %d", c) } } +