这批是 10-26~27 已在生产二进制(zhiyid-protectrails-20261027)里运行、 但一直留在工作树未提交的改动。收口原因:三件改造要开分支,工作树脏 = 分支基准与回滚点都不存在。 主要内容: - api/routes/contract.go + trace.go:召回响应契约字段(trace_id/status/scope/ continuation/channel)与 /api/v1/trace 回溯端点 - governance/entity_quality.go:实体质量闸 - storage/trace.go + recall.go/recall_write_buffer.go:召回写入缓冲与轨迹 - 测试:contract/recall_contract_fields/recall_lifecycle/stale_lifecycle/ entity_quality/trace 同时修一个 flaky:recall_lifecycle_test 原本硬编码「今天 11:00 已过」, 凌晨跑就会把「已失效」fixture 当成仍有效。改成相对时间(过期=now-1h), 生命周期判据与运行时刻无关。修完 go test ./internal/api/routes ./internal/storage 全绿。
332 lines
9.2 KiB
Go
332 lines
9.2 KiB
Go
// 织忆 MemoryWeave — 召回链路追踪(trace_id 贯穿,t_trace_p0)
|
||
//
|
||
// 背景(CS249r Vol III Ch14 判据):全仓 grep trace_id|span = 0 ⇒「这条为什么被召回」
|
||
// 永远答不出,后续所有调参都是盲修。本文件提供最小可用链路追踪:
|
||
//
|
||
// ① 每次 recall 生成 trace_id(tr_<16字节hex>),贯穿 编码→粗检索→FTS→重排→MMR→图谱→返回;
|
||
// ② 各阶段耗时 + 各通道贡献条数 + 每条结果的来源通道(RecallResult.Channel);
|
||
// ③ GET /api/v1/trace/<trace_id> 可重建整条链路(内存环形缓冲 + JSONL 落盘,重启后仍可查)。
|
||
//
|
||
// 设计约束:
|
||
// - 不引入新依赖(crypto/rand 生成 ID,不新增 uuid 直接依赖);
|
||
// - 落盘异步(有界 channel + 单写协程),recall 主路径只做内存 append,绝不为追踪拖慢召回;
|
||
// - 落盘失败只降级为「仅内存」,不报错、不影响召回结果(追踪是观测设施,不是业务依赖);
|
||
// - **这不是缓存**(不参与召回读写路径、不返回旧结果),与 searchcache/Redis 层的职责不同;
|
||
// 写一次读多次的观测日志,用环形缓冲 + JSONL 是合适的形态,无需引入 Redis 键。
|
||
package storage
|
||
|
||
import (
|
||
"bufio"
|
||
"crypto/rand"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"fmt"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
)
|
||
|
||
// TraceStage 单个阶段耗时。Count 语义随阶段不同(候选数/重排数/扩展数…)。
|
||
type TraceStage struct {
|
||
Name string `json:"name"`
|
||
MS float64 `json:"ms"`
|
||
Count int `json:"count,omitempty"`
|
||
Note string `json:"note,omitempty"`
|
||
}
|
||
|
||
// TraceItem 单条返回结果的来源归因。
|
||
type TraceItem struct {
|
||
Rank int `json:"rank"`
|
||
ID string `json:"id"`
|
||
Channel string `json:"channel"`
|
||
Score float64 `json:"score"`
|
||
}
|
||
|
||
// RecallTrace 一次召回的完整链路记录。
|
||
type RecallTrace struct {
|
||
TraceID string `json:"trace_id"`
|
||
Query string `json:"query"`
|
||
Namespace string `json:"namespace"`
|
||
Mode string `json:"mode"`
|
||
Status string `json:"status"` // complete | partial
|
||
PartialReason string `json:"partial_reason,omitempty"`
|
||
StartedAt time.Time `json:"started_at"`
|
||
DurationMS float64 `json:"duration_ms"`
|
||
CacheHit bool `json:"cache_hit"`
|
||
CandidatesScanned int `json:"candidates_scanned"`
|
||
Truncated bool `json:"truncated"`
|
||
StandingAnswerHit string `json:"standing_answer_hit,omitempty"`
|
||
TopK int `json:"top_k"`
|
||
Channels map[string]int `json:"channels"`
|
||
Stages []TraceStage `json:"stages"`
|
||
Items []TraceItem `json:"items"`
|
||
Scope map[string]interface{} `json:"scope,omitempty"`
|
||
Error string `json:"error,omitempty"`
|
||
|
||
started time.Time
|
||
}
|
||
|
||
const (
|
||
traceRingCap = 2000
|
||
traceMaxFileSize = 32 << 20 // 32MB 轮转
|
||
)
|
||
|
||
// traceDir 可通过 ZHIYI_TRACE_DIR 覆盖(测试用)。
|
||
func traceDir() string {
|
||
if d := os.Getenv("ZHIYI_TRACE_DIR"); d != "" {
|
||
return d
|
||
}
|
||
return "/var/lib/memoryweave/traces"
|
||
}
|
||
|
||
func tracePersistEnabled() bool {
|
||
switch strings.ToLower(os.Getenv("ZHIYI_TRACE_PERSIST")) {
|
||
case "0", "false", "off":
|
||
return false
|
||
}
|
||
return true
|
||
}
|
||
|
||
// NewTraceID 生成 tr_<32hex>(16 字节随机,碰撞概率可忽略)。
|
||
func NewTraceID() string {
|
||
var b [16]byte
|
||
if _, err := rand.Read(b[:]); err != nil {
|
||
// 极端情况下退化为纳秒时间戳,仍保证非空、可区分。
|
||
return fmt.Sprintf("tr_%d", time.Now().UnixNano())
|
||
}
|
||
return "tr_" + hex.EncodeToString(b[:])
|
||
}
|
||
|
||
// NewRecallTrace 创建一条追踪记录(StartedAt 同时作为计时起点)。
|
||
func NewRecallTrace(traceID, query, namespace, mode string) *RecallTrace {
|
||
now := time.Now()
|
||
if traceID == "" {
|
||
traceID = NewTraceID()
|
||
}
|
||
return &RecallTrace{
|
||
TraceID: traceID,
|
||
Query: query,
|
||
Namespace: namespace,
|
||
Mode: mode,
|
||
Status: "complete",
|
||
StartedAt: now,
|
||
Channels: map[string]int{},
|
||
Stages: []TraceStage{},
|
||
Items: []TraceItem{},
|
||
started: now,
|
||
}
|
||
}
|
||
|
||
// Stage 记录一个阶段耗时。start 由调用方在阶段开始前取;count<0 表示不记录。
|
||
func (t *RecallTrace) Stage(name string, start time.Time, count int) {
|
||
if t == nil {
|
||
return
|
||
}
|
||
t.Stages = append(t.Stages, TraceStage{
|
||
Name: name,
|
||
MS: float64(time.Since(start).Microseconds()) / 1000.0,
|
||
Count: count,
|
||
})
|
||
}
|
||
|
||
// StageNote 记录带备注的阶段(如失败原因)。
|
||
func (t *RecallTrace) StageNote(name string, start time.Time, count int, note string) {
|
||
if t == nil {
|
||
return
|
||
}
|
||
t.Stages = append(t.Stages, TraceStage{
|
||
Name: name,
|
||
MS: float64(time.Since(start).Microseconds()) / 1000.0,
|
||
Count: count,
|
||
Note: note,
|
||
})
|
||
}
|
||
|
||
// MarkPartial 标记本次召回为部分结果(deadline / 行数上限 / 阶段错误)。
|
||
func (t *RecallTrace) MarkPartial(reason string) {
|
||
if t == nil || reason == "" {
|
||
return
|
||
}
|
||
t.Status = "partial"
|
||
if t.PartialReason == "" {
|
||
t.PartialReason = reason
|
||
} else if !strings.Contains(t.PartialReason, reason) {
|
||
t.PartialReason += "," + reason
|
||
}
|
||
}
|
||
|
||
// SetItems 用最终返回集填充 item 归因(Rank 从 1 起)。
|
||
func (t *RecallTrace) SetItems(items []TraceItem) {
|
||
if t == nil {
|
||
return
|
||
}
|
||
t.Items = items
|
||
t.Channels = map[string]int{}
|
||
for _, it := range items {
|
||
ch := it.Channel
|
||
if ch == "" {
|
||
ch = "unknown"
|
||
}
|
||
t.Channels[ch]++
|
||
}
|
||
}
|
||
|
||
var (
|
||
traceMu sync.RWMutex
|
||
traceRing = make([]*RecallTrace, 0, traceRingCap)
|
||
traceIdx = map[string]*RecallTrace{}
|
||
|
||
traceFileMu sync.Mutex
|
||
traceFile *os.File
|
||
traceFileSize int64
|
||
traceFileDay string
|
||
|
||
traceCh = make(chan *RecallTrace, 1024)
|
||
traceOnce sync.Once
|
||
)
|
||
|
||
// SaveTrace 写入内存环形缓冲,并异步落盘(非阻塞;缓冲满则丢落盘、保留内存)。
|
||
func SaveTrace(t *RecallTrace) {
|
||
if t == nil {
|
||
return
|
||
}
|
||
if t.DurationMS == 0 {
|
||
t.DurationMS = float64(time.Since(t.started).Microseconds()) / 1000.0
|
||
}
|
||
traceMu.Lock()
|
||
if len(traceRing) >= traceRingCap {
|
||
// 覆盖最旧:同时从索引删除
|
||
old := traceRing[0]
|
||
copy(traceRing, traceRing[1:])
|
||
traceRing = traceRing[:len(traceRing)-1]
|
||
if old != nil {
|
||
delete(traceIdx, old.TraceID)
|
||
}
|
||
}
|
||
traceRing = append(traceRing, t)
|
||
traceIdx[t.TraceID] = t
|
||
traceMu.Unlock()
|
||
|
||
if !tracePersistEnabled() {
|
||
return
|
||
}
|
||
traceOnce.Do(func() { go traceWriter() })
|
||
select {
|
||
case traceCh <- t:
|
||
default:
|
||
// 落盘队列满:丢这一条(内存仍可查),不阻塞召回主路径。
|
||
}
|
||
}
|
||
|
||
// GetTrace 先查内存,再回落到落盘文件(支持服务重启后按 trace_id 回溯)。
|
||
func GetTrace(traceID string) (*RecallTrace, bool) {
|
||
if traceID == "" {
|
||
return nil, false
|
||
}
|
||
traceMu.RLock()
|
||
if t, ok := traceIdx[traceID]; ok {
|
||
traceMu.RUnlock()
|
||
return t, true
|
||
}
|
||
traceMu.RUnlock()
|
||
return getTraceFromDisk(traceID)
|
||
}
|
||
|
||
// TraceCount 当前内存中可查的 trace 数(自检/测试用)。
|
||
func TraceCount() int {
|
||
traceMu.RLock()
|
||
defer traceMu.RUnlock()
|
||
return len(traceRing)
|
||
}
|
||
|
||
func traceWriter() {
|
||
for t := range traceCh {
|
||
writeTraceLine(t)
|
||
}
|
||
}
|
||
|
||
// writeTraceLine 追加一行 JSONL。失败静默(仅内存可查),不向调用方冒泡。
|
||
func writeTraceLine(t *RecallTrace) {
|
||
line, err := json.Marshal(t)
|
||
if err != nil {
|
||
return
|
||
}
|
||
line = append(line, '\n')
|
||
|
||
dir := traceDir()
|
||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||
return
|
||
}
|
||
day := time.Now().Format("20060102")
|
||
|
||
traceFileMu.Lock()
|
||
defer traceFileMu.Unlock()
|
||
|
||
if traceFile == nil || traceFileDay != day || traceFileSize > traceMaxFileSize {
|
||
if traceFile != nil {
|
||
_ = traceFile.Close()
|
||
traceFile = nil
|
||
}
|
||
path := filepath.Join(dir, "recall-"+day+".jsonl")
|
||
if traceFileSize > traceMaxFileSize {
|
||
// 轮转:同日超大时改名 .1(覆盖上一份)
|
||
_ = os.Rename(path, path+".1")
|
||
}
|
||
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
|
||
if err != nil {
|
||
return
|
||
}
|
||
traceFile = f
|
||
traceFileDay = day
|
||
if st, err := f.Stat(); err == nil {
|
||
traceFileSize = st.Size()
|
||
} else {
|
||
traceFileSize = 0
|
||
}
|
||
}
|
||
if _, err := traceFile.Write(line); err == nil {
|
||
traceFileSize += int64(len(line))
|
||
}
|
||
}
|
||
|
||
// getTraceFromDisk 从落地文件逆序扫描(只读,不影响运行中的服务)。
|
||
func getTraceFromDisk(traceID string) (*RecallTrace, bool) {
|
||
dir := traceDir()
|
||
entries, err := os.ReadDir(dir)
|
||
if err != nil {
|
||
return nil, false
|
||
}
|
||
needle := `"trace_id":"` + traceID + `"`
|
||
// 文件名含日期,倒序 = 新文件先查。
|
||
for i := len(entries) - 1; i >= 0; i-- {
|
||
e := entries[i]
|
||
if e.IsDir() || !strings.HasPrefix(e.Name(), "recall-") {
|
||
continue
|
||
}
|
||
f, err := os.Open(filepath.Join(dir, e.Name()))
|
||
if err != nil {
|
||
continue
|
||
}
|
||
var hit *RecallTrace
|
||
sc := bufio.NewScanner(f)
|
||
sc.Buffer(make([]byte, 0, 64*1024), 4<<20)
|
||
for sc.Scan() {
|
||
b := sc.Bytes()
|
||
if !strings.Contains(string(b), needle) {
|
||
continue
|
||
}
|
||
var t RecallTrace
|
||
if json.Unmarshal(b, &t) == nil && t.TraceID == traceID {
|
||
hit = &t
|
||
}
|
||
}
|
||
_ = f.Close()
|
||
if hit != nil {
|
||
return hit, true
|
||
}
|
||
}
|
||
return nil, false
|
||
}
|