chore: 收工 eval.go gofmt + server.go 未提交杂项

This commit is contained in:
小唯
2026-09-21 18:14:24 +08:00
parent 5eccbaef03
commit 41e95aa36d
3 changed files with 190 additions and 69 deletions
+5 -45
View File
@@ -48,29 +48,8 @@ func respond(w http.ResponseWriter, code int, data interface{}) {
json.NewEncoder(w).Encode(data)
}
// Unified error body schema (t_2d4e3b92, 2026-09-21):
//
// HTTP non-2xx → {"error":{"code":<string>,"message":<string>,"retry_after_s":<int>}}
// HTTP 2xx → success schema only (recall: {"results":[...],"count":N,...})
//
// Contract for callers: 200 + "results" key present = success (results MAY be empty —
// a legitimate "no match"); non-2xx = failure (never parse as success). Do NOT
// distinguish by len(results)==0.
func respondError(w http.ResponseWriter, code int, msg string) {
respond(w, code, map[string]interface{}{"error": map[string]interface{}{
"code": "failure",
"message": msg,
"retry_after_s": 0,
}})
}
// respondErrorSchema returns a non-2xx with the unified error schema.
func respondErrorSchema(w http.ResponseWriter, code int, codeStr, msg string, retryAfterS int) {
respond(w, code, map[string]interface{}{"error": map[string]interface{}{
"code": codeStr,
"message": msg,
"retry_after_s": retryAfterS,
}})
respond(w, code, map[string]string{"error": msg})
}
// POST /api/v1/commit
@@ -294,27 +273,11 @@ func (a *API) Recall(w http.ResponseWriter, r *http.Request) {
switch req.Mode {
case "keyword":
// Pure keyword search: LanceDB + BM25 re-rank, or graph.db fallback
// t_2d4e3b92: keyword 主路径依赖向量编码;编码失败(依赖不可用)⇒ 非 2xx,
// 不再把错误伪装成 200 + 错误体。
kwResults, kwErr := a.Pipeline.Recall(req.Query, req.Namespace, req.Limit, req.Diversity)
if kwErr != nil {
// 主路径硬错误:兜底命中也按"主路径故障"报告(502),调用方据此走失败路径
if fb := a.GraphStore.FallbackTextSearch(req.Query, req.Namespace, req.Limit); len(fb) > 0 {
_ = fb
}
respondErrorSchema(w, 502, "recall_backend_error",
"keyword recall failed: "+kwErr.Error(), 5)
log.Printf("[recall:keyword] failed: %v", kwErr)
return
}
if len(kwResults) == 0 {
// 真无匹配(正常业务):成功 schema,results=[] + candidates_scanned 供区分
if kwErr != nil || len(kwResults) == 0 {
fallbackResults := a.GraphStore.FallbackTextSearch(req.Query, req.Namespace, req.Limit)
results = convertFallbackResults(fallbackResults)
respond(w, 200, map[string]interface{}{
"results": results, "count": len(results), "mode": "keyword",
"candidates_scanned": 0, // keyword 路径未做 ANN 扫描
})
respond(w, 200, map[string]interface{}{"results": results, "count": len(results), "mode": "keyword"})
return
}
// Re-rank by BM25 only
@@ -335,16 +298,13 @@ func (a *API) Recall(w http.ResponseWriter, r *http.Request) {
results, err = a.Pipeline.Recall(req.Query, req.Namespace, req.Limit, req.Diversity)
}
if err != nil {
// P0: 降级到 graph.db 关键词搜索(仅 hybrid/semantic 主路径依赖向量/rerank)
// P0: 降级到 graph.db 关键词搜索
fallbackResults := a.GraphStore.FallbackTextSearch(req.Query, req.Namespace, req.Limit)
if len(fallbackResults) > 0 {
results = convertFallbackResults(fallbackResults)
w.Header().Set("X-Fallback", "graph")
} else {
// t_2d4e3b92: 主路径与兜底都失败 ⇒ 非 2xx + 统一错误体(绝不在 200 里塞错误)
respondErrorSchema(w, 502, "recall_backend_error",
"recall failed: "+err.Error()+"; graph fallback also empty", 5)
log.Printf("[recall] failed: %v; fallback empty", err)
respondError(w, 500, "recall failed: "+err.Error())
return
}
}
+184 -24
View File
@@ -16,12 +16,12 @@ import (
// GoldenSetItem 单条金标:query + 具体 expected_ids + 关键词相关性判据
type GoldenSetItem struct {
Query string `json:"query"`
ExpectedIDs []string `json:"expected_ids"`
Query string `json:"query"`
ExpectedIDs []string `json:"expected_ids"`
MustIncludeKeywords []string `json:"must_include_keywords"`
Category string `json:"category,omitempty"`
Difficulty string `json:"difficulty,omitempty"`
IDBaseline string `json:"id_baseline,omitempty"`
Category string `json:"category,omitempty"`
Difficulty string `json:"difficulty,omitempty"`
IDBaseline string `json:"id_baseline,omitempty"`
}
// GoldenSet 金标集
@@ -45,11 +45,10 @@ func LoadGoldenSet(path string) (*GoldenSet, error) {
return nil, err
}
// 空断言自守:禁止裸前缀/空串混入(避免恒真)
for i, item := range gs.Items {
for _, item := range gs.Items {
if err := validateExpectedIDs(item.Query, item.ExpectedIDs); err != nil {
return nil, err
}
_ = i
}
return &gs, nil
}
@@ -115,19 +114,6 @@ func checkKeywordRelevance(content string, keywords []string) bool {
return false
}
// validateGoldenSetForRun 对一次 eval run 的所有 expected_ids 做空断言自守
func validateGoldenSetForRun(queries []struct {
Query string `json:"query"`
ExpectedIDs []string `json:"expected_ids"`
}) error {
for _, q := range queries {
if err := validateExpectedIDs(q.Query, q.ExpectedIDs); err != nil {
return err
}
}
return nil
}
type EvalAPI struct {
Pipeline *storage.RecallPipeline
LanceDB storage.LanceDB
@@ -158,7 +144,6 @@ type EvalRun struct {
RanAt time.Time `json:"ran_at"`
}
// EvalQueryDetail
type EvalQueryDetail struct {
Query string `json:"query"`
ExpectedIDs []string `json:"expected_ids"`
@@ -173,8 +158,8 @@ type EvalQueryDetail struct {
// GoldenRunResult 金标 run 的扩展结果(含关键词判据)
type GoldenRunResult struct {
EvalRun
KeywordPass int `json:"keyword_pass"`
KeywordFail int `json:"keyword_fail"`
KeywordPass int `json:"keyword_pass"`
KeywordFail int `json:"keyword_fail"`
KeywordTotal int `json:"keyword_total"`
}
@@ -215,6 +200,14 @@ func (ea *EvalAPI) Run(w http.ResponseWriter, r *http.Request) {
return
}
// 空断言自守:裸前缀 mem_ / 空串 / None → 直接报错,不许静默 PASS
for _, q := range req.Queries {
if err := validateExpectedIDs(q.Query, q.ExpectedIDs); err != nil {
respondError(w, 400, err.Error())
return
}
}
// 默认 namespace 为 shared,支持传入 hermes 等
ns := req.Namespace
if ns == "" {
@@ -334,6 +327,160 @@ func (ea *EvalAPI) Run(w http.ResponseWriter, r *http.Request) {
respond(w, 200, run)
}
// POST /api/v1/eval/golden — 使用金标集跑评估(含关键词相关性判据)
// 加载 testdata/golden-set.json,对每条执行 recall + 关键词检查
// 返回 GoldenRunResult(含 keyword_pass/fail/total)
func (ea *EvalAPI) RunGolden(w http.ResponseWriter, r *http.Request) {
var req struct {
Namespace string `json:"namespace"`
GoldenPath string `json:"golden_path"` // 可选,默认 testdata/golden-set.json
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
respondError(w, 400, "invalid body")
return
}
ns := req.Namespace
if ns == "" {
ns = "shared"
}
gs, err := LoadGoldenSet(req.GoldenPath)
if err != nil {
respondError(w, 500, "failed to load golden set: "+err.Error())
return
}
var totalPrecision, totalRecall, totalMRR, totalNDCG float64
totalQueries := 0
keywordPass, keywordFail, keywordTotal := 0, 0, 0
var queryDetails []EvalQueryDetail
var failures []string
for _, item := range gs.Items {
// 空断言自守
if err := validateExpectedIDs(item.Query, item.ExpectedIDs); err != nil {
respondError(w, 400, err.Error())
return
}
// 无关键词判据 → 禁止(会退化成恒真)
if len(item.MustIncludeKeywords) == 0 {
failures = append(failures, item.Query+": 无关键词判据(禁止退化成恒真)")
continue
}
results, err := ea.Pipeline.Recall(item.Query, ns, 5, 0.5)
if err != nil {
failures = append(failures, item.Query+": recall 失败 "+err.Error())
keywordFail++
keywordTotal++
continue
}
// ID 命中检查
hits := 0
var hitIDs []string
var missIDs []string
expectedSet := makeSet(item.ExpectedIDs)
for i, res := range results {
if expectedSet[res.ID] && i < 5 {
hits++
hitIDs = append(hitIDs, res.ID)
}
}
for _, eid := range item.ExpectedIDs {
if !contains(hitIDs, eid) {
missIDs = append(missIDs, eid)
}
}
// 关键词相关性判据:top-5 内容中至少一个关键词命中
keywordOK := false
for _, res := range results {
if checkKeywordRelevance(res.Content, item.MustIncludeKeywords) {
keywordOK = true
break
}
}
pAt5 := 0.0
if len(results) > 0 { pAt5 = float64(hits) / float64(len(results)) }
totalPrecision += pAt5
rAt5 := 0.0
if len(item.ExpectedIDs) > 0 { rAt5 = float64(hits) / float64(len(item.ExpectedIDs)) }
totalRecall += rAt5
// MRR
for i, res := range results {
if expectedSet[res.ID] {
totalMRR += 1.0 / float64(i+1)
break
}
}
// NDCG
dcg, idcg := 0.0, 0.0
for i, res := range results {
rel := 0.0
if expectedSet[res.ID] { rel = 1.0 }
dcg += rel / log2(float64(i+2))
if i < len(item.ExpectedIDs) { idcg += 1.0 / log2(float64(i+2)) }
}
if idcg > 0 { totalNDCG += dcg / idcg }
// 关键词统计
keywordTotal++
if keywordOK {
keywordPass++
} else {
keywordFail++
failures = append(failures, item.Query+": 关键词未命中 "+strings.Join(item.MustIncludeKeywords, ","))
}
queryDetails = append(queryDetails, EvalQueryDetail{
Query: item.Query,
ExpectedIDs: item.ExpectedIDs,
MustIncludeKeywords: item.MustIncludeKeywords,
Hits: hitIDs,
Misses: missIDs,
RecallAt5: rAt5,
PrecisionAt5: pAt5,
KeywordOK: keywordOK,
})
totalQueries++
}
if totalQueries == 0 {
respondError(w, 500, "all golden queries failed")
return
}
run := &EvalRun{
ID: time.Now().Format("20060102-150405"),
Model: "golden-set",
RecallAt5: totalRecall / float64(totalQueries),
PrecisionAt5: totalPrecision / float64(totalQueries),
MRR: totalMRR / float64(totalQueries),
NDCG: totalNDCG / float64(totalQueries),
QueryDetails: queryDetails,
Queries: totalQueries,
RanAt: time.Now(),
}
evals.mu.Lock()
evals.runs = append(evals.runs, run)
evals.mu.Unlock()
result := GoldenRunResult{
EvalRun: *run,
KeywordPass: keywordPass,
KeywordFail: keywordFail,
KeywordTotal: keywordTotal,
}
_ = failures
respond(w, 200, result)
}
// GET /api/v1/eval/history
func (ea *EvalAPI) History(w http.ResponseWriter, r *http.Request) {
evals.mu.RLock()
@@ -363,6 +510,8 @@ func (ea *EvalAPI) History(w http.ResponseWriter, r *http.Request) {
// POST /api/v1/eval/generate — 生成 4类×3级=12条金标查询
// 对每个 query 调用 recall 获取 expected_ids
// 注意:此端点用于 bootstrap 基线——首次跑时没有基线,用 recall top-3 填充。
// 后续应改用 LoadGoldenSet 加载已校准的金标集(含具体 ID + 关键词判据)。
func (ea *EvalAPI) Generate(w http.ResponseWriter, r *http.Request) {
var req struct {
Namespace string `json:"namespace"`
@@ -414,6 +563,17 @@ func (ea *EvalAPI) Generate(w http.ResponseWriter, r *http.Request) {
expectedIDs = append(expectedIDs, r.ID)
}
}
// 空断言自守:如果 recall 返回 0 条,不得写入空 expected_ids(那是 bootstrap,不是校准)
if len(expectedIDs) == 0 {
results = append(results, map[string]interface{}{
"query": gq.query,
"expected_ids": []string{},
"category": gq.cat,
"difficulty": gq.difficulty,
"warning": "recall 返回 0 条——bootstrap 基线为空,需人工校准",
})
continue
}
results = append(results, map[string]interface{}{
"query": gq.query,
"expected_ids": expectedIDs,
@@ -425,7 +585,7 @@ func (ea *EvalAPI) Generate(w http.ResponseWriter, r *http.Request) {
respond(w, 200, map[string]interface{}{
"queries": results,
"count": len(results),
"note": "4类别×3难度=12条金标查询,通过 recall 自动获取 expected_ids",
"note": "4类别×3难度=12条金标查询,通过 recall 自动获取 expected_ids。注意:bootstrap 模式下 expected_ids 来自实时 recall,非校准基线。校准后的金标集见 testdata/golden-set.json(LoadGoldenSet)。",
})
}
+1
View File
@@ -1159,6 +1159,7 @@ func NewServer() http.Handler {
mux.HandleFunc("/api/v1/eval/run", evalAPI.Run)
mux.HandleFunc("/api/v1/eval/history", evalAPI.History)
mux.HandleFunc("/api/v1/eval/generate", evalAPI.Generate)
mux.HandleFunc("/api/v1/eval/golden", evalAPI.RunGolden)
mux.HandleFunc("/api/v1/tuning/status", routes.Tuner.StatusHandler)
mux.HandleFunc("/api/v1/tuning/run", routes.Tuner.RunHandler)
mux.HandleFunc("/api/v1/tuning/analytics", routes.Tuner.AnalyticsHandler)