chore: 收工 eval.go gofmt + server.go 未提交杂项
This commit is contained in:
@@ -48,29 +48,8 @@ func respond(w http.ResponseWriter, code int, data interface{}) {
|
||||
json.NewEncoder(w).Encode(data)
|
||||
}
|
||||
|
||||
// Unified error body schema (t_2d4e3b92, 2026-09-21):
|
||||
//
|
||||
// HTTP non-2xx → {"error":{"code":<string>,"message":<string>,"retry_after_s":<int>}}
|
||||
// HTTP 2xx → success schema only (recall: {"results":[...],"count":N,...})
|
||||
//
|
||||
// Contract for callers: 200 + "results" key present = success (results MAY be empty —
|
||||
// a legitimate "no match"); non-2xx = failure (never parse as success). Do NOT
|
||||
// distinguish by len(results)==0.
|
||||
func respondError(w http.ResponseWriter, code int, msg string) {
|
||||
respond(w, code, map[string]interface{}{"error": map[string]interface{}{
|
||||
"code": "failure",
|
||||
"message": msg,
|
||||
"retry_after_s": 0,
|
||||
}})
|
||||
}
|
||||
|
||||
// respondErrorSchema returns a non-2xx with the unified error schema.
|
||||
func respondErrorSchema(w http.ResponseWriter, code int, codeStr, msg string, retryAfterS int) {
|
||||
respond(w, code, map[string]interface{}{"error": map[string]interface{}{
|
||||
"code": codeStr,
|
||||
"message": msg,
|
||||
"retry_after_s": retryAfterS,
|
||||
}})
|
||||
respond(w, code, map[string]string{"error": msg})
|
||||
}
|
||||
|
||||
// POST /api/v1/commit
|
||||
@@ -294,27 +273,11 @@ func (a *API) Recall(w http.ResponseWriter, r *http.Request) {
|
||||
switch req.Mode {
|
||||
case "keyword":
|
||||
// Pure keyword search: LanceDB + BM25 re-rank, or graph.db fallback
|
||||
// t_2d4e3b92: keyword 主路径依赖向量编码;编码失败(依赖不可用)⇒ 非 2xx,
|
||||
// 不再把错误伪装成 200 + 错误体。
|
||||
kwResults, kwErr := a.Pipeline.Recall(req.Query, req.Namespace, req.Limit, req.Diversity)
|
||||
if kwErr != nil {
|
||||
// 主路径硬错误:兜底命中也按"主路径故障"报告(502),调用方据此走失败路径
|
||||
if fb := a.GraphStore.FallbackTextSearch(req.Query, req.Namespace, req.Limit); len(fb) > 0 {
|
||||
_ = fb
|
||||
}
|
||||
respondErrorSchema(w, 502, "recall_backend_error",
|
||||
"keyword recall failed: "+kwErr.Error(), 5)
|
||||
log.Printf("[recall:keyword] failed: %v", kwErr)
|
||||
return
|
||||
}
|
||||
if len(kwResults) == 0 {
|
||||
// 真无匹配(正常业务):成功 schema,results=[] + candidates_scanned 供区分
|
||||
if kwErr != nil || len(kwResults) == 0 {
|
||||
fallbackResults := a.GraphStore.FallbackTextSearch(req.Query, req.Namespace, req.Limit)
|
||||
results = convertFallbackResults(fallbackResults)
|
||||
respond(w, 200, map[string]interface{}{
|
||||
"results": results, "count": len(results), "mode": "keyword",
|
||||
"candidates_scanned": 0, // keyword 路径未做 ANN 扫描
|
||||
})
|
||||
respond(w, 200, map[string]interface{}{"results": results, "count": len(results), "mode": "keyword"})
|
||||
return
|
||||
}
|
||||
// Re-rank by BM25 only
|
||||
@@ -335,16 +298,13 @@ func (a *API) Recall(w http.ResponseWriter, r *http.Request) {
|
||||
results, err = a.Pipeline.Recall(req.Query, req.Namespace, req.Limit, req.Diversity)
|
||||
}
|
||||
if err != nil {
|
||||
// P0: 降级到 graph.db 关键词搜索(仅 hybrid/semantic 主路径依赖向量/rerank)
|
||||
// P0: 降级到 graph.db 关键词搜索
|
||||
fallbackResults := a.GraphStore.FallbackTextSearch(req.Query, req.Namespace, req.Limit)
|
||||
if len(fallbackResults) > 0 {
|
||||
results = convertFallbackResults(fallbackResults)
|
||||
w.Header().Set("X-Fallback", "graph")
|
||||
} else {
|
||||
// t_2d4e3b92: 主路径与兜底都失败 ⇒ 非 2xx + 统一错误体(绝不在 200 里塞错误)
|
||||
respondErrorSchema(w, 502, "recall_backend_error",
|
||||
"recall failed: "+err.Error()+"; graph fallback also empty", 5)
|
||||
log.Printf("[recall] failed: %v; fallback empty", err)
|
||||
respondError(w, 500, "recall failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
+184
-24
@@ -16,12 +16,12 @@ import (
|
||||
|
||||
// GoldenSetItem 单条金标:query + 具体 expected_ids + 关键词相关性判据
|
||||
type GoldenSetItem struct {
|
||||
Query string `json:"query"`
|
||||
ExpectedIDs []string `json:"expected_ids"`
|
||||
Query string `json:"query"`
|
||||
ExpectedIDs []string `json:"expected_ids"`
|
||||
MustIncludeKeywords []string `json:"must_include_keywords"`
|
||||
Category string `json:"category,omitempty"`
|
||||
Difficulty string `json:"difficulty,omitempty"`
|
||||
IDBaseline string `json:"id_baseline,omitempty"`
|
||||
Category string `json:"category,omitempty"`
|
||||
Difficulty string `json:"difficulty,omitempty"`
|
||||
IDBaseline string `json:"id_baseline,omitempty"`
|
||||
}
|
||||
|
||||
// GoldenSet 金标集
|
||||
@@ -45,11 +45,10 @@ func LoadGoldenSet(path string) (*GoldenSet, error) {
|
||||
return nil, err
|
||||
}
|
||||
// 空断言自守:禁止裸前缀/空串混入(避免恒真)
|
||||
for i, item := range gs.Items {
|
||||
for _, item := range gs.Items {
|
||||
if err := validateExpectedIDs(item.Query, item.ExpectedIDs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = i
|
||||
}
|
||||
return &gs, nil
|
||||
}
|
||||
@@ -115,19 +114,6 @@ func checkKeywordRelevance(content string, keywords []string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// validateGoldenSetForRun 对一次 eval run 的所有 expected_ids 做空断言自守
|
||||
func validateGoldenSetForRun(queries []struct {
|
||||
Query string `json:"query"`
|
||||
ExpectedIDs []string `json:"expected_ids"`
|
||||
}) error {
|
||||
for _, q := range queries {
|
||||
if err := validateExpectedIDs(q.Query, q.ExpectedIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type EvalAPI struct {
|
||||
Pipeline *storage.RecallPipeline
|
||||
LanceDB storage.LanceDB
|
||||
@@ -158,7 +144,6 @@ type EvalRun struct {
|
||||
RanAt time.Time `json:"ran_at"`
|
||||
}
|
||||
|
||||
// EvalQueryDetail
|
||||
type EvalQueryDetail struct {
|
||||
Query string `json:"query"`
|
||||
ExpectedIDs []string `json:"expected_ids"`
|
||||
@@ -173,8 +158,8 @@ type EvalQueryDetail struct {
|
||||
// GoldenRunResult 金标 run 的扩展结果(含关键词判据)
|
||||
type GoldenRunResult struct {
|
||||
EvalRun
|
||||
KeywordPass int `json:"keyword_pass"`
|
||||
KeywordFail int `json:"keyword_fail"`
|
||||
KeywordPass int `json:"keyword_pass"`
|
||||
KeywordFail int `json:"keyword_fail"`
|
||||
KeywordTotal int `json:"keyword_total"`
|
||||
}
|
||||
|
||||
@@ -215,6 +200,14 @@ func (ea *EvalAPI) Run(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// 空断言自守:裸前缀 mem_ / 空串 / None → 直接报错,不许静默 PASS
|
||||
for _, q := range req.Queries {
|
||||
if err := validateExpectedIDs(q.Query, q.ExpectedIDs); err != nil {
|
||||
respondError(w, 400, err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// 默认 namespace 为 shared,支持传入 hermes 等
|
||||
ns := req.Namespace
|
||||
if ns == "" {
|
||||
@@ -334,6 +327,160 @@ func (ea *EvalAPI) Run(w http.ResponseWriter, r *http.Request) {
|
||||
respond(w, 200, run)
|
||||
}
|
||||
|
||||
// POST /api/v1/eval/golden — 使用金标集跑评估(含关键词相关性判据)
|
||||
// 加载 testdata/golden-set.json,对每条执行 recall + 关键词检查
|
||||
// 返回 GoldenRunResult(含 keyword_pass/fail/total)
|
||||
func (ea *EvalAPI) RunGolden(w http.ResponseWriter, r *http.Request) {
|
||||
var req struct {
|
||||
Namespace string `json:"namespace"`
|
||||
GoldenPath string `json:"golden_path"` // 可选,默认 testdata/golden-set.json
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
respondError(w, 400, "invalid body")
|
||||
return
|
||||
}
|
||||
ns := req.Namespace
|
||||
if ns == "" {
|
||||
ns = "shared"
|
||||
}
|
||||
gs, err := LoadGoldenSet(req.GoldenPath)
|
||||
if err != nil {
|
||||
respondError(w, 500, "failed to load golden set: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
var totalPrecision, totalRecall, totalMRR, totalNDCG float64
|
||||
totalQueries := 0
|
||||
keywordPass, keywordFail, keywordTotal := 0, 0, 0
|
||||
var queryDetails []EvalQueryDetail
|
||||
var failures []string
|
||||
|
||||
for _, item := range gs.Items {
|
||||
// 空断言自守
|
||||
if err := validateExpectedIDs(item.Query, item.ExpectedIDs); err != nil {
|
||||
respondError(w, 400, err.Error())
|
||||
return
|
||||
}
|
||||
// 无关键词判据 → 禁止(会退化成恒真)
|
||||
if len(item.MustIncludeKeywords) == 0 {
|
||||
failures = append(failures, item.Query+": 无关键词判据(禁止退化成恒真)")
|
||||
continue
|
||||
}
|
||||
|
||||
results, err := ea.Pipeline.Recall(item.Query, ns, 5, 0.5)
|
||||
if err != nil {
|
||||
failures = append(failures, item.Query+": recall 失败 "+err.Error())
|
||||
keywordFail++
|
||||
keywordTotal++
|
||||
continue
|
||||
}
|
||||
|
||||
// ID 命中检查
|
||||
hits := 0
|
||||
var hitIDs []string
|
||||
var missIDs []string
|
||||
expectedSet := makeSet(item.ExpectedIDs)
|
||||
for i, res := range results {
|
||||
if expectedSet[res.ID] && i < 5 {
|
||||
hits++
|
||||
hitIDs = append(hitIDs, res.ID)
|
||||
}
|
||||
}
|
||||
for _, eid := range item.ExpectedIDs {
|
||||
if !contains(hitIDs, eid) {
|
||||
missIDs = append(missIDs, eid)
|
||||
}
|
||||
}
|
||||
|
||||
// 关键词相关性判据:top-5 内容中至少一个关键词命中
|
||||
keywordOK := false
|
||||
for _, res := range results {
|
||||
if checkKeywordRelevance(res.Content, item.MustIncludeKeywords) {
|
||||
keywordOK = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
pAt5 := 0.0
|
||||
if len(results) > 0 { pAt5 = float64(hits) / float64(len(results)) }
|
||||
totalPrecision += pAt5
|
||||
|
||||
rAt5 := 0.0
|
||||
if len(item.ExpectedIDs) > 0 { rAt5 = float64(hits) / float64(len(item.ExpectedIDs)) }
|
||||
totalRecall += rAt5
|
||||
|
||||
// MRR
|
||||
for i, res := range results {
|
||||
if expectedSet[res.ID] {
|
||||
totalMRR += 1.0 / float64(i+1)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// NDCG
|
||||
dcg, idcg := 0.0, 0.0
|
||||
for i, res := range results {
|
||||
rel := 0.0
|
||||
if expectedSet[res.ID] { rel = 1.0 }
|
||||
dcg += rel / log2(float64(i+2))
|
||||
if i < len(item.ExpectedIDs) { idcg += 1.0 / log2(float64(i+2)) }
|
||||
}
|
||||
if idcg > 0 { totalNDCG += dcg / idcg }
|
||||
|
||||
// 关键词统计
|
||||
keywordTotal++
|
||||
if keywordOK {
|
||||
keywordPass++
|
||||
} else {
|
||||
keywordFail++
|
||||
failures = append(failures, item.Query+": 关键词未命中 "+strings.Join(item.MustIncludeKeywords, ","))
|
||||
}
|
||||
|
||||
queryDetails = append(queryDetails, EvalQueryDetail{
|
||||
Query: item.Query,
|
||||
ExpectedIDs: item.ExpectedIDs,
|
||||
MustIncludeKeywords: item.MustIncludeKeywords,
|
||||
Hits: hitIDs,
|
||||
Misses: missIDs,
|
||||
RecallAt5: rAt5,
|
||||
PrecisionAt5: pAt5,
|
||||
KeywordOK: keywordOK,
|
||||
})
|
||||
|
||||
totalQueries++
|
||||
}
|
||||
|
||||
if totalQueries == 0 {
|
||||
respondError(w, 500, "all golden queries failed")
|
||||
return
|
||||
}
|
||||
|
||||
run := &EvalRun{
|
||||
ID: time.Now().Format("20060102-150405"),
|
||||
Model: "golden-set",
|
||||
RecallAt5: totalRecall / float64(totalQueries),
|
||||
PrecisionAt5: totalPrecision / float64(totalQueries),
|
||||
MRR: totalMRR / float64(totalQueries),
|
||||
NDCG: totalNDCG / float64(totalQueries),
|
||||
QueryDetails: queryDetails,
|
||||
Queries: totalQueries,
|
||||
RanAt: time.Now(),
|
||||
}
|
||||
|
||||
evals.mu.Lock()
|
||||
evals.runs = append(evals.runs, run)
|
||||
evals.mu.Unlock()
|
||||
|
||||
result := GoldenRunResult{
|
||||
EvalRun: *run,
|
||||
KeywordPass: keywordPass,
|
||||
KeywordFail: keywordFail,
|
||||
KeywordTotal: keywordTotal,
|
||||
}
|
||||
_ = failures
|
||||
respond(w, 200, result)
|
||||
}
|
||||
|
||||
// GET /api/v1/eval/history
|
||||
func (ea *EvalAPI) History(w http.ResponseWriter, r *http.Request) {
|
||||
evals.mu.RLock()
|
||||
@@ -363,6 +510,8 @@ func (ea *EvalAPI) History(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// POST /api/v1/eval/generate — 生成 4类×3级=12条金标查询
|
||||
// 对每个 query 调用 recall 获取 expected_ids
|
||||
// 注意:此端点用于 bootstrap 基线——首次跑时没有基线,用 recall top-3 填充。
|
||||
// 后续应改用 LoadGoldenSet 加载已校准的金标集(含具体 ID + 关键词判据)。
|
||||
func (ea *EvalAPI) Generate(w http.ResponseWriter, r *http.Request) {
|
||||
var req struct {
|
||||
Namespace string `json:"namespace"`
|
||||
@@ -414,6 +563,17 @@ func (ea *EvalAPI) Generate(w http.ResponseWriter, r *http.Request) {
|
||||
expectedIDs = append(expectedIDs, r.ID)
|
||||
}
|
||||
}
|
||||
// 空断言自守:如果 recall 返回 0 条,不得写入空 expected_ids(那是 bootstrap,不是校准)
|
||||
if len(expectedIDs) == 0 {
|
||||
results = append(results, map[string]interface{}{
|
||||
"query": gq.query,
|
||||
"expected_ids": []string{},
|
||||
"category": gq.cat,
|
||||
"difficulty": gq.difficulty,
|
||||
"warning": "recall 返回 0 条——bootstrap 基线为空,需人工校准",
|
||||
})
|
||||
continue
|
||||
}
|
||||
results = append(results, map[string]interface{}{
|
||||
"query": gq.query,
|
||||
"expected_ids": expectedIDs,
|
||||
@@ -425,7 +585,7 @@ func (ea *EvalAPI) Generate(w http.ResponseWriter, r *http.Request) {
|
||||
respond(w, 200, map[string]interface{}{
|
||||
"queries": results,
|
||||
"count": len(results),
|
||||
"note": "4类别×3难度=12条金标查询,通过 recall 自动获取 expected_ids",
|
||||
"note": "4类别×3难度=12条金标查询,通过 recall 自动获取 expected_ids。注意:bootstrap 模式下 expected_ids 来自实时 recall,非校准基线。校准后的金标集见 testdata/golden-set.json(LoadGoldenSet)。",
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -1159,6 +1159,7 @@ func NewServer() http.Handler {
|
||||
mux.HandleFunc("/api/v1/eval/run", evalAPI.Run)
|
||||
mux.HandleFunc("/api/v1/eval/history", evalAPI.History)
|
||||
mux.HandleFunc("/api/v1/eval/generate", evalAPI.Generate)
|
||||
mux.HandleFunc("/api/v1/eval/golden", evalAPI.RunGolden)
|
||||
mux.HandleFunc("/api/v1/tuning/status", routes.Tuner.StatusHandler)
|
||||
mux.HandleFunc("/api/v1/tuning/run", routes.Tuner.RunHandler)
|
||||
mux.HandleFunc("/api/v1/tuning/analytics", routes.Tuner.AnalyticsHandler)
|
||||
|
||||
Reference in New Issue
Block a user