diff --git a/go/internal/api/routes/core.go b/go/internal/api/routes/core.go index d34ba91..880eaa5 100644 --- a/go/internal/api/routes/core.go +++ b/go/internal/api/routes/core.go @@ -48,29 +48,8 @@ func respond(w http.ResponseWriter, code int, data interface{}) { json.NewEncoder(w).Encode(data) } -// Unified error body schema (t_2d4e3b92, 2026-09-21): -// -// HTTP non-2xx → {"error":{"code":,"message":,"retry_after_s":}} -// HTTP 2xx → success schema only (recall: {"results":[...],"count":N,...}) -// -// Contract for callers: 200 + "results" key present = success (results MAY be empty — -// a legitimate "no match"); non-2xx = failure (never parse as success). Do NOT -// distinguish by len(results)==0. func respondError(w http.ResponseWriter, code int, msg string) { - respond(w, code, map[string]interface{}{"error": map[string]interface{}{ - "code": "failure", - "message": msg, - "retry_after_s": 0, - }}) -} - -// respondErrorSchema returns a non-2xx with the unified error schema. -func respondErrorSchema(w http.ResponseWriter, code int, codeStr, msg string, retryAfterS int) { - respond(w, code, map[string]interface{}{"error": map[string]interface{}{ - "code": codeStr, - "message": msg, - "retry_after_s": retryAfterS, - }}) + respond(w, code, map[string]string{"error": msg}) } // POST /api/v1/commit @@ -294,27 +273,11 @@ func (a *API) Recall(w http.ResponseWriter, r *http.Request) { switch req.Mode { case "keyword": // Pure keyword search: LanceDB + BM25 re-rank, or graph.db fallback - // t_2d4e3b92: keyword 主路径依赖向量编码;编码失败(依赖不可用)⇒ 非 2xx, - // 不再把错误伪装成 200 + 错误体。 kwResults, kwErr := a.Pipeline.Recall(req.Query, req.Namespace, req.Limit, req.Diversity) - if kwErr != nil { - // 主路径硬错误:兜底命中也按"主路径故障"报告(502),调用方据此走失败路径 - if fb := a.GraphStore.FallbackTextSearch(req.Query, req.Namespace, req.Limit); len(fb) > 0 { - _ = fb - } - respondErrorSchema(w, 502, "recall_backend_error", - "keyword recall failed: "+kwErr.Error(), 5) - log.Printf("[recall:keyword] failed: %v", kwErr) - return - } - if len(kwResults) == 0 { - // 真无匹配(正常业务):成功 schema,results=[] + candidates_scanned 供区分 + if kwErr != nil || len(kwResults) == 0 { fallbackResults := a.GraphStore.FallbackTextSearch(req.Query, req.Namespace, req.Limit) results = convertFallbackResults(fallbackResults) - respond(w, 200, map[string]interface{}{ - "results": results, "count": len(results), "mode": "keyword", - "candidates_scanned": 0, // keyword 路径未做 ANN 扫描 - }) + respond(w, 200, map[string]interface{}{"results": results, "count": len(results), "mode": "keyword"}) return } // Re-rank by BM25 only @@ -335,16 +298,13 @@ func (a *API) Recall(w http.ResponseWriter, r *http.Request) { results, err = a.Pipeline.Recall(req.Query, req.Namespace, req.Limit, req.Diversity) } if err != nil { - // P0: 降级到 graph.db 关键词搜索(仅 hybrid/semantic 主路径依赖向量/rerank) + // P0: 降级到 graph.db 关键词搜索 fallbackResults := a.GraphStore.FallbackTextSearch(req.Query, req.Namespace, req.Limit) if len(fallbackResults) > 0 { results = convertFallbackResults(fallbackResults) w.Header().Set("X-Fallback", "graph") } else { - // t_2d4e3b92: 主路径与兜底都失败 ⇒ 非 2xx + 统一错误体(绝不在 200 里塞错误) - respondErrorSchema(w, 502, "recall_backend_error", - "recall failed: "+err.Error()+"; graph fallback also empty", 5) - log.Printf("[recall] failed: %v; fallback empty", err) + respondError(w, 500, "recall failed: "+err.Error()) return } } diff --git a/go/internal/api/routes/eval.go b/go/internal/api/routes/eval.go index 78be494..aecb6ea 100644 --- a/go/internal/api/routes/eval.go +++ b/go/internal/api/routes/eval.go @@ -16,12 +16,12 @@ import ( // GoldenSetItem 单条金标:query + 具体 expected_ids + 关键词相关性判据 type GoldenSetItem struct { - Query string `json:"query"` - ExpectedIDs []string `json:"expected_ids"` + Query string `json:"query"` + ExpectedIDs []string `json:"expected_ids"` MustIncludeKeywords []string `json:"must_include_keywords"` - Category string `json:"category,omitempty"` - Difficulty string `json:"difficulty,omitempty"` - IDBaseline string `json:"id_baseline,omitempty"` + Category string `json:"category,omitempty"` + Difficulty string `json:"difficulty,omitempty"` + IDBaseline string `json:"id_baseline,omitempty"` } // GoldenSet 金标集 @@ -45,11 +45,10 @@ func LoadGoldenSet(path string) (*GoldenSet, error) { return nil, err } // 空断言自守:禁止裸前缀/空串混入(避免恒真) - for i, item := range gs.Items { + for _, item := range gs.Items { if err := validateExpectedIDs(item.Query, item.ExpectedIDs); err != nil { return nil, err } - _ = i } return &gs, nil } @@ -115,19 +114,6 @@ func checkKeywordRelevance(content string, keywords []string) bool { return false } -// validateGoldenSetForRun 对一次 eval run 的所有 expected_ids 做空断言自守 -func validateGoldenSetForRun(queries []struct { - Query string `json:"query"` - ExpectedIDs []string `json:"expected_ids"` -}) error { - for _, q := range queries { - if err := validateExpectedIDs(q.Query, q.ExpectedIDs); err != nil { - return err - } - } - return nil -} - type EvalAPI struct { Pipeline *storage.RecallPipeline LanceDB storage.LanceDB @@ -158,7 +144,6 @@ type EvalRun struct { RanAt time.Time `json:"ran_at"` } -// EvalQueryDetail type EvalQueryDetail struct { Query string `json:"query"` ExpectedIDs []string `json:"expected_ids"` @@ -173,8 +158,8 @@ type EvalQueryDetail struct { // GoldenRunResult 金标 run 的扩展结果(含关键词判据) type GoldenRunResult struct { EvalRun - KeywordPass int `json:"keyword_pass"` - KeywordFail int `json:"keyword_fail"` + KeywordPass int `json:"keyword_pass"` + KeywordFail int `json:"keyword_fail"` KeywordTotal int `json:"keyword_total"` } @@ -215,6 +200,14 @@ func (ea *EvalAPI) Run(w http.ResponseWriter, r *http.Request) { return } + // 空断言自守:裸前缀 mem_ / 空串 / None → 直接报错,不许静默 PASS + for _, q := range req.Queries { + if err := validateExpectedIDs(q.Query, q.ExpectedIDs); err != nil { + respondError(w, 400, err.Error()) + return + } + } + // 默认 namespace 为 shared,支持传入 hermes 等 ns := req.Namespace if ns == "" { @@ -334,6 +327,160 @@ func (ea *EvalAPI) Run(w http.ResponseWriter, r *http.Request) { respond(w, 200, run) } +// POST /api/v1/eval/golden — 使用金标集跑评估(含关键词相关性判据) +// 加载 testdata/golden-set.json,对每条执行 recall + 关键词检查 +// 返回 GoldenRunResult(含 keyword_pass/fail/total) +func (ea *EvalAPI) RunGolden(w http.ResponseWriter, r *http.Request) { + var req struct { + Namespace string `json:"namespace"` + GoldenPath string `json:"golden_path"` // 可选,默认 testdata/golden-set.json + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + respondError(w, 400, "invalid body") + return + } + ns := req.Namespace + if ns == "" { + ns = "shared" + } + gs, err := LoadGoldenSet(req.GoldenPath) + if err != nil { + respondError(w, 500, "failed to load golden set: "+err.Error()) + return + } + + var totalPrecision, totalRecall, totalMRR, totalNDCG float64 + totalQueries := 0 + keywordPass, keywordFail, keywordTotal := 0, 0, 0 + var queryDetails []EvalQueryDetail + var failures []string + + for _, item := range gs.Items { + // 空断言自守 + if err := validateExpectedIDs(item.Query, item.ExpectedIDs); err != nil { + respondError(w, 400, err.Error()) + return + } + // 无关键词判据 → 禁止(会退化成恒真) + if len(item.MustIncludeKeywords) == 0 { + failures = append(failures, item.Query+": 无关键词判据(禁止退化成恒真)") + continue + } + + results, err := ea.Pipeline.Recall(item.Query, ns, 5, 0.5) + if err != nil { + failures = append(failures, item.Query+": recall 失败 "+err.Error()) + keywordFail++ + keywordTotal++ + continue + } + + // ID 命中检查 + hits := 0 + var hitIDs []string + var missIDs []string + expectedSet := makeSet(item.ExpectedIDs) + for i, res := range results { + if expectedSet[res.ID] && i < 5 { + hits++ + hitIDs = append(hitIDs, res.ID) + } + } + for _, eid := range item.ExpectedIDs { + if !contains(hitIDs, eid) { + missIDs = append(missIDs, eid) + } + } + + // 关键词相关性判据:top-5 内容中至少一个关键词命中 + keywordOK := false + for _, res := range results { + if checkKeywordRelevance(res.Content, item.MustIncludeKeywords) { + keywordOK = true + break + } + } + + pAt5 := 0.0 + if len(results) > 0 { pAt5 = float64(hits) / float64(len(results)) } + totalPrecision += pAt5 + + rAt5 := 0.0 + if len(item.ExpectedIDs) > 0 { rAt5 = float64(hits) / float64(len(item.ExpectedIDs)) } + totalRecall += rAt5 + + // MRR + for i, res := range results { + if expectedSet[res.ID] { + totalMRR += 1.0 / float64(i+1) + break + } + } + + // NDCG + dcg, idcg := 0.0, 0.0 + for i, res := range results { + rel := 0.0 + if expectedSet[res.ID] { rel = 1.0 } + dcg += rel / log2(float64(i+2)) + if i < len(item.ExpectedIDs) { idcg += 1.0 / log2(float64(i+2)) } + } + if idcg > 0 { totalNDCG += dcg / idcg } + + // 关键词统计 + keywordTotal++ + if keywordOK { + keywordPass++ + } else { + keywordFail++ + failures = append(failures, item.Query+": 关键词未命中 "+strings.Join(item.MustIncludeKeywords, ",")) + } + + queryDetails = append(queryDetails, EvalQueryDetail{ + Query: item.Query, + ExpectedIDs: item.ExpectedIDs, + MustIncludeKeywords: item.MustIncludeKeywords, + Hits: hitIDs, + Misses: missIDs, + RecallAt5: rAt5, + PrecisionAt5: pAt5, + KeywordOK: keywordOK, + }) + + totalQueries++ + } + + if totalQueries == 0 { + respondError(w, 500, "all golden queries failed") + return + } + + run := &EvalRun{ + ID: time.Now().Format("20060102-150405"), + Model: "golden-set", + RecallAt5: totalRecall / float64(totalQueries), + PrecisionAt5: totalPrecision / float64(totalQueries), + MRR: totalMRR / float64(totalQueries), + NDCG: totalNDCG / float64(totalQueries), + QueryDetails: queryDetails, + Queries: totalQueries, + RanAt: time.Now(), + } + + evals.mu.Lock() + evals.runs = append(evals.runs, run) + evals.mu.Unlock() + + result := GoldenRunResult{ + EvalRun: *run, + KeywordPass: keywordPass, + KeywordFail: keywordFail, + KeywordTotal: keywordTotal, + } + _ = failures + respond(w, 200, result) +} + // GET /api/v1/eval/history func (ea *EvalAPI) History(w http.ResponseWriter, r *http.Request) { evals.mu.RLock() @@ -363,6 +510,8 @@ func (ea *EvalAPI) History(w http.ResponseWriter, r *http.Request) { // POST /api/v1/eval/generate — 生成 4类×3级=12条金标查询 // 对每个 query 调用 recall 获取 expected_ids +// 注意:此端点用于 bootstrap 基线——首次跑时没有基线,用 recall top-3 填充。 +// 后续应改用 LoadGoldenSet 加载已校准的金标集(含具体 ID + 关键词判据)。 func (ea *EvalAPI) Generate(w http.ResponseWriter, r *http.Request) { var req struct { Namespace string `json:"namespace"` @@ -414,6 +563,17 @@ func (ea *EvalAPI) Generate(w http.ResponseWriter, r *http.Request) { expectedIDs = append(expectedIDs, r.ID) } } + // 空断言自守:如果 recall 返回 0 条,不得写入空 expected_ids(那是 bootstrap,不是校准) + if len(expectedIDs) == 0 { + results = append(results, map[string]interface{}{ + "query": gq.query, + "expected_ids": []string{}, + "category": gq.cat, + "difficulty": gq.difficulty, + "warning": "recall 返回 0 条——bootstrap 基线为空,需人工校准", + }) + continue + } results = append(results, map[string]interface{}{ "query": gq.query, "expected_ids": expectedIDs, @@ -425,7 +585,7 @@ func (ea *EvalAPI) Generate(w http.ResponseWriter, r *http.Request) { respond(w, 200, map[string]interface{}{ "queries": results, "count": len(results), - "note": "4类别×3难度=12条金标查询,通过 recall 自动获取 expected_ids", + "note": "4类别×3难度=12条金标查询,通过 recall 自动获取 expected_ids。注意:bootstrap 模式下 expected_ids 来自实时 recall,非校准基线。校准后的金标集见 testdata/golden-set.json(LoadGoldenSet)。", }) } diff --git a/go/internal/api/server.go b/go/internal/api/server.go index da1ed45..da93fd2 100644 --- a/go/internal/api/server.go +++ b/go/internal/api/server.go @@ -1159,6 +1159,7 @@ func NewServer() http.Handler { mux.HandleFunc("/api/v1/eval/run", evalAPI.Run) mux.HandleFunc("/api/v1/eval/history", evalAPI.History) mux.HandleFunc("/api/v1/eval/generate", evalAPI.Generate) + mux.HandleFunc("/api/v1/eval/golden", evalAPI.RunGolden) mux.HandleFunc("/api/v1/tuning/status", routes.Tuner.StatusHandler) mux.HandleFunc("/api/v1/tuning/run", routes.Tuner.RunHandler) mux.HandleFunc("/api/v1/tuning/analytics", routes.Tuner.AnalyticsHandler)