diff --git a/go/internal/api/routes/core.go b/go/internal/api/routes/core.go index 880eaa5..d34ba91 100644 --- a/go/internal/api/routes/core.go +++ b/go/internal/api/routes/core.go @@ -48,8 +48,29 @@ func respond(w http.ResponseWriter, code int, data interface{}) { json.NewEncoder(w).Encode(data) } +// Unified error body schema (t_2d4e3b92, 2026-09-21): +// +// HTTP non-2xx → {"error":{"code":,"message":,"retry_after_s":}} +// HTTP 2xx → success schema only (recall: {"results":[...],"count":N,...}) +// +// Contract for callers: 200 + "results" key present = success (results MAY be empty — +// a legitimate "no match"); non-2xx = failure (never parse as success). Do NOT +// distinguish by len(results)==0. func respondError(w http.ResponseWriter, code int, msg string) { - respond(w, code, map[string]string{"error": msg}) + respond(w, code, map[string]interface{}{"error": map[string]interface{}{ + "code": "failure", + "message": msg, + "retry_after_s": 0, + }}) +} + +// respondErrorSchema returns a non-2xx with the unified error schema. +func respondErrorSchema(w http.ResponseWriter, code int, codeStr, msg string, retryAfterS int) { + respond(w, code, map[string]interface{}{"error": map[string]interface{}{ + "code": codeStr, + "message": msg, + "retry_after_s": retryAfterS, + }}) } // POST /api/v1/commit @@ -273,11 +294,27 @@ func (a *API) Recall(w http.ResponseWriter, r *http.Request) { switch req.Mode { case "keyword": // Pure keyword search: LanceDB + BM25 re-rank, or graph.db fallback + // t_2d4e3b92: keyword 主路径依赖向量编码;编码失败(依赖不可用)⇒ 非 2xx, + // 不再把错误伪装成 200 + 错误体。 kwResults, kwErr := a.Pipeline.Recall(req.Query, req.Namespace, req.Limit, req.Diversity) - if kwErr != nil || len(kwResults) == 0 { + if kwErr != nil { + // 主路径硬错误:兜底命中也按"主路径故障"报告(502),调用方据此走失败路径 + if fb := a.GraphStore.FallbackTextSearch(req.Query, req.Namespace, req.Limit); len(fb) > 0 { + _ = fb + } + respondErrorSchema(w, 502, "recall_backend_error", + "keyword recall failed: "+kwErr.Error(), 5) + log.Printf("[recall:keyword] failed: %v", kwErr) + return + } + if len(kwResults) == 0 { + // 真无匹配(正常业务):成功 schema,results=[] + candidates_scanned 供区分 fallbackResults := a.GraphStore.FallbackTextSearch(req.Query, req.Namespace, req.Limit) results = convertFallbackResults(fallbackResults) - respond(w, 200, map[string]interface{}{"results": results, "count": len(results), "mode": "keyword"}) + respond(w, 200, map[string]interface{}{ + "results": results, "count": len(results), "mode": "keyword", + "candidates_scanned": 0, // keyword 路径未做 ANN 扫描 + }) return } // Re-rank by BM25 only @@ -298,13 +335,16 @@ func (a *API) Recall(w http.ResponseWriter, r *http.Request) { results, err = a.Pipeline.Recall(req.Query, req.Namespace, req.Limit, req.Diversity) } if err != nil { - // P0: 降级到 graph.db 关键词搜索 + // P0: 降级到 graph.db 关键词搜索(仅 hybrid/semantic 主路径依赖向量/rerank) fallbackResults := a.GraphStore.FallbackTextSearch(req.Query, req.Namespace, req.Limit) if len(fallbackResults) > 0 { results = convertFallbackResults(fallbackResults) w.Header().Set("X-Fallback", "graph") } else { - respondError(w, 500, "recall failed: "+err.Error()) + // t_2d4e3b92: 主路径与兜底都失败 ⇒ 非 2xx + 统一错误体(绝不在 200 里塞错误) + respondErrorSchema(w, 502, "recall_backend_error", + "recall failed: "+err.Error()+"; graph fallback also empty", 5) + log.Printf("[recall] failed: %v; fallback empty", err) return } } diff --git a/go/internal/api/routes/eval.go b/go/internal/api/routes/eval.go index 6041cfd..78be494 100644 --- a/go/internal/api/routes/eval.go +++ b/go/internal/api/routes/eval.go @@ -4,11 +4,130 @@ package routes import ( "encoding/json" "net/http" + "os" + "strings" "sync" "time" "github.com/xiaoxue/memoryweave/internal/storage" ) + +// ─── 金标集定义 ────────────────────────────────────────────── + +// GoldenSetItem 单条金标:query + 具体 expected_ids + 关键词相关性判据 +type GoldenSetItem struct { + Query string `json:"query"` + ExpectedIDs []string `json:"expected_ids"` + MustIncludeKeywords []string `json:"must_include_keywords"` + Category string `json:"category,omitempty"` + Difficulty string `json:"difficulty,omitempty"` + IDBaseline string `json:"id_baseline,omitempty"` +} + +// GoldenSet 金标集 +type GoldenSet struct { + Description string `json:"description"` + Items []GoldenSetItem `json:"items"` +} + +// LoadGoldenSet 从 testdata/ 加载金标集,支持相对路径(生产用)和绝对路径(测试用) +func LoadGoldenSet(path string) (*GoldenSet, error) { + if path == "" { + // 编译后无法直接拿到源码路径,回退到 CWD 的 testdata/ + path = "testdata/golden-set.json" + } + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var gs GoldenSet + if err := json.Unmarshal(data, &gs); err != nil { + return nil, err + } + // 空断言自守:禁止裸前缀/空串混入(避免恒真) + for i, item := range gs.Items { + if err := validateExpectedIDs(item.Query, item.ExpectedIDs); err != nil { + return nil, err + } + _ = i + } + return &gs, nil +} + +// validateExpectedIDs 空断言自守:裸前缀 "mem_"、空串、None → 直接报错,不许静默 PASS +func validateExpectedIDs(query string, expectedIDs []string) error { + if len(expectedIDs) == 0 { + return nil // 允许空(bootstrap 时还没基线) + } + for _, id := range expectedIDs { + trimmed := strings.TrimSpace(id) + // 裸前缀 "mem_"(不含具体 ID 数字) + if trimmed == "mem_" || trimmed == "mem" { + return &VacuousAssertionError{ + Query: query, + BadID: id, + Reason: "裸前缀 mem_ 是恒真断言——所有记忆 ID 都以 mem_ 开头", + } + } + // 空串 + if trimmed == "" { + return &VacuousAssertionError{ + Query: query, + BadID: id, + Reason: "空串期望值(无意义断言)", + } + } + // None(JSON null 解码成空字符串已处理,但检查 "None" 字符串) + if strings.EqualFold(trimmed, "none") { + return &VacuousAssertionError{ + Query: query, + BadID: id, + Reason: "None 期望值(无意义断言)", + } + } + } + return nil +} + +// VacuousAssertionError 空断言错误 +type VacuousAssertionError struct { + Query string + BadID string + Reason string +} + +func (e *VacuousAssertionError) Error() string { + return "FATAL 空断言被检出(恒真): query=" + e.Query + " 期望=" + e.BadID + " — " + e.Reason +} + +// checkKeywordRelevance 关键词相关性判据:返回内容必须包含至少一个关键词 +func checkKeywordRelevance(content string, keywords []string) bool { + if len(keywords) == 0 { + // 无关键词判据 → 视为不通过(禁止退化为恒真) + return false + } + lower := strings.ToLower(content) + for _, kw := range keywords { + if strings.Contains(lower, strings.ToLower(kw)) { + return true + } + } + return false +} + +// validateGoldenSetForRun 对一次 eval run 的所有 expected_ids 做空断言自守 +func validateGoldenSetForRun(queries []struct { + Query string `json:"query"` + ExpectedIDs []string `json:"expected_ids"` +}) error { + for _, q := range queries { + if err := validateExpectedIDs(q.Query, q.ExpectedIDs); err != nil { + return err + } + } + return nil +} + type EvalAPI struct { Pipeline *storage.RecallPipeline LanceDB storage.LanceDB @@ -39,13 +158,24 @@ type EvalRun struct { RanAt time.Time `json:"ran_at"` } +// EvalQueryDetail type EvalQueryDetail struct { - Query string `json:"query"` - ExpectedIDs []string `json:"expected_ids"` - Hits []string `json:"hits"` - Misses []string `json:"misses"` - RecallAt5 float64 `json:"recall_at_5"` - PrecisionAt5 float64 `json:"precision_at_5"` + Query string `json:"query"` + ExpectedIDs []string `json:"expected_ids"` + MustIncludeKeywords []string `json:"must_include_keywords,omitempty"` + Hits []string `json:"hits"` + Misses []string `json:"misses"` + RecallAt5 float64 `json:"recall_at_5"` + PrecisionAt5 float64 `json:"precision_at_5"` + KeywordOK bool `json:"keyword_ok,omitempty"` +} + +// GoldenRunResult 金标 run 的扩展结果(含关键词判据) +type GoldenRunResult struct { + EvalRun + KeywordPass int `json:"keyword_pass"` + KeywordFail int `json:"keyword_fail"` + KeywordTotal int `json:"keyword_total"` } type ConsolidationHits struct { diff --git a/go/internal/api/routes/testdata/golden-set.json b/go/internal/api/routes/testdata/golden-set.json new file mode 100644 index 0000000..e68124a --- /dev/null +++ b/go/internal/api/routes/testdata/golden-set.json @@ -0,0 +1,117 @@ +{ + "description": "织忆召回金标检查集(12条基线 + 2条目标知识)", + "items": [ + { + "query": "织忆是什么", + "expected_ids": ["mem_1789153726378940805"], + "must_include_keywords": ["织忆"], + "category": "system_fact", + "difficulty": "easy", + "id_baseline": "mem_1789153726378940805" + }, + { + "query": "织忆的图谱扩展机制是什么", + "expected_ids": ["mem_1789823837902661249"], + "must_include_keywords": ["图谱"], + "category": "system_fact", + "difficulty": "medium", + "id_baseline": "mem_1789823837902661249" + }, + { + "query": "织忆如何通过 E1 提取和 MMR 实现多样去重", + "expected_ids": ["mem_1789915227303612056"], + "must_include_keywords": ["MMR"], + "category": "system_fact", + "difficulty": "hard", + "id_baseline": "mem_1789915227303612056" + }, + { + "query": "牧尘的偏好是什么", + "expected_ids": ["mem_1789667544872361813"], + "must_include_keywords": ["牧尘"], + "category": "user_pref", + "difficulty": "easy", + "id_baseline": "mem_1789667544872361813" + }, + { + "query": "牧尘喜欢什么样的工作方式", + "expected_ids": ["mem_1789837840173161705"], + "must_include_keywords": ["牧尘"], + "category": "user_pref", + "difficulty": "medium", + "id_baseline": "mem_1789837840173161705" + }, + { + "query": "如何根据牧尘的偏好调整记忆检索策略", + "expected_ids": ["mem_1789190217889697697"], + "must_include_keywords": ["记忆"], + "category": "user_pref", + "difficulty": "hard", + "id_baseline": "mem_1789190217889697697" + }, + { + "query": "当前项目的技术栈是什么", + "expected_ids": ["mem_1789680870868739202"], + "must_include_keywords": ["技术栈"], + "category": "proj_context", + "difficulty": "easy", + "id_baseline": "mem_1789680870868739202" + }, + { + "query": "织忆项目有哪些核心组件", + "expected_ids": ["mem_1789154058288011112"], + "must_include_keywords": ["织忆"], + "category": "proj_context", + "difficulty": "medium", + "id_baseline": "mem_1789154058288011112" + }, + { + "query": "织忆和其他记忆系统相比有什么架构优势", + "expected_ids": ["mem_1789153726378940805"], + "must_include_keywords": ["织忆"], + "category": "proj_context", + "difficulty": "hard", + "id_baseline": "mem_1789153726378940805" + }, + { + "query": "如何使用 recall 接口", + "expected_ids": ["mem_1787272787836687543"], + "must_include_keywords": ["recall"], + "category": "tool_usage", + "difficulty": "easy", + "id_baseline": "mem_1787272787836687543" + }, + { + "query": "织忆有哪些管理工具", + "expected_ids": ["mem_1789153726378940805"], + "must_include_keywords": ["织忆"], + "category": "tool_usage", + "difficulty": "medium", + "id_baseline": "mem_1789153726378940805" + }, + { + "query": "如何通过 API 扩展织忆功能", + "expected_ids": ["mem_1789153726378940805"], + "must_include_keywords": ["织忆"], + "category": "tool_usage", + "difficulty": "hard", + "id_baseline": "mem_1789153726378940805" + }, + { + "query": "本地 llama 服务 8080 端口", + "expected_ids": ["mem_1789229548476546193"], + "must_include_keywords": ["8080"], + "category": "target_knowledge", + "difficulty": "easy", + "id_baseline": "mem_1789229548476546193" + }, + { + "query": "session-to-zhiyi.py 脚本名", + "expected_ids": ["mem_1788977361686058470"], + "must_include_keywords": ["session-to-zhiyi"], + "category": "target_knowledge", + "difficulty": "easy", + "id_baseline": "mem_1788977361686058470" + } + ] +}