Files
memoryweave/go/internal/api/routes/eval.go
T

484 lines
13 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// 织忆 MemoryWeave — 评估系统 API
package routes
import (
"encoding/json"
"net/http"
"os"
"strings"
"sync"
"time"
"github.com/xiaoxue/memoryweave/internal/storage"
)
// ─── 金标集定义 ──────────────────────────────────────────────
// GoldenSetItem 单条金标:query + 具体 expected_ids + 关键词相关性判据
type GoldenSetItem struct {
Query string `json:"query"`
ExpectedIDs []string `json:"expected_ids"`
MustIncludeKeywords []string `json:"must_include_keywords"`
Category string `json:"category,omitempty"`
Difficulty string `json:"difficulty,omitempty"`
IDBaseline string `json:"id_baseline,omitempty"`
}
// GoldenSet 金标集
type GoldenSet struct {
Description string `json:"description"`
Items []GoldenSetItem `json:"items"`
}
// LoadGoldenSet 从 testdata/ 加载金标集,支持相对路径(生产用)和绝对路径(测试用)
func LoadGoldenSet(path string) (*GoldenSet, error) {
if path == "" {
// 编译后无法直接拿到源码路径,回退到 CWD 的 testdata/
path = "testdata/golden-set.json"
}
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
var gs GoldenSet
if err := json.Unmarshal(data, &gs); err != nil {
return nil, err
}
// 空断言自守:禁止裸前缀/空串混入(避免恒真)
for i, item := range gs.Items {
if err := validateExpectedIDs(item.Query, item.ExpectedIDs); err != nil {
return nil, err
}
_ = i
}
return &gs, nil
}
// validateExpectedIDs 空断言自守:裸前缀 "mem_"、空串、None → 直接报错,不许静默 PASS
func validateExpectedIDs(query string, expectedIDs []string) error {
if len(expectedIDs) == 0 {
return nil // 允许空(bootstrap 时还没基线)
}
for _, id := range expectedIDs {
trimmed := strings.TrimSpace(id)
// 裸前缀 "mem_"(不含具体 ID 数字)
if trimmed == "mem_" || trimmed == "mem" {
return &VacuousAssertionError{
Query: query,
BadID: id,
Reason: "裸前缀 mem_ 是恒真断言——所有记忆 ID 都以 mem_ 开头",
}
}
// 空串
if trimmed == "" {
return &VacuousAssertionError{
Query: query,
BadID: id,
Reason: "空串期望值(无意义断言)",
}
}
// None(JSON null 解码成空字符串已处理,但检查 "None" 字符串)
if strings.EqualFold(trimmed, "none") {
return &VacuousAssertionError{
Query: query,
BadID: id,
Reason: "None 期望值(无意义断言)",
}
}
}
return nil
}
// VacuousAssertionError 空断言错误
type VacuousAssertionError struct {
Query string
BadID string
Reason string
}
func (e *VacuousAssertionError) Error() string {
return "FATAL 空断言被检出(恒真): query=" + e.Query + " 期望=" + e.BadID + " — " + e.Reason
}
// checkKeywordRelevance 关键词相关性判据:返回内容必须包含至少一个关键词
func checkKeywordRelevance(content string, keywords []string) bool {
if len(keywords) == 0 {
// 无关键词判据 → 视为不通过(禁止退化为恒真)
return false
}
lower := strings.ToLower(content)
for _, kw := range keywords {
if strings.Contains(lower, strings.ToLower(kw)) {
return true
}
}
return false
}
// validateGoldenSetForRun 对一次 eval run 的所有 expected_ids 做空断言自守
func validateGoldenSetForRun(queries []struct {
Query string `json:"query"`
ExpectedIDs []string `json:"expected_ids"`
}) error {
for _, q := range queries {
if err := validateExpectedIDs(q.Query, q.ExpectedIDs); err != nil {
return err
}
}
return nil
}
type EvalAPI struct {
Pipeline *storage.RecallPipeline
LanceDB storage.LanceDB
}
type EvalRun struct {
ID string `json:"id"`
Model string `json:"model"`
RecallAt5 float64 `json:"recall_at_5"`
PrecisionAt5 float64 `json:"precision_at_5"`
MRR float64 `json:"mean_reciprocal_rank"`
NDCG float64 `json:"ndcg"`
// 按标签分组
RecallByTag map[string]float64 `json:"recall_at_5_by_tag,omitempty"`
PrecisionByTag map[string]float64 `json:"precision_at_5_by_tag,omitempty"`
// 按 Agent 分组
RecallByAgent map[string]float64 `json:"recall_by_agent,omitempty"`
// 详细查询结果
QueryDetails []EvalQueryDetail `json:"query_details,omitempty"`
// 深度整合影响
ConsolidationAwareHits *ConsolidationHits `json:"consolidation_aware_hits,omitempty"`
Queries int `json:"queries"`
RanAt time.Time `json:"ran_at"`
}
// EvalQueryDetail
type EvalQueryDetail struct {
Query string `json:"query"`
ExpectedIDs []string `json:"expected_ids"`
MustIncludeKeywords []string `json:"must_include_keywords,omitempty"`
Hits []string `json:"hits"`
Misses []string `json:"misses"`
RecallAt5 float64 `json:"recall_at_5"`
PrecisionAt5 float64 `json:"precision_at_5"`
KeywordOK bool `json:"keyword_ok,omitempty"`
}
// GoldenRunResult 金标 run 的扩展结果(含关键词判据)
type GoldenRunResult struct {
EvalRun
KeywordPass int `json:"keyword_pass"`
KeywordFail int `json:"keyword_fail"`
KeywordTotal int `json:"keyword_total"`
}
type ConsolidationHits struct {
TotalDistilledUsed int `json:"total_distilled_used"`
HitsAfterConsolidation int `json:"hits_after_consolidation"`
ConsolidationBenefit float64 `json:"consolidation_benefit"`
}
type evalStore struct {
mu sync.RWMutex
runs []*EvalRun
}
var evals = &evalStore{}
func NewEvalAPI(p *storage.RecallPipeline, ldb storage.LanceDB) *EvalAPI {
return &EvalAPI{Pipeline: p, LanceDB: ldb}
}
// POST /api/v1/eval/run
func (ea *EvalAPI) Run(w http.ResponseWriter, r *http.Request) {
var req struct {
Queries []struct {
Query string `json:"query"`
ExpectedIDs []string `json:"expected_ids"`
} `json:"queries"`
Model string `json:"model"`
Namespace string `json:"namespace"` // 支持指定 namespace
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
respondError(w, 400, "invalid body")
return
}
if len(req.Queries) == 0 {
respondError(w, 400, "at least one query required")
return
}
// 默认 namespace 为 shared,支持传入 hermes 等
ns := req.Namespace
if ns == "" {
ns = "shared"
}
var totalPrecision, totalRecall, totalMRR, totalNDCG float64
totalQueries := 0
// 按标签/Agent 分组统计
recallByTagMap := make(map[string][]float64)
precisionByTagMap := make(map[string][]float64)
recallByAgentMap := make(map[string][]float64)
var queryDetails []EvalQueryDetail
for _, q := range req.Queries {
results, err := ea.Pipeline.Recall(q.Query, ns, 5, 0.5)
if err != nil {
continue
}
// Precision@k
hits := 0
var hitIDs []string
var missIDs []string
expectedSet := makeSet(q.ExpectedIDs)
for i, res := range results {
if expectedSet[res.ID] && i < 5 {
hits++
hitIDs = append(hitIDs, res.ID)
}
}
// 找未命中的
for _, eid := range q.ExpectedIDs {
if !contains(hitIDs, eid) {
missIDs = append(missIDs, eid)
}
}
pAt5 := 0.0
if len(results) > 0 { pAt5 = float64(hits) / float64(len(results)) }
totalPrecision += pAt5
rAt5 := 0.0
if len(q.ExpectedIDs) > 0 { rAt5 = float64(hits) / float64(len(q.ExpectedIDs)) }
totalRecall += rAt5
// MRR
for i, res := range results {
if expectedSet[res.ID] {
totalMRR += 1.0 / float64(i+1)
break
}
}
// NDCG
dcg, idcg := 0.0, 0.0
for i, res := range results {
rel := 0.0
if expectedSet[res.ID] { rel = 1.0 }
dcg += rel / log2(float64(i+2))
if i < len(q.ExpectedIDs) { idcg += 1.0 / log2(float64(i+2)) }
}
if idcg > 0 { totalNDCG += dcg / idcg }
// 分组统计
// (category 从 query 推断)
cat := inferCategory(q.Query, "system_fact")
recallByTagMap[cat] = append(recallByTagMap[cat], rAt5)
precisionByTagMap[cat] = append(precisionByTagMap[cat], pAt5)
// Agent 分组 (从请求头获取)
agent := "hermes"
recallByAgentMap[agent] = append(recallByAgentMap[agent], rAt5)
queryDetails = append(queryDetails, EvalQueryDetail{
Query: q.Query,
ExpectedIDs: q.ExpectedIDs,
Hits: hitIDs,
Misses: missIDs,
RecallAt5: rAt5,
PrecisionAt5: pAt5,
})
totalQueries++
}
if totalQueries == 0 {
respondError(w, 500, "all queries failed")
return
}
// 平均分组统计
recallByTag := avgByGroup(recallByTagMap)
precisionByTag := avgByGroup(precisionByTagMap)
recallByAgent := avgByGroup(recallByAgentMap)
run := &EvalRun{
ID: time.Now().Format("20060102-150405"),
Model: req.Model,
RecallAt5: totalRecall / float64(totalQueries),
PrecisionAt5: totalPrecision / float64(totalQueries),
MRR: totalMRR / float64(totalQueries),
NDCG: totalNDCG / float64(totalQueries),
RecallByTag: recallByTag,
PrecisionByTag: precisionByTag,
RecallByAgent: recallByAgent,
QueryDetails: queryDetails,
Queries: totalQueries,
RanAt: time.Now(),
}
evals.mu.Lock()
evals.runs = append(evals.runs, run)
evals.mu.Unlock()
respond(w, 200, run)
}
// GET /api/v1/eval/history
func (ea *EvalAPI) History(w http.ResponseWriter, r *http.Request) {
evals.mu.RLock()
defer evals.mu.RUnlock()
if len(evals.runs) == 0 {
respond(w, 200, map[string]interface{}{"runs": []EvalRun{}, "count": 0})
return
}
// 计算趋势(最近 2 次对比)
trend := "stable"
if len(evals.runs) >= 2 {
curr := evals.runs[len(evals.runs)-1].MRR
prev := evals.runs[len(evals.runs)-2].MRR
if curr > prev*1.05 {
trend = "improving"
} else if curr < prev*0.95 {
trend = "degrading"
}
}
respond(w, 200, map[string]interface{}{
"runs": evals.runs, "count": len(evals.runs), "trend": trend,
})
}
// POST /api/v1/eval/generate — 生成 4类×3级=12条金标查询
// 对每个 query 调用 recall 获取 expected_ids
func (ea *EvalAPI) Generate(w http.ResponseWriter, r *http.Request) {
var req struct {
Namespace string `json:"namespace"`
Count int `json:"count"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
respondError(w, 400, "invalid body")
return
}
if req.Namespace == "" {
req.Namespace = "hermes"
}
if req.Count <= 0 {
req.Count = 12
}
// 固定的 12 条金标查询(4类×3级)
// 对每条生成 query → recall 获得真实 expected_ids
goldenQueries := []struct {
cat string
difficulty string
query string
}{
// system_fact
{"system_fact", "easy", "织忆是什么"},
{"system_fact", "medium", "织忆的图谱扩展机制是什么"},
{"system_fact", "hard", "织忆如何通过 E1 提取和 MMR 实现多样去重"},
// user_pref
{"user_pref", "easy", "牧尘的偏好是什么"},
{"user_pref", "medium", "牧尘喜欢什么样的工作方式"},
{"user_pref", "hard", "如何根据牧尘的偏好调整记忆检索策略"},
// proj_context
{"proj_context", "easy", "当前项目的技术栈是什么"},
{"proj_context", "medium", "织忆项目有哪些核心组件"},
{"proj_context", "hard", "织忆和其他记忆系统相比有什么架构优势"},
// tool_usage
{"tool_usage", "easy", "如何使用 recall 接口"},
{"tool_usage", "medium", "织忆有哪些管理工具"},
{"tool_usage", "hard", "如何通过 API 扩展织忆功能"},
}
var results []map[string]interface{}
for _, gq := range goldenQueries {
// recall 获取 top-3 结果作为 expected_ids
recallResults, _ := ea.Pipeline.Recall(gq.query, req.Namespace, 3, 0.5)
var expectedIDs []string
for _, r := range recallResults {
if r.ID != "" {
expectedIDs = append(expectedIDs, r.ID)
}
}
results = append(results, map[string]interface{}{
"query": gq.query,
"expected_ids": expectedIDs,
"category": gq.cat,
"difficulty": gq.difficulty,
})
}
respond(w, 200, map[string]interface{}{
"queries": results,
"count": len(results),
"note": "4类别×3难度=12条金标查询,通过 recall 自动获取 expected_ids",
})
}
// 辅助
func makeSet(ids []string) map[string]bool {
s := make(map[string]bool, len(ids))
for _, id := range ids {
s[id] = true
}
return s
}
func log2(x float64) float64 {
result := 0.0
for x > 2 { x /= 2; result += 1 }
if x > 1 { result += (x - 1) }
return result
}
func contains(list []string, item string) bool {
for _, s := range list {
if s == item { return true }
}
return false
}
func avgByGroup(m map[string][]float64) map[string]float64 {
result := make(map[string]float64)
for k, vals := range m {
if len(vals) > 0 {
sum := 0.0
for _, v := range vals { sum += v }
result[k] = sum / float64(len(vals))
}
}
return result
}
func inferCategory(query, defaultCat string) string {
for cat, keywords := range map[string][]string{
"system_fact": {"os", "系统", "gpu", "内存", "ram", "端口", "配置"},
"user_pref": {"偏好", "风格", "喜欢", "牧尘"},
"proj_context":{"项目", "路径", "代码", "设计文档"},
"tool_usage": {"工具", "comfyui", "hermes", "opencode"},
} {
for _, kw := range keywords {
if len(query) >= len(kw) {
for i := 0; i <= len(query)-len(kw); i++ {
if query[i:i+len(kw)] == kw { return cat }
}
}
}
}
return defaultCat
}