484 lines
13 KiB
Go
484 lines
13 KiB
Go
// 织忆 MemoryWeave — 评估系统 API
|
||
package routes
|
||
|
||
import (
|
||
"encoding/json"
|
||
"net/http"
|
||
"os"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
"github.com/xiaoxue/memoryweave/internal/storage"
|
||
)
|
||
|
||
// ─── 金标集定义 ──────────────────────────────────────────────
|
||
|
||
// GoldenSetItem 单条金标:query + 具体 expected_ids + 关键词相关性判据
|
||
type GoldenSetItem struct {
|
||
Query string `json:"query"`
|
||
ExpectedIDs []string `json:"expected_ids"`
|
||
MustIncludeKeywords []string `json:"must_include_keywords"`
|
||
Category string `json:"category,omitempty"`
|
||
Difficulty string `json:"difficulty,omitempty"`
|
||
IDBaseline string `json:"id_baseline,omitempty"`
|
||
}
|
||
|
||
// GoldenSet 金标集
|
||
type GoldenSet struct {
|
||
Description string `json:"description"`
|
||
Items []GoldenSetItem `json:"items"`
|
||
}
|
||
|
||
// LoadGoldenSet 从 testdata/ 加载金标集,支持相对路径(生产用)和绝对路径(测试用)
|
||
func LoadGoldenSet(path string) (*GoldenSet, error) {
|
||
if path == "" {
|
||
// 编译后无法直接拿到源码路径,回退到 CWD 的 testdata/
|
||
path = "testdata/golden-set.json"
|
||
}
|
||
data, err := os.ReadFile(path)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
var gs GoldenSet
|
||
if err := json.Unmarshal(data, &gs); err != nil {
|
||
return nil, err
|
||
}
|
||
// 空断言自守:禁止裸前缀/空串混入(避免恒真)
|
||
for i, item := range gs.Items {
|
||
if err := validateExpectedIDs(item.Query, item.ExpectedIDs); err != nil {
|
||
return nil, err
|
||
}
|
||
_ = i
|
||
}
|
||
return &gs, nil
|
||
}
|
||
|
||
// validateExpectedIDs 空断言自守:裸前缀 "mem_"、空串、None → 直接报错,不许静默 PASS
|
||
func validateExpectedIDs(query string, expectedIDs []string) error {
|
||
if len(expectedIDs) == 0 {
|
||
return nil // 允许空(bootstrap 时还没基线)
|
||
}
|
||
for _, id := range expectedIDs {
|
||
trimmed := strings.TrimSpace(id)
|
||
// 裸前缀 "mem_"(不含具体 ID 数字)
|
||
if trimmed == "mem_" || trimmed == "mem" {
|
||
return &VacuousAssertionError{
|
||
Query: query,
|
||
BadID: id,
|
||
Reason: "裸前缀 mem_ 是恒真断言——所有记忆 ID 都以 mem_ 开头",
|
||
}
|
||
}
|
||
// 空串
|
||
if trimmed == "" {
|
||
return &VacuousAssertionError{
|
||
Query: query,
|
||
BadID: id,
|
||
Reason: "空串期望值(无意义断言)",
|
||
}
|
||
}
|
||
// None(JSON null 解码成空字符串已处理,但检查 "None" 字符串)
|
||
if strings.EqualFold(trimmed, "none") {
|
||
return &VacuousAssertionError{
|
||
Query: query,
|
||
BadID: id,
|
||
Reason: "None 期望值(无意义断言)",
|
||
}
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// VacuousAssertionError 空断言错误
|
||
type VacuousAssertionError struct {
|
||
Query string
|
||
BadID string
|
||
Reason string
|
||
}
|
||
|
||
func (e *VacuousAssertionError) Error() string {
|
||
return "FATAL 空断言被检出(恒真): query=" + e.Query + " 期望=" + e.BadID + " — " + e.Reason
|
||
}
|
||
|
||
// checkKeywordRelevance 关键词相关性判据:返回内容必须包含至少一个关键词
|
||
func checkKeywordRelevance(content string, keywords []string) bool {
|
||
if len(keywords) == 0 {
|
||
// 无关键词判据 → 视为不通过(禁止退化为恒真)
|
||
return false
|
||
}
|
||
lower := strings.ToLower(content)
|
||
for _, kw := range keywords {
|
||
if strings.Contains(lower, strings.ToLower(kw)) {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// validateGoldenSetForRun 对一次 eval run 的所有 expected_ids 做空断言自守
|
||
func validateGoldenSetForRun(queries []struct {
|
||
Query string `json:"query"`
|
||
ExpectedIDs []string `json:"expected_ids"`
|
||
}) error {
|
||
for _, q := range queries {
|
||
if err := validateExpectedIDs(q.Query, q.ExpectedIDs); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
type EvalAPI struct {
|
||
Pipeline *storage.RecallPipeline
|
||
LanceDB storage.LanceDB
|
||
}
|
||
|
||
type EvalRun struct {
|
||
ID string `json:"id"`
|
||
Model string `json:"model"`
|
||
RecallAt5 float64 `json:"recall_at_5"`
|
||
PrecisionAt5 float64 `json:"precision_at_5"`
|
||
MRR float64 `json:"mean_reciprocal_rank"`
|
||
NDCG float64 `json:"ndcg"`
|
||
|
||
// 按标签分组
|
||
RecallByTag map[string]float64 `json:"recall_at_5_by_tag,omitempty"`
|
||
PrecisionByTag map[string]float64 `json:"precision_at_5_by_tag,omitempty"`
|
||
|
||
// 按 Agent 分组
|
||
RecallByAgent map[string]float64 `json:"recall_by_agent,omitempty"`
|
||
|
||
// 详细查询结果
|
||
QueryDetails []EvalQueryDetail `json:"query_details,omitempty"`
|
||
|
||
// 深度整合影响
|
||
ConsolidationAwareHits *ConsolidationHits `json:"consolidation_aware_hits,omitempty"`
|
||
|
||
Queries int `json:"queries"`
|
||
RanAt time.Time `json:"ran_at"`
|
||
}
|
||
|
||
// EvalQueryDetail
|
||
type EvalQueryDetail struct {
|
||
Query string `json:"query"`
|
||
ExpectedIDs []string `json:"expected_ids"`
|
||
MustIncludeKeywords []string `json:"must_include_keywords,omitempty"`
|
||
Hits []string `json:"hits"`
|
||
Misses []string `json:"misses"`
|
||
RecallAt5 float64 `json:"recall_at_5"`
|
||
PrecisionAt5 float64 `json:"precision_at_5"`
|
||
KeywordOK bool `json:"keyword_ok,omitempty"`
|
||
}
|
||
|
||
// GoldenRunResult 金标 run 的扩展结果(含关键词判据)
|
||
type GoldenRunResult struct {
|
||
EvalRun
|
||
KeywordPass int `json:"keyword_pass"`
|
||
KeywordFail int `json:"keyword_fail"`
|
||
KeywordTotal int `json:"keyword_total"`
|
||
}
|
||
|
||
type ConsolidationHits struct {
|
||
TotalDistilledUsed int `json:"total_distilled_used"`
|
||
HitsAfterConsolidation int `json:"hits_after_consolidation"`
|
||
ConsolidationBenefit float64 `json:"consolidation_benefit"`
|
||
}
|
||
|
||
type evalStore struct {
|
||
mu sync.RWMutex
|
||
runs []*EvalRun
|
||
}
|
||
|
||
var evals = &evalStore{}
|
||
|
||
func NewEvalAPI(p *storage.RecallPipeline, ldb storage.LanceDB) *EvalAPI {
|
||
return &EvalAPI{Pipeline: p, LanceDB: ldb}
|
||
}
|
||
|
||
// POST /api/v1/eval/run
|
||
func (ea *EvalAPI) Run(w http.ResponseWriter, r *http.Request) {
|
||
var req struct {
|
||
Queries []struct {
|
||
Query string `json:"query"`
|
||
ExpectedIDs []string `json:"expected_ids"`
|
||
} `json:"queries"`
|
||
Model string `json:"model"`
|
||
Namespace string `json:"namespace"` // 支持指定 namespace
|
||
}
|
||
|
||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||
respondError(w, 400, "invalid body")
|
||
return
|
||
}
|
||
if len(req.Queries) == 0 {
|
||
respondError(w, 400, "at least one query required")
|
||
return
|
||
}
|
||
|
||
// 默认 namespace 为 shared,支持传入 hermes 等
|
||
ns := req.Namespace
|
||
if ns == "" {
|
||
ns = "shared"
|
||
}
|
||
|
||
var totalPrecision, totalRecall, totalMRR, totalNDCG float64
|
||
totalQueries := 0
|
||
|
||
// 按标签/Agent 分组统计
|
||
recallByTagMap := make(map[string][]float64)
|
||
precisionByTagMap := make(map[string][]float64)
|
||
recallByAgentMap := make(map[string][]float64)
|
||
var queryDetails []EvalQueryDetail
|
||
|
||
for _, q := range req.Queries {
|
||
results, err := ea.Pipeline.Recall(q.Query, ns, 5, 0.5)
|
||
if err != nil {
|
||
continue
|
||
}
|
||
|
||
// Precision@k
|
||
hits := 0
|
||
var hitIDs []string
|
||
var missIDs []string
|
||
expectedSet := makeSet(q.ExpectedIDs)
|
||
for i, res := range results {
|
||
if expectedSet[res.ID] && i < 5 {
|
||
hits++
|
||
hitIDs = append(hitIDs, res.ID)
|
||
}
|
||
}
|
||
// 找未命中的
|
||
for _, eid := range q.ExpectedIDs {
|
||
if !contains(hitIDs, eid) {
|
||
missIDs = append(missIDs, eid)
|
||
}
|
||
}
|
||
|
||
pAt5 := 0.0
|
||
if len(results) > 0 { pAt5 = float64(hits) / float64(len(results)) }
|
||
totalPrecision += pAt5
|
||
|
||
rAt5 := 0.0
|
||
if len(q.ExpectedIDs) > 0 { rAt5 = float64(hits) / float64(len(q.ExpectedIDs)) }
|
||
totalRecall += rAt5
|
||
|
||
// MRR
|
||
for i, res := range results {
|
||
if expectedSet[res.ID] {
|
||
totalMRR += 1.0 / float64(i+1)
|
||
break
|
||
}
|
||
}
|
||
|
||
// NDCG
|
||
dcg, idcg := 0.0, 0.0
|
||
for i, res := range results {
|
||
rel := 0.0
|
||
if expectedSet[res.ID] { rel = 1.0 }
|
||
dcg += rel / log2(float64(i+2))
|
||
if i < len(q.ExpectedIDs) { idcg += 1.0 / log2(float64(i+2)) }
|
||
}
|
||
if idcg > 0 { totalNDCG += dcg / idcg }
|
||
|
||
// 分组统计
|
||
// (category 从 query 推断)
|
||
cat := inferCategory(q.Query, "system_fact")
|
||
recallByTagMap[cat] = append(recallByTagMap[cat], rAt5)
|
||
precisionByTagMap[cat] = append(precisionByTagMap[cat], pAt5)
|
||
|
||
// Agent 分组 (从请求头获取)
|
||
agent := "hermes"
|
||
recallByAgentMap[agent] = append(recallByAgentMap[agent], rAt5)
|
||
|
||
queryDetails = append(queryDetails, EvalQueryDetail{
|
||
Query: q.Query,
|
||
ExpectedIDs: q.ExpectedIDs,
|
||
Hits: hitIDs,
|
||
Misses: missIDs,
|
||
RecallAt5: rAt5,
|
||
PrecisionAt5: pAt5,
|
||
})
|
||
|
||
totalQueries++
|
||
}
|
||
|
||
if totalQueries == 0 {
|
||
respondError(w, 500, "all queries failed")
|
||
return
|
||
}
|
||
|
||
// 平均分组统计
|
||
recallByTag := avgByGroup(recallByTagMap)
|
||
precisionByTag := avgByGroup(precisionByTagMap)
|
||
recallByAgent := avgByGroup(recallByAgentMap)
|
||
|
||
run := &EvalRun{
|
||
ID: time.Now().Format("20060102-150405"),
|
||
Model: req.Model,
|
||
RecallAt5: totalRecall / float64(totalQueries),
|
||
PrecisionAt5: totalPrecision / float64(totalQueries),
|
||
MRR: totalMRR / float64(totalQueries),
|
||
NDCG: totalNDCG / float64(totalQueries),
|
||
RecallByTag: recallByTag,
|
||
PrecisionByTag: precisionByTag,
|
||
RecallByAgent: recallByAgent,
|
||
QueryDetails: queryDetails,
|
||
Queries: totalQueries,
|
||
RanAt: time.Now(),
|
||
}
|
||
|
||
evals.mu.Lock()
|
||
evals.runs = append(evals.runs, run)
|
||
evals.mu.Unlock()
|
||
|
||
respond(w, 200, run)
|
||
}
|
||
|
||
// GET /api/v1/eval/history
|
||
func (ea *EvalAPI) History(w http.ResponseWriter, r *http.Request) {
|
||
evals.mu.RLock()
|
||
defer evals.mu.RUnlock()
|
||
|
||
if len(evals.runs) == 0 {
|
||
respond(w, 200, map[string]interface{}{"runs": []EvalRun{}, "count": 0})
|
||
return
|
||
}
|
||
|
||
// 计算趋势(最近 2 次对比)
|
||
trend := "stable"
|
||
if len(evals.runs) >= 2 {
|
||
curr := evals.runs[len(evals.runs)-1].MRR
|
||
prev := evals.runs[len(evals.runs)-2].MRR
|
||
if curr > prev*1.05 {
|
||
trend = "improving"
|
||
} else if curr < prev*0.95 {
|
||
trend = "degrading"
|
||
}
|
||
}
|
||
|
||
respond(w, 200, map[string]interface{}{
|
||
"runs": evals.runs, "count": len(evals.runs), "trend": trend,
|
||
})
|
||
}
|
||
|
||
// POST /api/v1/eval/generate — 生成 4类×3级=12条金标查询
|
||
// 对每个 query 调用 recall 获取 expected_ids
|
||
func (ea *EvalAPI) Generate(w http.ResponseWriter, r *http.Request) {
|
||
var req struct {
|
||
Namespace string `json:"namespace"`
|
||
Count int `json:"count"`
|
||
}
|
||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||
respondError(w, 400, "invalid body")
|
||
return
|
||
}
|
||
if req.Namespace == "" {
|
||
req.Namespace = "hermes"
|
||
}
|
||
if req.Count <= 0 {
|
||
req.Count = 12
|
||
}
|
||
|
||
// 固定的 12 条金标查询(4类×3级)
|
||
// 对每条生成 query → recall 获得真实 expected_ids
|
||
goldenQueries := []struct {
|
||
cat string
|
||
difficulty string
|
||
query string
|
||
}{
|
||
// system_fact
|
||
{"system_fact", "easy", "织忆是什么"},
|
||
{"system_fact", "medium", "织忆的图谱扩展机制是什么"},
|
||
{"system_fact", "hard", "织忆如何通过 E1 提取和 MMR 实现多样去重"},
|
||
// user_pref
|
||
{"user_pref", "easy", "牧尘的偏好是什么"},
|
||
{"user_pref", "medium", "牧尘喜欢什么样的工作方式"},
|
||
{"user_pref", "hard", "如何根据牧尘的偏好调整记忆检索策略"},
|
||
// proj_context
|
||
{"proj_context", "easy", "当前项目的技术栈是什么"},
|
||
{"proj_context", "medium", "织忆项目有哪些核心组件"},
|
||
{"proj_context", "hard", "织忆和其他记忆系统相比有什么架构优势"},
|
||
// tool_usage
|
||
{"tool_usage", "easy", "如何使用 recall 接口"},
|
||
{"tool_usage", "medium", "织忆有哪些管理工具"},
|
||
{"tool_usage", "hard", "如何通过 API 扩展织忆功能"},
|
||
}
|
||
|
||
var results []map[string]interface{}
|
||
for _, gq := range goldenQueries {
|
||
// recall 获取 top-3 结果作为 expected_ids
|
||
recallResults, _ := ea.Pipeline.Recall(gq.query, req.Namespace, 3, 0.5)
|
||
var expectedIDs []string
|
||
for _, r := range recallResults {
|
||
if r.ID != "" {
|
||
expectedIDs = append(expectedIDs, r.ID)
|
||
}
|
||
}
|
||
results = append(results, map[string]interface{}{
|
||
"query": gq.query,
|
||
"expected_ids": expectedIDs,
|
||
"category": gq.cat,
|
||
"difficulty": gq.difficulty,
|
||
})
|
||
}
|
||
|
||
respond(w, 200, map[string]interface{}{
|
||
"queries": results,
|
||
"count": len(results),
|
||
"note": "4类别×3难度=12条金标查询,通过 recall 自动获取 expected_ids",
|
||
})
|
||
}
|
||
|
||
// 辅助
|
||
func makeSet(ids []string) map[string]bool {
|
||
s := make(map[string]bool, len(ids))
|
||
for _, id := range ids {
|
||
s[id] = true
|
||
}
|
||
return s
|
||
}
|
||
|
||
func log2(x float64) float64 {
|
||
result := 0.0
|
||
for x > 2 { x /= 2; result += 1 }
|
||
if x > 1 { result += (x - 1) }
|
||
return result
|
||
}
|
||
|
||
func contains(list []string, item string) bool {
|
||
for _, s := range list {
|
||
if s == item { return true }
|
||
}
|
||
return false
|
||
}
|
||
|
||
func avgByGroup(m map[string][]float64) map[string]float64 {
|
||
result := make(map[string]float64)
|
||
for k, vals := range m {
|
||
if len(vals) > 0 {
|
||
sum := 0.0
|
||
for _, v := range vals { sum += v }
|
||
result[k] = sum / float64(len(vals))
|
||
}
|
||
}
|
||
return result
|
||
}
|
||
|
||
func inferCategory(query, defaultCat string) string {
|
||
for cat, keywords := range map[string][]string{
|
||
"system_fact": {"os", "系统", "gpu", "内存", "ram", "端口", "配置"},
|
||
"user_pref": {"偏好", "风格", "喜欢", "牧尘"},
|
||
"proj_context":{"项目", "路径", "代码", "设计文档"},
|
||
"tool_usage": {"工具", "comfyui", "hermes", "opencode"},
|
||
} {
|
||
for _, kw := range keywords {
|
||
if len(query) >= len(kw) {
|
||
for i := 0; i <= len(query)-len(kw); i++ {
|
||
if query[i:i+len(kw)] == kw { return cat }
|
||
}
|
||
}
|
||
}
|
||
}
|
||
return defaultCat
|
||
}
|