320 lines
7.8 KiB
Go
320 lines
7.8 KiB
Go
// 织忆 MemoryWeave — 评估系统 API
|
|
package routes
|
|
|
|
import (
|
|
"encoding/json"
|
|
"net/http"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/xiaoxue/memoryweave/internal/storage"
|
|
)
|
|
|
|
type EvalAPI struct {
|
|
Pipeline *storage.RecallPipeline
|
|
LanceDB storage.LanceDB
|
|
}
|
|
|
|
type EvalRun struct {
|
|
ID string `json:"id"`
|
|
Model string `json:"model"`
|
|
RecallAt5 float64 `json:"recall_at_5"`
|
|
PrecisionAt5 float64 `json:"precision_at_5"`
|
|
MRR float64 `json:"mean_reciprocal_rank"`
|
|
NDCG float64 `json:"ndcg"`
|
|
|
|
// 按标签分组
|
|
RecallByTag map[string]float64 `json:"recall_at_5_by_tag,omitempty"`
|
|
PrecisionByTag map[string]float64 `json:"precision_at_5_by_tag,omitempty"`
|
|
|
|
// 按 Agent 分组
|
|
RecallByAgent map[string]float64 `json:"recall_by_agent,omitempty"`
|
|
|
|
// 详细查询结果
|
|
QueryDetails []EvalQueryDetail `json:"query_details,omitempty"`
|
|
|
|
// 深度整合影响
|
|
ConsolidationAwareHits *ConsolidationHits `json:"consolidation_aware_hits,omitempty"`
|
|
|
|
Queries int `json:"queries"`
|
|
RanAt time.Time `json:"ran_at"`
|
|
}
|
|
|
|
type EvalQueryDetail struct {
|
|
Query string `json:"query"`
|
|
ExpectedIDs []string `json:"expected_ids"`
|
|
Hits []string `json:"hits"`
|
|
Misses []string `json:"misses"`
|
|
RecallAt5 float64 `json:"recall_at_5"`
|
|
PrecisionAt5 float64 `json:"precision_at_5"`
|
|
}
|
|
|
|
type ConsolidationHits struct {
|
|
TotalDistilledUsed int `json:"total_distilled_used"`
|
|
HitsAfterConsolidation int `json:"hits_after_consolidation"`
|
|
ConsolidationBenefit float64 `json:"consolidation_benefit"`
|
|
}
|
|
|
|
type evalStore struct {
|
|
mu sync.RWMutex
|
|
runs []*EvalRun
|
|
}
|
|
|
|
var evals = &evalStore{}
|
|
|
|
func NewEvalAPI(p *storage.RecallPipeline, ldb storage.LanceDB) *EvalAPI {
|
|
return &EvalAPI{Pipeline: p, LanceDB: ldb}
|
|
}
|
|
|
|
// POST /api/v1/eval/run
|
|
func (ea *EvalAPI) Run(w http.ResponseWriter, r *http.Request) {
|
|
var req struct {
|
|
Queries []struct {
|
|
Query string `json:"query"`
|
|
ExpectedIDs []string `json:"expected_ids"`
|
|
} `json:"queries"`
|
|
Model string `json:"model"`
|
|
}
|
|
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
respondError(w, 400, "invalid body")
|
|
return
|
|
}
|
|
if len(req.Queries) == 0 {
|
|
respondError(w, 400, "at least one query required")
|
|
return
|
|
}
|
|
|
|
var totalPrecision, totalRecall, totalMRR, totalNDCG float64
|
|
totalQueries := 0
|
|
|
|
// 按标签/Agent 分组统计
|
|
recallByTagMap := make(map[string][]float64)
|
|
precisionByTagMap := make(map[string][]float64)
|
|
recallByAgentMap := make(map[string][]float64)
|
|
var queryDetails []EvalQueryDetail
|
|
|
|
for _, q := range req.Queries {
|
|
results, err := ea.Pipeline.Recall(q.Query, "shared", 5, 0.5)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
|
|
// Precision@k
|
|
hits := 0
|
|
var hitIDs []string
|
|
var missIDs []string
|
|
expectedSet := makeSet(q.ExpectedIDs)
|
|
for i, res := range results {
|
|
if expectedSet[res.ID] && i < 5 {
|
|
hits++
|
|
hitIDs = append(hitIDs, res.ID)
|
|
}
|
|
}
|
|
// 找未命中的
|
|
for _, eid := range q.ExpectedIDs {
|
|
if !contains(hitIDs, eid) {
|
|
missIDs = append(missIDs, eid)
|
|
}
|
|
}
|
|
|
|
pAt5 := 0.0
|
|
if len(results) > 0 { pAt5 = float64(hits) / float64(len(results)) }
|
|
totalPrecision += pAt5
|
|
|
|
rAt5 := 0.0
|
|
if len(q.ExpectedIDs) > 0 { rAt5 = float64(hits) / float64(len(q.ExpectedIDs)) }
|
|
totalRecall += rAt5
|
|
|
|
// MRR
|
|
for i, res := range results {
|
|
if expectedSet[res.ID] {
|
|
totalMRR += 1.0 / float64(i+1)
|
|
break
|
|
}
|
|
}
|
|
|
|
// NDCG
|
|
dcg, idcg := 0.0, 0.0
|
|
for i, res := range results {
|
|
rel := 0.0
|
|
if expectedSet[res.ID] { rel = 1.0 }
|
|
dcg += rel / log2(float64(i+2))
|
|
if i < len(q.ExpectedIDs) { idcg += 1.0 / log2(float64(i+2)) }
|
|
}
|
|
if idcg > 0 { totalNDCG += dcg / idcg }
|
|
|
|
// 分组统计
|
|
// (category 从 query 推断)
|
|
cat := inferCategory(q.Query, "system_fact")
|
|
recallByTagMap[cat] = append(recallByTagMap[cat], rAt5)
|
|
precisionByTagMap[cat] = append(precisionByTagMap[cat], pAt5)
|
|
|
|
// Agent 分组 (从请求头获取)
|
|
agent := "hermes"
|
|
recallByAgentMap[agent] = append(recallByAgentMap[agent], rAt5)
|
|
|
|
queryDetails = append(queryDetails, EvalQueryDetail{
|
|
Query: q.Query,
|
|
ExpectedIDs: q.ExpectedIDs,
|
|
Hits: hitIDs,
|
|
Misses: missIDs,
|
|
RecallAt5: rAt5,
|
|
PrecisionAt5: pAt5,
|
|
})
|
|
|
|
totalQueries++
|
|
}
|
|
|
|
if totalQueries == 0 {
|
|
respondError(w, 500, "all queries failed")
|
|
return
|
|
}
|
|
|
|
// 平均分组统计
|
|
recallByTag := avgByGroup(recallByTagMap)
|
|
precisionByTag := avgByGroup(precisionByTagMap)
|
|
recallByAgent := avgByGroup(recallByAgentMap)
|
|
|
|
run := &EvalRun{
|
|
ID: time.Now().Format("20060102-150405"),
|
|
Model: req.Model,
|
|
RecallAt5: totalRecall / float64(totalQueries),
|
|
PrecisionAt5: totalPrecision / float64(totalQueries),
|
|
MRR: totalMRR / float64(totalQueries),
|
|
NDCG: totalNDCG / float64(totalQueries),
|
|
RecallByTag: recallByTag,
|
|
PrecisionByTag: precisionByTag,
|
|
RecallByAgent: recallByAgent,
|
|
QueryDetails: queryDetails,
|
|
Queries: totalQueries,
|
|
RanAt: time.Now(),
|
|
}
|
|
|
|
evals.mu.Lock()
|
|
evals.runs = append(evals.runs, run)
|
|
evals.mu.Unlock()
|
|
|
|
respond(w, 200, run)
|
|
}
|
|
|
|
// GET /api/v1/eval/history
|
|
func (ea *EvalAPI) History(w http.ResponseWriter, r *http.Request) {
|
|
evals.mu.RLock()
|
|
defer evals.mu.RUnlock()
|
|
|
|
if len(evals.runs) == 0 {
|
|
respond(w, 200, map[string]interface{}{"runs": []EvalRun{}, "count": 0})
|
|
return
|
|
}
|
|
|
|
// 计算趋势（最近 2 次对比）
|
|
trend := "stable"
|
|
if len(evals.runs) >= 2 {
|
|
curr := evals.runs[len(evals.runs)-1].MRR
|
|
prev := evals.runs[len(evals.runs)-2].MRR
|
|
if curr > prev*1.05 {
|
|
trend = "improving"
|
|
} else if curr < prev*0.95 {
|
|
trend = "degrading"
|
|
}
|
|
}
|
|
|
|
respond(w, 200, map[string]interface{}{
|
|
"runs": evals.runs, "count": len(evals.runs), "trend": trend,
|
|
})
|
|
}
|
|
|
|
// POST /api/v1/eval/generate — 自动生成金标查询集
|
|
func (ea *EvalAPI) Generate(w http.ResponseWriter, r *http.Request) {
|
|
var req struct {
|
|
Namespace string `json:"namespace"`
|
|
Count int `json:"count"`
|
|
}
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
respondError(w, 400, "invalid body")
|
|
return
|
|
}
|
|
if req.Namespace == "" {
|
|
req.Namespace = "shared"
|
|
}
|
|
if req.Count <= 0 {
|
|
req.Count = 10
|
|
}
|
|
|
|
// 获取高质量记忆作为金标基础
|
|
memories, err := ea.LanceDB.GetTopByQuality("", req.Count)
|
|
if err != nil {
|
|
respondError(w, 500, "generate failed: "+err.Error())
|
|
return
|
|
}
|
|
|
|
var queries []map[string]interface{}
|
|
for _, mem := range memories {
|
|
query := mem.Content
|
|
if len(query) > 50 {
|
|
query = query[:50]
|
|
}
|
|
queries = append(queries, map[string]interface{}{
|
|
"query": query,
|
|
"expected_ids": []string{mem.ID},
|
|
})
|
|
}
|
|
respond(w, 200, map[string]interface{}{
|
|
"queries": queries, "count": len(queries),
|
|
})
|
|
}
|
|
|
|
// 辅助
|
|
func makeSet(ids []string) map[string]bool {
|
|
s := make(map[string]bool, len(ids))
|
|
for _, id := range ids {
|
|
s[id] = true
|
|
}
|
|
return s
|
|
}
|
|
|
|
func log2(x float64) float64 {
|
|
result := 0.0
|
|
for x > 2 { x /= 2; result += 1 }
|
|
if x > 1 { result += (x - 1) }
|
|
return result
|
|
}
|
|
|
|
func contains(list []string, item string) bool {
|
|
for _, s := range list {
|
|
if s == item { return true }
|
|
}
|
|
return false
|
|
}
|
|
|
|
func avgByGroup(m map[string][]float64) map[string]float64 {
|
|
result := make(map[string]float64)
|
|
for k, vals := range m {
|
|
if len(vals) > 0 {
|
|
sum := 0.0
|
|
for _, v := range vals { sum += v }
|
|
result[k] = sum / float64(len(vals))
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
func inferCategory(query, defaultCat string) string {
|
|
for cat, keywords := range map[string][]string{
|
|
"system_fact": {"os", "系统", "gpu", "内存", "ram", "端口", "配置"},
|
|
"user_pref": {"偏好", "风格", "喜欢", "牧尘"},
|
|
"proj_context":{"项目", "路径", "代码", "设计文档"},
|
|
"tool_usage": {"工具", "comfyui", "hermes", "opencode"},
|
|
} {
|
|
for _, kw := range keywords {
|
|
if len(query) >= len(kw) {
|
|
for i := 0; i <= len(query)-len(kw); i++ {
|
|
if query[i:i+len(kw)] == kw { return cat }
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return defaultCat
|
|
}
|