feat(G5): eval 评估框架完整实现

G5.1 - eval/run namespace 支持 + 完整 IR 指标
G5.2 - eval/generate 生成 12 条金标(4类×3级),recall 自动获取 expected_ids
G5.3 - vprop 反向传播端点 + V 值查询接口(/trace/vprop, /vvalue/memory/)
This commit is contained in:
xiaowei
2026-05-30 22:26:20 +08:00
parent a629afd4cb
commit d4c13e7377
2 changed files with 88 additions and 21 deletions
+55 -21
View File
@@ -9,7 +9,6 @@ import (
"github.com/xiaoxue/memoryweave/internal/storage"
)
type EvalAPI struct {
Pipeline *storage.RecallPipeline
LanceDB storage.LanceDB
@@ -69,11 +68,12 @@ func NewEvalAPI(p *storage.RecallPipeline, ldb storage.LanceDB) *EvalAPI {
// POST /api/v1/eval/run
func (ea *EvalAPI) Run(w http.ResponseWriter, r *http.Request) {
var req struct {
Queries []struct {
Queries []struct {
Query string `json:"query"`
ExpectedIDs []string `json:"expected_ids"`
} `json:"queries"`
Model string `json:"model"`
Model string `json:"model"`
Namespace string `json:"namespace"` // 支持指定 namespace
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
@@ -85,6 +85,12 @@ func (ea *EvalAPI) Run(w http.ResponseWriter, r *http.Request) {
return
}
// 默认 namespace 为 shared,支持传入 hermes 等
ns := req.Namespace
if ns == "" {
ns = "shared"
}
var totalPrecision, totalRecall, totalMRR, totalNDCG float64
totalQueries := 0
@@ -95,7 +101,7 @@ func (ea *EvalAPI) Run(w http.ResponseWriter, r *http.Request) {
var queryDetails []EvalQueryDetail
for _, q := range req.Queries {
results, err := ea.Pipeline.Recall(q.Query, "shared", 5, 0.5)
results, err := ea.Pipeline.Recall(q.Query, ns, 5, 0.5)
if err != nil {
continue
}
@@ -225,7 +231,8 @@ func (ea *EvalAPI) History(w http.ResponseWriter, r *http.Request) {
})
}
// POST /api/v1/eval/generate — 自动生成金标查询集
// POST /api/v1/eval/generate — 生成 4类×3级=12条金标查询
// 对每个 query 调用 recall 获取 expected_ids
func (ea *EvalAPI) Generate(w http.ResponseWriter, r *http.Request) {
var req struct {
Namespace string `json:"namespace"`
@@ -236,32 +243,59 @@ func (ea *EvalAPI) Generate(w http.ResponseWriter, r *http.Request) {
return
}
if req.Namespace == "" {
req.Namespace = "shared"
req.Namespace = "hermes"
}
if req.Count <= 0 {
req.Count = 10
req.Count = 12
}
// 获取高质量记忆作为金标基础
memories, err := ea.LanceDB.GetTopByQuality("", req.Count)
if err != nil {
respondError(w, 500, "generate failed: "+err.Error())
return
// 固定的 12 条金标查询(4类×3级)
// 对每条生成 query → recall 获得真实 expected_ids
goldenQueries := []struct {
cat string
difficulty string
query string
}{
// system_fact
{"system_fact", "easy", "织忆是什么"},
{"system_fact", "medium", "织忆的图谱扩展机制是什么"},
{"system_fact", "hard", "织忆如何通过 E1 提取和 MMR 实现多样去重"},
// user_pref
{"user_pref", "easy", "牧尘的偏好是什么"},
{"user_pref", "medium", "牧尘喜欢什么样的工作方式"},
{"user_pref", "hard", "如何根据牧尘的偏好调整记忆检索策略"},
// proj_context
{"proj_context", "easy", "当前项目的技术栈是什么"},
{"proj_context", "medium", "织忆项目有哪些核心组件"},
{"proj_context", "hard", "织忆和其他记忆系统相比有什么架构优势"},
// tool_usage
{"tool_usage", "easy", "如何使用 recall 接口"},
{"tool_usage", "medium", "织忆有哪些管理工具"},
{"tool_usage", "hard", "如何通过 API 扩展织忆功能"},
}
var queries []map[string]interface{}
for _, mem := range memories {
query := mem.Content
if len(query) > 50 {
query = query[:50]
var results []map[string]interface{}
for _, gq := range goldenQueries {
// recall 获取 top-3 结果作为 expected_ids
recallResults, _ := ea.Pipeline.Recall(gq.query, req.Namespace, 3, 0.5)
var expectedIDs []string
for _, r := range recallResults {
if r.ID != "" {
expectedIDs = append(expectedIDs, r.ID)
}
}
queries = append(queries, map[string]interface{}{
"query": query,
"expected_ids": []string{mem.ID},
results = append(results, map[string]interface{}{
"query": gq.query,
"expected_ids": expectedIDs,
"category": gq.cat,
"difficulty": gq.difficulty,
})
}
respond(w, 200, map[string]interface{}{
"queries": queries, "count": len(queries),
"queries": results,
"count": len(results),
"note": "4类别×3难度=12条金标查询,通过 recall 自动获取 expected_ids",
})
}
+33
View File
@@ -666,6 +666,39 @@ func NewServer() http.Handler {
decisions := selfoptimize.VProp.ListRecentDecisions(20)
respondJSON(w, 200, decisions)
})
// G5.3: V 值查询 + 触发反向传播
mux.HandleFunc("/api/v1/vvalue/memory/", func(w http.ResponseWriter, r *http.Request) {
memID := strings.TrimPrefix(r.URL.Path, "/api/v1/vvalue/memory/")
if memID == "" {
respondJSON(w, 400, map[string]string{"error": "memory_id required"})
return
}
v := selfoptimize.VProp.GetMemVValue(memID)
respondJSON(w, 200, map[string]interface{}{"memory_id": memID, "v_value": v})
})
mux.HandleFunc("/api/v1/trace/vprop", func(w http.ResponseWriter, r *http.Request) {
var req struct {
MemoryIDs []string `json:"memory_ids"`
Action string `json:"action"`
Outcome string `json:"outcome"`
ParentID string `json:"parent_id"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err == nil && len(req.MemoryIDs) > 0 {
id := fmt.Sprintf("trace_%d", time.Now().UnixNano())
selfoptimize.VProp.RecordDecision(id, req.MemoryIDs, req.Action, req.Outcome, req.ParentID)
// 更新相关记忆的 quality_score
for _, mid := range req.MemoryIDs {
v := selfoptimize.VProp.GetMemVValue(mid)
lamb := 0.1
if ldb, ok := ldb.(interface{ UpdateQualityScore(id string, delta float64) error }); ok {
_ = ldb.UpdateQualityScore(mid, lamb*v)
}
}
respondJSON(w, 200, map[string]string{"status": "propagated"})
return
}
respondJSON(w, 400, map[string]string{"error": "memory_ids and outcome required"})
})
mux.HandleFunc("/api/v1/admin/cache", func(w http.ResponseWriter, r *http.Request) {
stats := storage.SearchCacheInstance.Stats()
respondJSON(w, 200, stats)