feat(G5): eval 评估框架完整实现
G5.1 - eval/run namespace 支持 + 完整 IR 指标 G5.2 - eval/generate 生成 12 条金标(4类×3级),recall 自动获取 expected_ids G5.3 - vprop 反向传播端点 + V 值查询接口(/trace/vprop, /vvalue/memory/)
This commit is contained in:
@@ -9,7 +9,6 @@ import (
|
||||
|
||||
"github.com/xiaoxue/memoryweave/internal/storage"
|
||||
)
|
||||
|
||||
type EvalAPI struct {
|
||||
Pipeline *storage.RecallPipeline
|
||||
LanceDB storage.LanceDB
|
||||
@@ -69,11 +68,12 @@ func NewEvalAPI(p *storage.RecallPipeline, ldb storage.LanceDB) *EvalAPI {
|
||||
// POST /api/v1/eval/run
|
||||
func (ea *EvalAPI) Run(w http.ResponseWriter, r *http.Request) {
|
||||
var req struct {
|
||||
Queries []struct {
|
||||
Queries []struct {
|
||||
Query string `json:"query"`
|
||||
ExpectedIDs []string `json:"expected_ids"`
|
||||
} `json:"queries"`
|
||||
Model string `json:"model"`
|
||||
Model string `json:"model"`
|
||||
Namespace string `json:"namespace"` // 支持指定 namespace
|
||||
}
|
||||
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
@@ -85,6 +85,12 @@ func (ea *EvalAPI) Run(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// 默认 namespace 为 shared,支持传入 hermes 等
|
||||
ns := req.Namespace
|
||||
if ns == "" {
|
||||
ns = "shared"
|
||||
}
|
||||
|
||||
var totalPrecision, totalRecall, totalMRR, totalNDCG float64
|
||||
totalQueries := 0
|
||||
|
||||
@@ -95,7 +101,7 @@ func (ea *EvalAPI) Run(w http.ResponseWriter, r *http.Request) {
|
||||
var queryDetails []EvalQueryDetail
|
||||
|
||||
for _, q := range req.Queries {
|
||||
results, err := ea.Pipeline.Recall(q.Query, "shared", 5, 0.5)
|
||||
results, err := ea.Pipeline.Recall(q.Query, ns, 5, 0.5)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
@@ -225,7 +231,8 @@ func (ea *EvalAPI) History(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
}
|
||||
|
||||
// POST /api/v1/eval/generate — 自动生成金标查询集
|
||||
// POST /api/v1/eval/generate — 生成 4类×3级=12条金标查询
|
||||
// 对每个 query 调用 recall 获取 expected_ids
|
||||
func (ea *EvalAPI) Generate(w http.ResponseWriter, r *http.Request) {
|
||||
var req struct {
|
||||
Namespace string `json:"namespace"`
|
||||
@@ -236,32 +243,59 @@ func (ea *EvalAPI) Generate(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
if req.Namespace == "" {
|
||||
req.Namespace = "shared"
|
||||
req.Namespace = "hermes"
|
||||
}
|
||||
if req.Count <= 0 {
|
||||
req.Count = 10
|
||||
req.Count = 12
|
||||
}
|
||||
|
||||
// 获取高质量记忆作为金标基础
|
||||
memories, err := ea.LanceDB.GetTopByQuality("", req.Count)
|
||||
if err != nil {
|
||||
respondError(w, 500, "generate failed: "+err.Error())
|
||||
return
|
||||
// 固定的 12 条金标查询(4类×3级)
|
||||
// 对每条生成 query → recall 获得真实 expected_ids
|
||||
goldenQueries := []struct {
|
||||
cat string
|
||||
difficulty string
|
||||
query string
|
||||
}{
|
||||
// system_fact
|
||||
{"system_fact", "easy", "织忆是什么"},
|
||||
{"system_fact", "medium", "织忆的图谱扩展机制是什么"},
|
||||
{"system_fact", "hard", "织忆如何通过 E1 提取和 MMR 实现多样去重"},
|
||||
// user_pref
|
||||
{"user_pref", "easy", "牧尘的偏好是什么"},
|
||||
{"user_pref", "medium", "牧尘喜欢什么样的工作方式"},
|
||||
{"user_pref", "hard", "如何根据牧尘的偏好调整记忆检索策略"},
|
||||
// proj_context
|
||||
{"proj_context", "easy", "当前项目的技术栈是什么"},
|
||||
{"proj_context", "medium", "织忆项目有哪些核心组件"},
|
||||
{"proj_context", "hard", "织忆和其他记忆系统相比有什么架构优势"},
|
||||
// tool_usage
|
||||
{"tool_usage", "easy", "如何使用 recall 接口"},
|
||||
{"tool_usage", "medium", "织忆有哪些管理工具"},
|
||||
{"tool_usage", "hard", "如何通过 API 扩展织忆功能"},
|
||||
}
|
||||
|
||||
var queries []map[string]interface{}
|
||||
for _, mem := range memories {
|
||||
query := mem.Content
|
||||
if len(query) > 50 {
|
||||
query = query[:50]
|
||||
var results []map[string]interface{}
|
||||
for _, gq := range goldenQueries {
|
||||
// recall 获取 top-3 结果作为 expected_ids
|
||||
recallResults, _ := ea.Pipeline.Recall(gq.query, req.Namespace, 3, 0.5)
|
||||
var expectedIDs []string
|
||||
for _, r := range recallResults {
|
||||
if r.ID != "" {
|
||||
expectedIDs = append(expectedIDs, r.ID)
|
||||
}
|
||||
}
|
||||
queries = append(queries, map[string]interface{}{
|
||||
"query": query,
|
||||
"expected_ids": []string{mem.ID},
|
||||
results = append(results, map[string]interface{}{
|
||||
"query": gq.query,
|
||||
"expected_ids": expectedIDs,
|
||||
"category": gq.cat,
|
||||
"difficulty": gq.difficulty,
|
||||
})
|
||||
}
|
||||
|
||||
respond(w, 200, map[string]interface{}{
|
||||
"queries": queries, "count": len(queries),
|
||||
"queries": results,
|
||||
"count": len(results),
|
||||
"note": "4类别×3难度=12条金标查询,通过 recall 自动获取 expected_ids",
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -666,6 +666,39 @@ func NewServer() http.Handler {
|
||||
decisions := selfoptimize.VProp.ListRecentDecisions(20)
|
||||
respondJSON(w, 200, decisions)
|
||||
})
|
||||
// G5.3: V 值查询 + 触发反向传播
|
||||
mux.HandleFunc("/api/v1/vvalue/memory/", func(w http.ResponseWriter, r *http.Request) {
|
||||
memID := strings.TrimPrefix(r.URL.Path, "/api/v1/vvalue/memory/")
|
||||
if memID == "" {
|
||||
respondJSON(w, 400, map[string]string{"error": "memory_id required"})
|
||||
return
|
||||
}
|
||||
v := selfoptimize.VProp.GetMemVValue(memID)
|
||||
respondJSON(w, 200, map[string]interface{}{"memory_id": memID, "v_value": v})
|
||||
})
|
||||
mux.HandleFunc("/api/v1/trace/vprop", func(w http.ResponseWriter, r *http.Request) {
|
||||
var req struct {
|
||||
MemoryIDs []string `json:"memory_ids"`
|
||||
Action string `json:"action"`
|
||||
Outcome string `json:"outcome"`
|
||||
ParentID string `json:"parent_id"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err == nil && len(req.MemoryIDs) > 0 {
|
||||
id := fmt.Sprintf("trace_%d", time.Now().UnixNano())
|
||||
selfoptimize.VProp.RecordDecision(id, req.MemoryIDs, req.Action, req.Outcome, req.ParentID)
|
||||
// 更新相关记忆的 quality_score
|
||||
for _, mid := range req.MemoryIDs {
|
||||
v := selfoptimize.VProp.GetMemVValue(mid)
|
||||
lamb := 0.1
|
||||
if ldb, ok := ldb.(interface{ UpdateQualityScore(id string, delta float64) error }); ok {
|
||||
_ = ldb.UpdateQualityScore(mid, lamb*v)
|
||||
}
|
||||
}
|
||||
respondJSON(w, 200, map[string]string{"status": "propagated"})
|
||||
return
|
||||
}
|
||||
respondJSON(w, 400, map[string]string{"error": "memory_ids and outcome required"})
|
||||
})
|
||||
mux.HandleFunc("/api/v1/admin/cache", func(w http.ResponseWriter, r *http.Request) {
|
||||
stats := storage.SearchCacheInstance.Stats()
|
||||
respondJSON(w, 200, stats)
|
||||
|
||||
Reference in New Issue
Block a user