diff --git a/go/go.mod b/go/go.mod index d5a66f4..838446a 100644 --- a/go/go.mod +++ b/go/go.mod @@ -1,19 +1,31 @@ module github.com/xiaoxue/memoryweave -go 1.23.0 +go 1.25.0 -toolchain go1.23.5 +require ( + github.com/gorilla/websocket v1.5.3 + github.com/prometheus/client_golang v1.23.2 + modernc.org/sqlite v1.59.0 +) require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/gorilla/websocket v1.5.3 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/prometheus/client_golang v1.23.2 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.16.1 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect - golang.org/x/sys v0.35.0 // indirect + golang.org/x/sys v0.47.0 // indirect google.golang.org/protobuf v1.36.8 // indirect + modernc.org/libc v1.75.7 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.12.1 // indirect ) diff --git a/go/go.sum b/go/go.sum index 8830437..508b214 100644 --- a/go/go.sum +++ b/go/go.sum @@ -2,10 +2,37 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= @@ -14,12 +41,56 @@ github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9Z github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= -golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= -golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= +golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.29.2 h1:h6+9ciCnPKutf4I03CvheAvDLX7+IHlqR6Iy6J+cgd8= +modernc.org/cc/v4 v4.29.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.35.0 h1:F+TUsmw09QxLzmi3aeYYGxjAXarmZaKgj3mKQHNaA8w= +modernc.org/ccgo/v4 v4.35.0/go.mod h1:qrVGs9S3Sr2Ztcg9ve+kTAYMp5a3YvWjo+SoN06kJ5I= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc= +modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.75.7 h1:o3DTP9/0p9pKmY2WCKQaySW6wIiZhNM7wc2lUoyhfew= +modernc.org/libc v1.75.7/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= +modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.59.0 h1:X1es1GpqBlS/5T+vbM4HLUdaa8OtQx468DF2vrx+38A= +modernc.org/sqlite v1.59.0/go.mod h1:+paeT2A3iPRHkQDwG7oA6Tk0zQd5woMEI8q7orfry8k= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/go/internal/api/routes/core.go b/go/internal/api/routes/core.go index 0a26160..e25d438 100644 --- a/go/internal/api/routes/core.go +++ b/go/internal/api/routes/core.go @@ -250,9 +250,10 @@ func cosineSimilarity(a, b []float32) float64 { // - 真无匹配(成功但 results 空,含 keyword 主路径 0 命中)⇒ 200 + results=[] + candidates_scanned // - 有匹配 ⇒ 200 + results + candidates_scanned // -// candidates_scanned 字段契约(t_a70b07e1 修订,不得再当最终返回数的代理值): -// = 主路径(Pipeline.Recall / graph 兜底)产出候选数,在 cross-namespace / 笔记补充扩展 -// 之前捕获;**不等于** 最终返回的 len(results)(E4.2 / wiki 补充扩展发生在捕获之后)。 +// candidates_scanned 字段契约(t_a70b07e1 修订 + t_949840b1 名实相符): +// = 主路径粗检索候选池真实大小(LanceDB ANN 检索返回 + 去重 + 原始对话过滤之后、 +// rerank/MMR 截断之前捕获;cross-namespace / 笔记补充扩展发生在捕获之后,不计入); +// **不等于** 最终返回的 len(results)(E4.2 / wiki 补充扩展 + 截断改变了产出数)。 // - keyword 主路径不做 ANN 扫描 ⇒ 恒 0 // - graph 兜底路径(X-Fallback: graph)= 兜底返回数 // 用途:区分「真无匹配」(results=[] 但候选数>0)与「依赖故障」(非 2xx)。 @@ -291,11 +292,12 @@ func (a *API) Recall(w http.ResponseWriter, r *http.Request) { var results []models.RecallResult var err error - // t_2d4e3b92 引入 + t_a70b07e1 语义修订:annScanned = 主路径(Pipeline.Recall / graph 兜底) - // 产出候选数,在 cross-namespace / 笔记补充扩展**之前**捕获。 - // ⚠️ 它**不等于**最终 len(results)(E4.2 / wiki 补充扩展发生在捕获之后); - // 字段名 candidates_scanned 语义 = 主路径召回前候选数(keyword 路径未做 ANN ⇒ 0), - // 调用方不得用它推断最终返回条数(那以 count 为准)。 + // t_2d4e3b92 引入 + t_a70b07e1 语义修订 + t_949840b1 名实相符: + // annScanned = 主路径(Pipeline.Recall / graph 兜底)粗检索候选池真实大小 + // (LanceDB ANN 返回 + 去重 + 原始对话过滤之后、rerank/MMR 截断之前捕获)。 + // 此前误用 len(results)(最终产出数)作代理值——字段名说 "scanned"(扫描候选池) + // 值却是"产出数",属"用文档补假名"。现改为 RecallWithMeta 返回的候选池真实大小。 + // keyword 路径未做 ANN 扫描 ⇒ 恒 0;graph 兜底路径 = 兜底返回数。 annScanned := 0 // H5: Mode routing — hybrid / semantic / keyword @@ -341,10 +343,15 @@ func (a *API) Recall(w http.ResponseWriter, r *http.Request) { return case "semantic": // Pure semantic — BM25 off (H1 skipped, Pipeline Recall has BM25 built in — handled by switch) - results, err = a.Pipeline.Recall(req.Query, req.Namespace, req.Limit, req.Diversity) + // t_949840b1:RecallWithMeta 额外返回候选池真实大小(ANN 粗检索池,非最终产出数) + var meta storage.RecallMeta + results, meta, err = a.Pipeline.RecallWithMeta(req.Query, req.Namespace, req.Limit, req.Diversity) + annScanned = meta.CandidatesScanned case "hybrid": // BM25 + vector combined (H1: BM25 scoring inside Pipeline.Recall) - results, err = a.Pipeline.Recall(req.Query, req.Namespace, req.Limit, req.Diversity) + var meta storage.RecallMeta + results, meta, err = a.Pipeline.RecallWithMeta(req.Query, req.Namespace, req.Limit, req.Diversity) + annScanned = meta.CandidatesScanned } if err != nil { // P0: 降级到 graph.db 关键词搜索(仅 hybrid/semantic 主路径依赖向量/rerank) @@ -361,10 +368,9 @@ func (a *API) Recall(w http.ResponseWriter, r *http.Request) { return } } else if req.Mode != "keyword" { - // 主路径成功:annScanned 取主路径(Pipeline.Recall)召回前候选数。 - // t_a70b07e1:语义 = 主路径产出候选数(后续 E4.2/wiki 补充扩展不计入), - // 与最终 count 解耦——count 才是最终返回条数。 - annScanned = len(results) + // 主路径成功:annScanned 已在 switch 里取 RecallWithMeta 返回的候选池真实大小。 + // t_949840b1:不再用 len(results)(最终产出数)覆盖——那是"用文档补假名"的根因。 + // (若 meta 返回 0(缓存命中无法重建候选池),保持 0,不虚报。) } // E4.2: 跨 agent 知识共享 — recall 结果在 (0, top_k) 时,补充搜索 "shared" namespace diff --git a/go/internal/api/routes/testdata/golden-set.json b/go/internal/api/routes/testdata/golden-set.json index 9e80b02..8dc8d4d 100644 --- a/go/internal/api/routes/testdata/golden-set.json +++ b/go/internal/api/routes/testdata/golden-set.json @@ -1,5 +1,5 @@ { - "description": "织忆召回金标检查集(12条基线 + 2条目标知识)。统一判据口径(t_a70b07e1):取 recall 返回的前 N=5 条内容查关键词,ID 命中 = expected_ids 任一 ∈ 前 N=5 条;N=5 与 Go/Python 两量具(eval_test.go / golden-check.py)一致。判定:PASS=id且kw命中 / WARN=kw命中但ID漂移 / FAIL=其余 / ERROR=recall调用失败。⚠️ 已知漂移(2026-09-21 校准):item4「牧尘喜欢什么样的工作方式」基线 id mem_1789837840173161705 已漂移——语料增长致 top-1 位置变化,现 top-5 稳定成员为 mem_1789834428978989104(前 N 内容仍切题);两量具对该条分类可能差一级(PASS vs WARN),待人工校准基线后 diff=0。其余 13 条分类一致。", + "description": "织忆召回金标检查集(12条基线 + 2条目标知识)。统一判据口径(t_a70b07e1):取 recall 返回的前 N=5 条内容查关键词,ID 命中 = expected_ids 任一 ∈ 前 N=5 条;N=5 与 Go/Python 两量具(eval_test.go / golden-check.py)一致。判定:PASS=id且kw命中 / WARN=kw命中但ID漂移 / FAIL=其余 / ERROR=recall调用失败。单一真相源(t_949840b1):本文件是金标数据唯一副本;Python 量具 golden-check.py 直接读本文件(旧 ~/.hermes/scripts/tools/golden-set.json 副副本已删除),基线校准一次即两量具同步(--refresh 写回本文件)。⚠️ item4「牧尘喜欢什么样的工作方式」基线 2026-09-21 t_949840b1 校准为当前稳定值 mem_1789834428978989104(语料增长致旧基线 mem_1789837840173161705 掉出 top-5,前 N 内容仍切题)。校准后同批金标两量具分类差异=0(diff=1 的根因——两份副本 expected_ids 不同步——已消除)。", "items": [ { "query": "织忆是什么", @@ -35,11 +35,11 @@ }, { "query": "牧尘喜欢什么样的工作方式", - "expected_ids": ["mem_1789837840173161705"], + "expected_ids": ["mem_1789834428978989104"], "must_include_keywords": ["牧尘"], "category": "user_pref", "difficulty": "medium", - "id_baseline": "mem_1789837840173161705" + "id_baseline": "mem_1789834428978989104" }, { "query": "如何根据牧尘的偏好调整记忆检索策略", diff --git a/go/internal/governance/conflict_retention.go b/go/internal/governance/conflict_retention.go index a9b48dc..22ff559 100644 --- a/go/internal/governance/conflict_retention.go +++ b/go/internal/governance/conflict_retention.go @@ -20,6 +20,8 @@ import ( "os" "path/filepath" "time" + + "github.com/xiaoxue/memoryweave/internal/metrics" ) // ConflictRetentionConfig 冲突保留策略配置 @@ -138,6 +140,15 @@ func RetainAndArchive(cd *ConflictDetector, config *ConflictRetentionConfig) (ar } log.Printf("[conflict-retention] 归档 %d 条 → %s", archivedCount, archivePath) + // 2026-09-21:更新 Prometheus gauge(保留策略归档后 pending 数变化) + // 注意:必须在 defer Unlock 前计算 pending 数(持有锁) + n := 0 + for _, c := range cd.active { + if c.Status == "pending" { + n++ + } + } + metrics.ConflictsPending.Set(float64(n)) return archivedCount, archivePath, nil } diff --git a/go/internal/governance/governance.go b/go/internal/governance/governance.go index 168c96d..53d6168 100644 --- a/go/internal/governance/governance.go +++ b/go/internal/governance/governance.go @@ -12,6 +12,8 @@ import ( "sync" "time" "unicode" + + "github.com/xiaoxue/memoryweave/internal/metrics" ) // ─── 冲突检测 ──────────────────────────────────────────── @@ -105,6 +107,8 @@ func NewPersistentConflictDetector(path string, seenPath ...string) (*ConflictDe if err := cd.loadSeenPairs(); err != nil { return cd, err } + // 2026-09-21:启动时更新 Prometheus gauge(加载 conflicts.json 后) + metrics.ConflictsPending.Set(float64(cd.PendingCount())) return cd, nil } @@ -281,6 +285,8 @@ func (cd *ConflictDetector) Scan(newContent string, newEntities []string, existi cd.persistLocked() // 2026-09-20:新增即落盘(重启不丢) cd.persistSeenPairsLocked() // 2026-09-21:seenPairs 也落盘(重启不丢) cd.mu.Unlock() + // 2026-09-21:更新 Prometheus gauge(冲突数) + metrics.ConflictsPending.Set(float64(cd.PendingCount())) // 只返回未被抑制的冲突 return registered } @@ -359,6 +365,15 @@ func (cd *ConflictDetector) DedupeActive(config *ConflictRetentionConfig) int { cd.persistLocked() log.Printf("[conflict-dedup] 去重归档 %d 条 → %s,剩余 %d 条", len(removed), archivePath, len(cd.active)) + // 2026-09-21:更新 Prometheus gauge(去重后 pending 数变化) + // 注意:必须在 defer Unlock 前计算 pending 数(持有锁) + n := 0 + for _, c := range cd.active { + if c.Status == "pending" { + n++ + } + } + metrics.ConflictsPending.Set(float64(n)) return len(removed) } @@ -852,6 +867,16 @@ func (cd *ConflictDetector) Resolve(id, resolution, winner string) error { c.Strategy = resolution cd.persistLocked() // 2026-09-20:状态变更即落盘 } + // 2026-09-21:更新 Prometheus gauge(Resolve 后 pending 数减少) + // 注意:PendingCount 会获取 RLock,所以必须在 defer Unlock 前调用 + // 正确顺序:先获取 PendingCount(拿 RLock → 释放),再 Set gauge + n := 0 + for _, c := range cd.active { + if c.Status == "pending" { + n++ + } + } + metrics.ConflictsPending.Set(float64(n)) return nil } diff --git a/go/internal/storage/recall.go b/go/internal/storage/recall.go index f3a9612..98bc325 100644 --- a/go/internal/storage/recall.go +++ b/go/internal/storage/recall.go @@ -57,32 +57,52 @@ func (p *RecallPipeline) SetMissRecorder(mr MissRecorder) { p.recordMiss = mr } -func (p *RecallPipeline) Recall(query, namespace string, topK int, diversity float64) ([]models.RecallResult, error) { - if topK <= 0 { - topK = 10 - } +// RecallMeta RecallWithMeta 返回的召回元数据(t_949840b1:candidates_scanned 名实相符)。 +// CandidatesScanned = 粗检索候选池真实大小(LanceDB ANN 检索返回 + 去重 + +// 原始对话过滤之后的候选数,在 rerank/MMR 截断之前捕获)。 +// 这才是 recall API 里 candidates_scanned 字段的真实语义——此前误用 +// len(results)(最终产出数)作代理值,字段名说 "scanned"(扫描候选池)值却是"产出数"。 +type RecallMeta struct { + CandidatesScanned int +} - // Step 0: 搜索缓存(§2.6 — 相同 query hash → TTL 1h → 命中直接返回) +// RecallWithMeta 同 Recall,但额外返回召回元数据(候选池真实大小)。 +// 缓存命中时无法重建候选池 ⇒ CandidatesScanned = 0(保守:无法测量就报 0,不虚报)。 +func (p *RecallPipeline) RecallWithMeta(query, namespace string, topK int, diversity float64) ([]models.RecallResult, RecallMeta, error) { + // 缓存命中分支:直接返回,候选池不可重建 if SearchCacheInstance != nil { if cached, ok := SearchCacheInstance.Get(query, namespace); ok { var results []models.RecallResult if err := json.Unmarshal(cached, &results); err == nil { - // Cache hit: 先过滤原始对话类别,再返回(防止旧缓存污染) filtered := filterRecallRawCategories(results) - // 降级:若过滤后为空,回退到原结果(避免 cache 造成的完全失效) if len(filtered) == 0 && len(results) > 0 { filtered = results } p.recordRecallWrites(filtered) - return filtered, nil + return filtered, RecallMeta{}, nil } } } + return p.recallUncached(query, namespace, topK, diversity) +} + +// Recall 保持原签名(缓存 + 全量流程),行为不变。 +func (p *RecallPipeline) Recall(query, namespace string, topK int, diversity float64) ([]models.RecallResult, error) { + results, _, err := p.RecallWithMeta(query, namespace, topK, diversity) + return results, err +} + +// recallUncached Recall 的非缓存路径。候选池大小在 Step 2.5 过滤后、 +// Step 3 截断前捕获(真实粗检索池,不是最终产出数)。 +func (p *RecallPipeline) recallUncached(query, namespace string, topK int, diversity float64) ([]models.RecallResult, RecallMeta, error) { + if topK <= 0 { + topK = 10 + } // Step 1: Encode query queryVec, err := p.embedder.EncodeSingle(query) if err != nil { - return nil, fmt.Errorf("recall encode: %w", err) + return nil, RecallMeta{}, fmt.Errorf("recall encode: %w", err) } // Step 2: Coarse search — own namespace + shared(§1.3 召回规则) @@ -150,6 +170,10 @@ func (p *RecallPipeline) Recall(query, namespace string, topK int, diversity flo } candidates = filtered + // 候选池真实大小捕获点:LanceDB ANN 返回 + 去重 + 原始对话过滤之后、 + // 截断/rerank/MMR 之前 —— 这才是 "scanned candidates" 的真实含义。 + poolSize := len(candidates) + // 截断候选池(全库模式候选来自更多命名空间,池子放宽到 100) candCap := 50 if namespace == "" { @@ -173,7 +197,7 @@ func (p *RecallPipeline) Recall(query, namespace string, topK int, diversity flo if p.recordMiss != nil { p.recordMiss(query, namespace) } - return []models.RecallResult{}, nil + return []models.RecallResult{}, RecallMeta{CandidatesScanned: poolSize}, nil } // Step 3: Rerank to top_k (graceful fallback) @@ -297,7 +321,7 @@ func (p *RecallPipeline) Recall(query, namespace string, topK int, diversity flo } }() - return results, nil + return results, RecallMeta{CandidatesScanned: poolSize}, nil } // ComputeBM25Score 基于词频的关键词匹配分数