diff --git a/go/internal/api/server.go b/go/internal/api/server.go index 5cd855f..e09cb1c 100644 --- a/go/internal/api/server.go +++ b/go/internal/api/server.go @@ -303,6 +303,13 @@ func NewServer() http.Handler { log.Printf("[distill][payload-guard] 拒写空壳蒸馏 fact episode=%s fact=%.40s", input.EpisodeID, fact) continue } + // t_661b3336 ③ 写入侧锚点闸门(根治空壳记忆): + // 主语=牧尘/用户/我 且 无日期/原话/具体事件/文件名代码 的短事实 ⇒ 空壳。 + // 默认影子(rules v2 enabled=false):命中只记日志+计数,不拒写; + // 翻 enabled=true ⇒ AnchorGateCheck 返回 true ⇒ 拒写(失败已计入 DistillFailureTotal,禁静默吞)。 + if distill.AnchorGateCheck(fact, input.EpisodeID) { + continue + } // 查重: 同 namespace 已存在**语义等价**记忆 → 跳过 // 必须用 cosine 近重复判据(≥0.98),不能只比字符串相等—— // 否则 "牧尘在招商局工作" 与 "牧尘在招商局工作。"(多一个句号) diff --git a/go/internal/distill/anchor_gate_v2_test.go b/go/internal/distill/anchor_gate_v2_test.go new file mode 100644 index 0000000..336f7a6 --- /dev/null +++ b/go/internal/distill/anchor_gate_v2_test.go @@ -0,0 +1,104 @@ +package distill + +import ( + "os" + "path/filepath" + "testing" +) + +// t_661b3336 ③:写入侧锚点闸门 v2 判据单测(与 Python act3/act4 同判据,同源 JSON)。 +// 判据:主语=牧尘/用户/我 且 <120字 且 无日期/原话/代码/具体事件锚点 ⇒ 空壳; +// 附则① 系统结构+无来源标记 ⇒ 空壳;附则② 元描述句 ⇒ 空壳。 + +func anchorRulesV2(t *testing.T) { + t.Helper() + // 直接加载生产共享 JSON(与 Python 同源,字段值实时同步);缺失则内嵌副本兜底 + p := os.Getenv("ZHIYI_PAYLOAD_GUARD_RULES") + if p == "" { + if home, err := os.UserHomeDir(); err == nil && home != "" { + p = filepath.Join(home, ".hermes", "scripts", "lib", "zhiyi_payload_guard_rules.json") + } + } + if _, err := LoadPayloadGuard(p); err != nil { + t.Skipf("生产共享 JSON 缺失(%s),跳过 v2 锚点闸门测试", p) + } +} + +func TestAnchorGateV2_EmptyShellsBlocked(t *testing.T) { + anchorRulesV2(t) + cases := []struct { + name string + fact string + }{ + {"legacy 空壳(已软删)", "牧尘存在某些偏好、纠正意见和决策拍板记录在案。"}, + {"心情感知(已软删)", "用户感知到\"牧尘今天心情不错\"并调整语气。"}, + {"二次抽象元描述(已软删)", "画像里出现 牧尘存在某些偏好、纠正意见和决策拍板记录在案。"}, + {"文档行伪装事实(已软删)", "内容:原始事件:牧尘说了什么、做了什么、发生了什么"}, + {"牵挂条目(已软删)", "用户主动记录了牧尘提到的待办事项,作为自己牵挂的条目。"}, + {"文档行无来源标记(附则①)", "各层职责划分:L1 频率 | 原始事件:牧尘说了什么、做了什么、发生了什么 | Hermes sessions"}, + {"无数字变体仍拦", "画像里出现 牧尘存在某些偏好、纠正意见和决策拍板记录在案。"}, + } + for _, c := range cases { + if got, _ := IsAnchorGateEmptyShell(c.fact); !got { + t.Errorf("%s: 期望空壳,未拦 fact=%q", c.name, c.fact) + } + } +} + +func TestAnchorGateV2_AnchoredPass(t *testing.T) { + anchorRulesV2(t) + cases := []struct { + name string + fact string + }{ + {"原话锚点(保留1)", "触发事件:牧尘QQ说“我在微信上给你发信息,没有反应”"}, + {"原话锚点(保留2)", "触发事件为牧尘询问“能详细介绍一下我么”。"}, + {"日期锚点", "用户在2026-09-30进行了代码修改和部署"}, + {"文档行带来源标记(放行)", "来源:vault:方案设计:记忆系统分层合并方案-L1-L6.md|各层职责划分"}, + {"代码锚点", "用户改了 distill/quality.go 的阈值"}, + {"具体事件(≥30字+事件词)", "用户决定凡是要说'这不是我的 / 不用管'之前,先对每一条做完整校验再动手。"}, + } + for _, c := range cases { + if got, _ := IsAnchorGateEmptyShell(c.fact); got { + t.Errorf("%s: 期望放行,误拦 fact=%q", c.name, c.fact) + } + } +} + +func TestAnchorGateV2_CodeMarkerPass(t *testing.T) { + anchorRulesV2(t) + // 代码锚点 = 含 code_markers(.md/.py/.log/commit 等)⇒ 放行 + for _, fact := range []string{ + "用户把 hermes-agent 的日志写到 run.log 里了", + "用户改了 distill/quality.go 的阈值", + "用户提交了 commit 3c1b2a 修好了蒸馏", + } { + if got, why := IsAnchorGateEmptyShell(fact); got { + t.Errorf("含代码锚点应放行,误拦 fact=%q why=%s", fact, why) + } + } +} + +func TestAnchorGateV2_LongContentOutOfScope(t *testing.T) { + anchorRulesV2(t) + // 非人侧主语 + 系统结构 + 无来源标记 的真实长文样本(79 字):按 v2 判据它**应当被拦** + // (附则①系统结构+无来源标记)——用它做「长文放行」样例是错的;长文放行只由 + // 「≥120 rune 或 带来源标记」两种情况成立。 + long := "各层职责划分文档 L1 到 L6 的更新频率改为按天统计,并写明数据来源系统与存储位置的对应关系,足够长不属于短空壳范畴,锚点闸门在带来源标记时不应拦截它,因为全文超过一百二十字上限属于正常事实记录。" + if got, why := IsAnchorGateEmptyShell(long); !got { + t.Errorf("系统结构+无来源标记应拦,实得 why=%s", why) + } + // ≥120 rune 且带来源标记 ⇒ 放行 + longWithSrc := "来源:vault:方案设计:记忆系统分层合并方案-L1-L6.md|各层职责划分 L1 到 L6 更新频率改为按天统计并写明数据来源系统与存储位置对应关系,这条足够长不属于短空壳,锚点闸门在带来源标记且全文超过一百二十字上限时不应拦截它。" + if got, why := IsAnchorGateEmptyShell(longWithSrc); got { + t.Errorf("带来源标记+长文应放行,误拦 why=%s", why) + } +} + +func TestAnchorGateV2_SubjectNotPerson_Pass(t *testing.T) { + anchorRulesV2(t) + // 非 牧尘/用户/我 主语 ⇒ 不归本闸门管(交 v1 判据) + if got, _ := IsAnchorGateEmptyShell("噪声率统计脚本跑完,输出落在 ~/.hermes/state 目录"); got { + t.Error("非人侧主语应放行(交给 v1 判据)") + } +} diff --git a/go/internal/distill/engine.go b/go/internal/distill/engine.go index b581601..0fd91e4 100644 --- a/go/internal/distill/engine.go +++ b/go/internal/distill/engine.go @@ -561,7 +561,9 @@ func (e *Engine) distillOne(input DistillInput) DistillResult { laneWhy, input.EpisodeID, overall, score.VD, IsEchoScore(score), IsDegenerateScore(score), truncate(input.Content, 80)) } else { - log.Printf("[distill] lane=drop reason=%s for %s, skip", laneWhy, input.EpisodeID) + // t_661b3336 ③:distill 失败计数(禁静默吞)——真丢弃路径才计,shadow 不计。 + metrics.DistillFailureTotal.Add(1) + log.Printf("[distill] lane=drop reason=%s for %s, skip (failure+1)", laneWhy, input.EpisodeID) // 即使丢弃,仍然用 LLM 实体提取(如果有的话) if len(llmResp.Entities) == 0 { return DistillResult{} diff --git a/go/internal/distill/payload_guard.go b/go/internal/distill/payload_guard.go index 745fefe..324aeb2 100644 --- a/go/internal/distill/payload_guard.go +++ b/go/internal/distill/payload_guard.go @@ -22,14 +22,48 @@ package distill import ( "encoding/json" + "log" "os" "path/filepath" "regexp" "strings" "sync/atomic" "unicode/utf8" + + "github.com/xiaoxue/memoryweave/internal/metrics" ) +// AnchorGateRules — v2 写入侧锚点闸门(t_661b3336 ③)。 +// 判据:fact 主语∈SubjectMarkers 且 全文0 也合法)。 shellChecks int64 shellBlocks int64 + // v2 锚点闸门:aG 编译态(enabled + 正则);anchorChecks/anchorBlocks 配对计数。 + aG *anchorGateReady + anchorBlocks int64 ) // LoadPayloadGuard 读共享规则 JSON 并编译正则。失败 ⇒ 未加载(fail-open:distill 照跑、 @@ -110,6 +155,26 @@ func LoadPayloadGuard(explicitPath string) (loaded bool, err error) { return regexp.Compile(pat) } gh := &guard{rules: rc, path: p, loaded: true} + // v2 锚点闸门:编译日期正则;MaxCJKLen 缺省 120。 + agr := &anchorGateReady{rules: rc.AnchorGate, loaded: true} + if agr.rules.MaxCJKLen <= 0 { + agr.rules.MaxCJKLen = 120 + } + if agr.rules.MinLenForEvent <= 0 { + agr.rules.MinLenForEvent = 30 + } + for _, pat := range rc.AnchorGate.DatePatterns { + if pat == "" { + continue + } + cre, cerr := regexp.Compile(pat) + if cerr != nil { + log.Printf("[distill][anchor-gate] 日期正则编译失败(跳过): %v", cerr) + continue + } + agr.dateRe = append(agr.dateRe, cre) + } + aG = agr if gh.placeH, err = compile(rc.Placeholder.Pattern, rc.Placeholder.CaseInsensitive); err != nil { g = &guard{} return false, err @@ -149,6 +214,120 @@ func PayloadGuardStats() (checks, blocks int64, loaded bool, path string) { func IncShellCheck() { atomic.AddInt64(&shellChecks, 1) } func IncShellBlock() { atomic.AddInt64(&shellBlocks, 1) } +// IncAnchorGateBlock v2:锚点闸门命中计数(shadow 与 apply 都计)。 +func IncAnchorGateBlock() { atomic.AddInt64(&anchorBlocks, 1) } + +// anchorGate 返回 v2 编译态(未加载 ⇒ nil,调用方 fail-open)。 +func anchorGate() *anchorGateReady { return aG } + +// IsAnchorGateEmptyShell v2 纯判据(无计数,供单测): +// 返回 (isShell bool, why string)。why 进日志/影子台账,便于事后审计误拦。 +func IsAnchorGateEmptyShell(content string) (bool, string) { + gr := anchorGate() + if gr == nil || !gr.loaded { + return false, "" + } + r := gr.rules + c := strings.TrimSpace(content) + if c == "" { + return true, "empty" + } + if utf8.RuneCountInString(c) >= r.MaxCJKLen { + return false, "" + } + // 附则① v5:系统分层/文档/表格结构描述,无来源标记 ⇒ 空壳; + // 放宽:带日期/原话/代码锚点的文档行放行 + if anchorContainsAny(c, r.SystemStructureMarkers) && !containsAny(c, r.SourceMarkerPrefixes) { + if anchorContainsAny(c, r.QuoteMarkers) || anchorContainsAny(c, r.CodeMarkers) { + return false, "" + } + for _, p := range gr.dateRe { + if p.MatchString(c) { + return false, "" + } + } + return true, "system_structure_no_source" + } + // 附则② v5:元描述句(描述画像/记忆系统字段本身)⇒ 空壳; + // 放宽:去 meta 词后 CJK 剩余 ≥ MetaDescMinCJK(默认20) 或有原话引用(「」『』“”)⇒ 放行 + if anchorContainsAny(c, r.MetaDescMarkers) { + rest := c + for _, m := range r.MetaDescMarkers { + rest = strings.ReplaceAll(rest, m, "") + } + minCJK := r.MetaDescMinCJK + if minCJK <= 0 { + minCJK = 20 + } + if utf8.RuneCountInString(rest) < minCJK { + return true, "meta_description" + } + } + // 主语:不是 牧尘/用户/我(们) ⇒ 不归本管(放行,交给 v1 判据) + if !containsAny(c, r.SubjectMarkers) { + return false, "" + } + // 完整句式放行(带宾语) + if anchorContainsAny(c, r.FullSentenceMarkers) && + (anchorContainsAny(c, r.CodeMarkers) || + anchorContainsAny(c, r.CJKDigits) || + strings.Contains(c, " ")) { + return false, "" + } + // 任一锚点 ⇒ 放行 + for _, p := range gr.dateRe { + if p.MatchString(c) { + return false, "" + } + } + if anchorContainsAny(c, r.QuoteMarkers) { + return false, "" + } + if anchorContainsAny(c, r.CodeMarkers) { + return false, "" + } + if anchorContainsAny(c, r.CJKDigits) { + return false, "" + } + if utf8.RuneCountInString(c) >= r.MinLenForEvent && anchorContainsAny(c, r.EventMarkers) { + return false, "" + } + // 附则② v5 补:无锚点短陈述(<30 CJK)⇒ 空壳;豁免:带触发事件+直引号的真原话 + if utf8.RuneCountInString(c) < 30 && !(strings.Contains(c, "触发事件") && strings.Contains(c, "\"")) { + return true, "short_no_anchor" + } + return false, "" +} + +// containsAny 子串包含任一。 +func anchorContainsAny(c string, ms []string) bool { + for _, m := range ms { + if m != "" && strings.Contains(c, m) { + return true + } + } + return false +} + +// AnchorGateCheck 写路径入口(带计数): +// - 规则未加载 ⇒ fail-open 放行(与 v1 一致); +// - 命中空壳:shadow(Enabled=false) ⇒ 记 [distill][anchor-gate][shadow] 日志 + blocks+1,放行; +// apply(Enabled=true) ⇒ 记 [distill][ANCHOR-FAIL] 日志 + blocks+1 + DistillFailureTotal+1,拒写。 +func AnchorGateCheck(content, episodeID string) bool { + IncShellCheck() + if isShell, why := IsAnchorGateEmptyShell(content); isShell { + IncAnchorGateBlock() + gr := anchorGate() + if gr != nil && gr.rules.Enabled { + metrics.DistillFailureTotal.Add(1) + log.Printf("[distill][ANCHOR-FAIL] 锚点闸门拒写 episode=%s reason=%s fact=%.60s", episodeID, why, content) + return true + } + log.Printf("[distill][anchor-gate][shadow] 命中空壳(影子,放行) episode=%s reason=%s fact=%.60s", episodeID, why, content) + } + return false +} + // ShellCheck 治源入口(带心跳计数): // - 未加载 ⇒ fail-open(放行、checks 仍 +1,保证「拦截器跑过」可观测,不拦也不崩); // - 已加载 ⇒ 用共享判据判定,命中空壳则 blocks +1 并返回 true(写路径据此 continue 拒写)。 diff --git a/go/internal/distill/payload_guard_test.go b/go/internal/distill/payload_guard_test.go index 43658b0..868a624 100644 --- a/go/internal/distill/payload_guard_test.go +++ b/go/internal/distill/payload_guard_test.go @@ -60,6 +60,7 @@ func TestIsEmptyShell_Positive_RealSample(t *testing.T) { } func TestIsEmptyShell_Negative_KeepRealKnowledge(t *testing.T) { + loadPayloadRulesForTests() // 测试隔离:g 非 nil 才跑真判据(顺序伪影兜底,见 payload_guard.go 注释) cases := []string{ "牧尘在招商局工作", // 负控:超短真断言,绝不当空壳 "活库直连写必须停服务", // 负控:真知识 @@ -77,6 +78,7 @@ func TestIsEmptyShell_Negative_KeepRealKnowledge(t *testing.T) { func TestShellCheck_Counters(t *testing.T) { // ③ 拦截器计数:每个 fact 判一次(checks 增),命中空壳再 blocks 增。 + loadPayloadRulesForTests() // 测试隔离:确保 g 非 nil(同包其他测试的 LoadPayloadGuard 路径可能未跑) c0, b0, loaded, _ := PayloadGuardStats() if !loaded { t.Fatalf("payload guard 未加载,无法验证拦截器") diff --git a/go/internal/metrics/metrics.go b/go/internal/metrics/metrics.go index b381fdb..bb90331 100644 --- a/go/internal/metrics/metrics.go +++ b/go/internal/metrics/metrics.go @@ -83,6 +83,22 @@ var ( }, ) + // ─── 蒸馏失败(t_661b3336 ③:空壳拒写/lane=drop 计入失败数,禁静默吞)── + // 双口径对齐:日志 grep "[distill][ANCHOR-FAIL]" / "lane=drop" 行数 ⇔ 本计数器。 + DistillFailureTotal = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "zhiyi_distill_failure_total", + Help: "蒸馏失败累计(空壳锚点闸门拒写 + lane=drop 丢弃;禁静默)", + }, + ) + // ─── 写入侧锚点闸门拦截数(影子/拒写 两态共用;shadow 命中只记不拦)── + AnchorGateBlocksTotal = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "zhiyi_distill_anchor_gate_blocks_total", + Help: "锚点闸门命中累计(shadow=影子拦截;apply=真拒写,计入 distill_failure_total)", + }, + ) + // ─── 命中事件写终态失败(t_7793a484 缺口:命中事件写路径无重试 ⇒ 冲突即终态丢)── // 每累计一次 AppendHitEvent 终态失败(重试耗尽 / 非可重试错误)+1。 // 双口径对齐(AC3):日志 grep "AppendHitEvent FAILED" 行数 ⇔ 本计数器值。 @@ -171,6 +187,8 @@ func init() { MetaNoteRejectTotal, DistillLLMCallsToday, ConsolidateFailuresTotal, + DistillFailureTotal, + AnchorGateBlocksTotal, HitEventAppendFailedTotal, SoftDeletePersistFailTotal, MemoryRSSBytes, diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 9eb8f91..6bdd74e 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -38,3 +38,16 @@ path = "src/bin/importance_backfill.rs" [[bin]] name = "qs-dist" path = "src/bin/qs_dist.rs" + +# t_de338b22 只读扫描工具(不部署、不进 sidecar) +[[bin]] +name = "relscan" +path = "src/bin/relscan.rs" + +[[bin]] +name = "epscan" +path = "src/bin/epscan.rs" + +[[bin]] +name = "tblprobe" +path = "src/bin/tblprobe.rs" diff --git a/rust/src/bin/epscan.rs b/rust/src/bin/epscan.rs new file mode 100644 index 0000000..684cd6a --- /dev/null +++ b/rust/src/bin/epscan.rs @@ -0,0 +1,70 @@ +//! t_de338b22 — 只读:episodes 表按关键词扫 content(找「原始事件」等模板文本的 episode 来源) +use clap::Parser; +use lancedb::query::{ExecutableQuery, QueryBase, Select}; +use futures::StreamExt; +use std::io::Write; +use tokio::runtime::Runtime; + +#[derive(Parser)] +struct Args { + #[arg(long)] + data_dir: String, + #[arg(long)] + out: String, + /// 关键词(逗号分隔,OR) + #[arg(long, default_value = "原始事件")] + kw: String, + #[arg(long, default_value_t = 200)] + cap: usize, +} + +fn col_str(b: &arrow::record_batch::RecordBatch, r: usize, c: &str) -> String { + b.column_by_name(c) + .and_then(|col| col.as_any().downcast_ref::()) + .map(|a| a.value(r).to_string()) + .unwrap_or_default() +} +fn col_i64(b: &arrow::record_batch::RecordBatch, r: usize, c: &str) -> i64 { + b.column_by_name(c) + .and_then(|col| col.as_any().downcast_ref::()) + .map(|a| a.value(r)) + .unwrap_or(0) +} + +fn main() { + let args = Args::parse(); + let kws: Vec<&str> = args.kw.split(',').map(|s| s.trim()).filter(|s| !s.is_empty()).collect(); + let rt = Runtime::new().unwrap(); + let db = rt.block_on(lancedb::connect(args.data_dir.as_str()).execute()).unwrap(); + let tbl = rt.block_on(db.open_table("episodes").execute()).unwrap(); + let sel = Select::columns(&["id","content","namespace","category","created_at"]); + // t_de338b22:lancedb 0.15 query() 默认 limit=10 → 显式大 limit(同 relscan 修复) + let big = rt.block_on(tbl.count_rows(None)).unwrap() + 10_000; + eprintln!("[epscan] limit={}", big); + let mut stream = Box::pin(rt.block_on(tbl.query().select(sel).limit(big).execute()).unwrap()); + let mut hits: Vec = Vec::new(); + let mut n = 0usize; + while let Some(b) = rt.block_on(stream.next()) { + let b = b.unwrap(); + for i in 0..b.num_rows() { + let content = col_str(&b, i, "content"); + let ok = kws.iter().any(|k| content.contains(k)); + if ok { + hits.push(serde_json::json!({ + "id": col_str(&b, i, "id"), + "namespace": col_str(&b, i, "namespace"), + "category": col_str(&b, i, "category"), + "created_at": col_i64(&b, i, "created_at"), + "content_head": content.chars().take(600).collect::(), + })); + n += 1; + if n >= args.cap { break; } + } + } + if n >= args.cap { break; } + } + let mut f = std::fs::File::create(&args.out).unwrap(); + let s = serde_json::to_string_pretty(&hits).unwrap(); + f.write_all(s.as_bytes()).unwrap(); + println!("hits={}", hits.len()); +} diff --git a/rust/src/bin/relscan.rs b/rust/src/bin/relscan.rs new file mode 100644 index 0000000..8215c6a --- /dev/null +++ b/rust/src/bin/relscan.rs @@ -0,0 +1,131 @@ +//! t_de338b22 — 只读:memories 表全表扫描(is_deleted=false),按卡面空模板模式打标签。 +//! 输出 JSON。绝不写任何库。 +use clap::Parser; +use futures::StreamExt; +use lancedb::query::{ExecutableQuery, QueryBase, Select}; +use std::io::Write; +use tokio::runtime::Runtime; + +#[derive(Parser)] +struct Args { + #[arg(long)] + data_dir: String, + #[arg(long)] + out: String, + /// 关系轨候选随机抽样上限(写进 sample_pool,Python 侧再抽 50 条) + #[arg(long, default_value_t = 3000)] + cap: usize, +} + +const PATTERNS: &[&str] = &[ + "^内容:", + "原始事件", + "说了什么、做了什么", + "用户感知到", + "用户主动记录了", + "存在某些", +]; + +fn hit_patterns(c: &str) -> Vec<&'static str> { + let mut hits = Vec::new(); + for p in PATTERNS.iter().copied() { + if p.starts_with('^') { + if c.starts_with(&p[1..]) { hits.push(p); } + } else if c.contains(p) { + hits.push(p); + } + } + hits +} + +fn col_str(b: &arrow::record_batch::RecordBatch, r: usize, c: &str) -> String { + b.column_by_name(c) + .and_then(|col| col.as_any().downcast_ref::()) + .map(|a| a.value(r).to_string()) + .unwrap_or_default() +} +fn col_f64(b: &arrow::record_batch::RecordBatch, r: usize, c: &str) -> f64 { + b.column_by_name(c) + .and_then(|col| col.as_any().downcast_ref::()) + .map(|a| a.value(r)) + .unwrap_or(0.0) +} +fn col_i64(b: &arrow::record_batch::RecordBatch, r: usize, c: &str) -> i64 { + b.column_by_name(c) + .and_then(|col| col.as_any().downcast_ref::()) + .map(|a| a.value(r)) + .unwrap_or(0) +} + +fn main() { + let args = Args::parse(); + let rt = Runtime::new().unwrap(); + let db = rt.block_on(lancedb::connect(args.data_dir.as_str()).execute()).unwrap(); + let tbl = rt.block_on(db.open_table("memories").execute()).unwrap(); + // relscan 专用:强制扫描全部现存行(含被删除的)。生产表 11-16 已做大批量遗忘/归档, + // 只扫 is_deleted=false 会把历史空模板全漏掉(本卡基线 58% 是 11-16 上午的实测)。 + // 行级 is_deleted 通过 select 的投影列读出(下方已含)。 + let sel = lancedb::query::Select::columns(&[ + "id","content","category","quality_score","created_at","source","is_deleted", + ]); + // lancedb 0.15 实测:query().select().execute() 的流默认 limit=10(同 query() 默认值, + // 见遗忘候选池修复 t_a4b8fa52),必须显式给大 limit。 + let big = rt.block_on(tbl.count_rows(None)).unwrap() + 10_000; + let mut stream = Box::pin(rt.block_on( + tbl.query().select(sel).limit(big).execute() + ).unwrap()); + eprintln!("[relscan] scan limit={} (count_rows)", big); + let mut sample_pool: Vec = Vec::new(); + let mut n_total = 0usize; + let mut n_distilled = 0usize; + let mut n_empty = 0usize; + let mut n_empty_distilled = 0usize; + let mut n_rel = 0usize; + let mut n_rel_empty = 0usize; + let mut cat_counts: std::collections::HashMap = std::collections::HashMap::new(); + 'outer: while let Some(b) = rt.block_on(stream.next()) { + let b = b.unwrap(); + for i in 0..b.num_rows() { + n_total += 1; + let content = col_str(&b, i, "content"); + let category = col_str(&b, i, "category"); + let is_del = col_i64(&b, i, "is_deleted") != 0; + if is_del { continue; } // 被软删的行不进统计(与遗忘语义一致) + *cat_counts.entry(category.clone()).or_insert(0) += 1; + let is_empty = !hit_patterns(&content).is_empty(); + if is_empty { n_empty += 1; } + let is_distilled = category == "distilled"; + if is_distilled { n_distilled += 1; } + if is_distilled && is_empty { n_empty_distilled += 1; } + let is_rel = content.contains("牧尘") || content.contains("家人") + || content.contains("心情") || content.contains("生气") + || content.contains("失望") || content.contains("牵挂") + || content.contains("偏好") || content.contains("小怡"); + if is_rel { + n_rel += 1; + if is_empty { n_rel_empty += 1; } + sample_pool.push(serde_json::json!({ + "id": col_str(&b, i, "id"), "content": content, + "category": category, "quality_score": col_f64(&b, i, "quality_score"), + "created_at": col_i64(&b, i, "created_at"), + "source": col_str(&b, i, "source"), + "empty": is_empty, "patterns": hit_patterns(&content), + })); + if sample_pool.len() >= args.cap { + break 'outer; + } + } + } + } + let out = serde_json::json!({ + "n_total": n_total, "n_distilled": n_distilled, + "n_empty_all": n_empty, "n_empty_distilled": n_empty_distilled, + "n_rel": n_rel, "n_rel_empty": n_rel_empty, + "cat_counts": cat_counts, + "sample_pool": sample_pool, + }); + let mut f = std::fs::File::create(&args.out).unwrap(); + f.write_all(out.to_string().as_bytes()).unwrap(); + println!("total={} distilled={} empty={} empty_distilled={} rel={} rel_empty={}", + n_total, n_distilled, n_empty, n_empty_distilled, n_rel, n_rel_empty); +} diff --git a/rust/src/bin/tblprobe.rs b/rust/src/bin/tblprobe.rs new file mode 100644 index 0000000..8edbfa5 --- /dev/null +++ b/rust/src/bin/tblprobe.rs @@ -0,0 +1,12 @@ +use lancedb::connect; +fn main() { + let rt = tokio::runtime::Runtime::new().unwrap(); + let db = rt.block_on(connect("/var/lib/memoryweave").execute()).unwrap(); + let names = rt.block_on(db.table_names().execute()).unwrap(); + println!("tables: {names:?}"); + for t in &names { + let tbl = rt.block_on(db.open_table(t).execute()).unwrap(); + let n = rt.block_on(tbl.count_rows(None)).unwrap(); + println!(" {t}: count_rows(None) = {n}"); + } +}