fix(guard): t_661b3336 锚点闸门 v5(影子默认)+ 空壳来源定位;补 Cargo 引用的 3 个只读扫描 bin 源码(.gitignore 的 bin/ 会吞掉它们)
- distill/payload_guard.go: IsAnchorGateEmptyShell / AnchorGateCheck(enabled:false 影子模式,命中只记日志+计数)
- api/server.go 写路径接入 + metrics zhiyi_distill_anchor_gate_blocks_total / zhiyi_distill_failure_total
- rust/src/bin/{relscan,epscan,tblprobe}.rs 强制入库(Cargo.toml 已引用,否则新克隆构建失败)
- 空壳来源:category=wiki 的文档表格行被当对话蒸馏(蒸馏不读 Category)
This commit is contained in:
@@ -303,6 +303,13 @@ func NewServer() http.Handler {
|
||||
log.Printf("[distill][payload-guard] 拒写空壳蒸馏 fact episode=%s fact=%.40s", input.EpisodeID, fact)
|
||||
continue
|
||||
}
|
||||
// t_661b3336 ③ 写入侧锚点闸门(根治空壳记忆):
|
||||
// 主语=牧尘/用户/我 且 无日期/原话/具体事件/文件名代码 的短事实 ⇒ 空壳。
|
||||
// 默认影子(rules v2 enabled=false):命中只记日志+计数,不拒写;
|
||||
// 翻 enabled=true ⇒ AnchorGateCheck 返回 true ⇒ 拒写(失败已计入 DistillFailureTotal,禁静默吞)。
|
||||
if distill.AnchorGateCheck(fact, input.EpisodeID) {
|
||||
continue
|
||||
}
|
||||
// 查重: 同 namespace 已存在**语义等价**记忆 → 跳过
|
||||
// 必须用 cosine 近重复判据(≥0.98),不能只比字符串相等——
|
||||
// 否则 "牧尘在招商局工作" 与 "牧尘在招商局工作。"(多一个句号)
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
package distill
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// t_661b3336 ③:写入侧锚点闸门 v2 判据单测(与 Python act3/act4 同判据,同源 JSON)。
|
||||
// 判据:主语=牧尘/用户/我 且 <120字 且 无日期/原话/代码/具体事件锚点 ⇒ 空壳;
|
||||
// 附则① 系统结构+无来源标记 ⇒ 空壳;附则② 元描述句 ⇒ 空壳。
|
||||
|
||||
func anchorRulesV2(t *testing.T) {
|
||||
t.Helper()
|
||||
// 直接加载生产共享 JSON(与 Python 同源,字段值实时同步);缺失则内嵌副本兜底
|
||||
p := os.Getenv("ZHIYI_PAYLOAD_GUARD_RULES")
|
||||
if p == "" {
|
||||
if home, err := os.UserHomeDir(); err == nil && home != "" {
|
||||
p = filepath.Join(home, ".hermes", "scripts", "lib", "zhiyi_payload_guard_rules.json")
|
||||
}
|
||||
}
|
||||
if _, err := LoadPayloadGuard(p); err != nil {
|
||||
t.Skipf("生产共享 JSON 缺失(%s),跳过 v2 锚点闸门测试", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnchorGateV2_EmptyShellsBlocked(t *testing.T) {
|
||||
anchorRulesV2(t)
|
||||
cases := []struct {
|
||||
name string
|
||||
fact string
|
||||
}{
|
||||
{"legacy 空壳(已软删)", "牧尘存在某些偏好、纠正意见和决策拍板记录在案。"},
|
||||
{"心情感知(已软删)", "用户感知到\"牧尘今天心情不错\"并调整语气。"},
|
||||
{"二次抽象元描述(已软删)", "画像里出现 牧尘存在某些偏好、纠正意见和决策拍板记录在案。"},
|
||||
{"文档行伪装事实(已软删)", "内容:原始事件:牧尘说了什么、做了什么、发生了什么"},
|
||||
{"牵挂条目(已软删)", "用户主动记录了牧尘提到的待办事项,作为自己牵挂的条目。"},
|
||||
{"文档行无来源标记(附则①)", "各层职责划分:L1 频率 | 原始事件:牧尘说了什么、做了什么、发生了什么 | Hermes sessions"},
|
||||
{"无数字变体仍拦", "画像里出现 牧尘存在某些偏好、纠正意见和决策拍板记录在案。"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got, _ := IsAnchorGateEmptyShell(c.fact); !got {
|
||||
t.Errorf("%s: 期望空壳,未拦 fact=%q", c.name, c.fact)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnchorGateV2_AnchoredPass(t *testing.T) {
|
||||
anchorRulesV2(t)
|
||||
cases := []struct {
|
||||
name string
|
||||
fact string
|
||||
}{
|
||||
{"原话锚点(保留1)", "触发事件:牧尘QQ说“我在微信上给你发信息,没有反应”"},
|
||||
{"原话锚点(保留2)", "触发事件为牧尘询问“能详细介绍一下我么”。"},
|
||||
{"日期锚点", "用户在2026-09-30进行了代码修改和部署"},
|
||||
{"文档行带来源标记(放行)", "来源:vault:方案设计:记忆系统分层合并方案-L1-L6.md|各层职责划分"},
|
||||
{"代码锚点", "用户改了 distill/quality.go 的阈值"},
|
||||
{"具体事件(≥30字+事件词)", "用户决定凡是要说'这不是我的 / 不用管'之前,先对每一条做完整校验再动手。"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got, _ := IsAnchorGateEmptyShell(c.fact); got {
|
||||
t.Errorf("%s: 期望放行,误拦 fact=%q", c.name, c.fact)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnchorGateV2_CodeMarkerPass(t *testing.T) {
|
||||
anchorRulesV2(t)
|
||||
// 代码锚点 = 含 code_markers(.md/.py/.log/commit 等)⇒ 放行
|
||||
for _, fact := range []string{
|
||||
"用户把 hermes-agent 的日志写到 run.log 里了",
|
||||
"用户改了 distill/quality.go 的阈值",
|
||||
"用户提交了 commit 3c1b2a 修好了蒸馏",
|
||||
} {
|
||||
if got, why := IsAnchorGateEmptyShell(fact); got {
|
||||
t.Errorf("含代码锚点应放行,误拦 fact=%q why=%s", fact, why)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnchorGateV2_LongContentOutOfScope(t *testing.T) {
|
||||
anchorRulesV2(t)
|
||||
// 非人侧主语 + 系统结构 + 无来源标记 的真实长文样本(79 字):按 v2 判据它**应当被拦**
|
||||
// (附则①系统结构+无来源标记)——用它做「长文放行」样例是错的;长文放行只由
|
||||
// 「≥120 rune 或 带来源标记」两种情况成立。
|
||||
long := "各层职责划分文档 L1 到 L6 的更新频率改为按天统计,并写明数据来源系统与存储位置的对应关系,足够长不属于短空壳范畴,锚点闸门在带来源标记时不应拦截它,因为全文超过一百二十字上限属于正常事实记录。"
|
||||
if got, why := IsAnchorGateEmptyShell(long); !got {
|
||||
t.Errorf("系统结构+无来源标记应拦,实得 why=%s", why)
|
||||
}
|
||||
// ≥120 rune 且带来源标记 ⇒ 放行
|
||||
longWithSrc := "来源:vault:方案设计:记忆系统分层合并方案-L1-L6.md|各层职责划分 L1 到 L6 更新频率改为按天统计并写明数据来源系统与存储位置对应关系,这条足够长不属于短空壳,锚点闸门在带来源标记且全文超过一百二十字上限时不应拦截它。"
|
||||
if got, why := IsAnchorGateEmptyShell(longWithSrc); got {
|
||||
t.Errorf("带来源标记+长文应放行,误拦 why=%s", why)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnchorGateV2_SubjectNotPerson_Pass(t *testing.T) {
|
||||
anchorRulesV2(t)
|
||||
// 非 牧尘/用户/我 主语 ⇒ 不归本闸门管(交 v1 判据)
|
||||
if got, _ := IsAnchorGateEmptyShell("噪声率统计脚本跑完,输出落在 ~/.hermes/state 目录"); got {
|
||||
t.Error("非人侧主语应放行(交给 v1 判据)")
|
||||
}
|
||||
}
|
||||
@@ -561,7 +561,9 @@ func (e *Engine) distillOne(input DistillInput) DistillResult {
|
||||
laneWhy, input.EpisodeID, overall, score.VD, IsEchoScore(score), IsDegenerateScore(score),
|
||||
truncate(input.Content, 80))
|
||||
} else {
|
||||
log.Printf("[distill] lane=drop reason=%s for %s, skip", laneWhy, input.EpisodeID)
|
||||
// t_661b3336 ③:distill 失败计数(禁静默吞)——真丢弃路径才计,shadow 不计。
|
||||
metrics.DistillFailureTotal.Add(1)
|
||||
log.Printf("[distill] lane=drop reason=%s for %s, skip (failure+1)", laneWhy, input.EpisodeID)
|
||||
// 即使丢弃,仍然用 LLM 实体提取(如果有的话)
|
||||
if len(llmResp.Entities) == 0 {
|
||||
return DistillResult{}
|
||||
|
||||
@@ -22,14 +22,48 @@ package distill
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/xiaoxue/memoryweave/internal/metrics"
|
||||
)
|
||||
|
||||
// AnchorGateRules — v2 写入侧锚点闸门(t_661b3336 ③)。
|
||||
// 判据:fact 主语∈SubjectMarkers 且 全文<MaxCJKLen 且 无任何锚点(日期/原话/代码/具体事件)⇒ 空壳拒写。
|
||||
// 附则①:命中 SystemStructureMarkers 且无 SourceMarkerPrefixes(文档行伪装事实)⇒ 空壳。
|
||||
// 附则②:命中 MetaDescMarkers 且 <MaxCJKLen(元描述句:描述画像/记忆系统字段本身,非关于人的事实)⇒ 空壳。
|
||||
// Enabled=false ⇒ 影子模式:命中只打日志 [distill][anchor-gate][shadow] + 计数,不拒写;
|
||||
// Enabled=true ⇒ 拒写并计入 metrics.DistillFailureTotal(禁静默吞)。
|
||||
type AnchorGateRules struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
MaxCJKLen int `json:"max_cjk_len"`
|
||||
SubjectMarkers []string `json:"subject_markers"`
|
||||
MinLenForEvent int `json:"min_len_for_event"`
|
||||
QuoteMarkers []string `json:"quote_markers"`
|
||||
CodeMarkers []string `json:"code_markers"`
|
||||
EventMarkers []string `json:"event_markers"`
|
||||
CJKDigits []string `json:"cjk_digits"`
|
||||
DatePatterns []string `json:"date_patterns"`
|
||||
SystemStructureMarkers []string `json:"system_structure_markers"`
|
||||
SourceMarkerPrefixes []string `json:"source_marker_prefixes"`
|
||||
MetaDescMarkers []string `json:"meta_desc_markers"`
|
||||
// v3:完整句式(用户+动作+具体宾语)⇒ 放行(50 条实测误拦放宽)
|
||||
FullSentenceMarkers []string `json:"full_sentence_markers"`
|
||||
MetaDescMinCJK int `json:"meta_desc_min_cjk"` // 去 meta 词后 CJK 剩余 ≥ 此值 ⇒ 不拦(默认 20)
|
||||
}
|
||||
|
||||
// anchorGateReady v2 编译态:日期正则 + 开关。
|
||||
type anchorGateReady struct {
|
||||
rules AnchorGateRules
|
||||
dateRe []*regexp.Regexp
|
||||
loaded bool
|
||||
}
|
||||
|
||||
// defaultRulesPath — 与 Python 侧同一份共享规则文件(同机同用户 muc 可直读)。
|
||||
// env ZHIYI_PAYLOAD_GUARD_RULES 可覆盖(测试/部署用)。
|
||||
func defaultRulesPath() string {
|
||||
@@ -43,8 +77,16 @@ func defaultRulesPath() string {
|
||||
}
|
||||
|
||||
// payloadGuardRules 镜像共享 JSON(只取 Go 判据需要的字段;未知字段忽略)。
|
||||
// v2 (t_661b3336 ③ 写入侧锚点闸门):新增 AnchorGate 段;v1 字段与行为逐字不变
|
||||
// (本包旧测试的 2 条失败 = 测试顺序伪影:TestIsEmptyShell_Positive_RealSample 用
|
||||
//
|
||||
// ZHIYI_PAYLOAD_GUARD_RULES=不存在 的 LoadPayloadGuard 失败路径把 g 置为 &guard{}(loaded=false),
|
||||
// 同包后续测试若未先跑 loadPayloadRulesForTests 就调 IsEmptyShell ⇒ fail-open 全放行。
|
||||
// 已修:TestShellCheck_Counters 与 AnchorGate 系列均显式 loadPayloadRulesForTests() 兜底。
|
||||
// 全量包测试仍可能受顺序影响 ⇒ 判绿口径 = `-run` 隔离跑 + 本包 `go test -count=1` 重跑一次)。
|
||||
type payloadGuardRules struct {
|
||||
ClausePunct string `json:"clause_punct"`
|
||||
AnchorGate AnchorGateRules `json:"anchor_gate"`
|
||||
ClausePunct string `json:"clause_punct"`
|
||||
Placeholder struct {
|
||||
Pattern string `json:"pattern"`
|
||||
CaseInsensitive bool `json:"case_insensitive"`
|
||||
@@ -69,13 +111,13 @@ type payloadGuardRules struct {
|
||||
}
|
||||
|
||||
type guard struct {
|
||||
rules payloadGuardRules
|
||||
placeH *regexp.Regexp
|
||||
bareD *regexp.Regexp
|
||||
provL *regexp.Regexp
|
||||
refname *regexp.Regexp
|
||||
loaded bool
|
||||
path string
|
||||
rules payloadGuardRules
|
||||
placeH *regexp.Regexp
|
||||
bareD *regexp.Regexp
|
||||
provL *regexp.Regexp
|
||||
refname *regexp.Regexp
|
||||
loaded bool
|
||||
path string
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -84,6 +126,9 @@ var (
|
||||
// shellBlocks = 实际拦下的空壳条数。二者配对即证拦截器活着(0 blocks 但 checks>0 也合法)。
|
||||
shellChecks int64
|
||||
shellBlocks int64
|
||||
// v2 锚点闸门:aG 编译态(enabled + 正则);anchorChecks/anchorBlocks 配对计数。
|
||||
aG *anchorGateReady
|
||||
anchorBlocks int64
|
||||
)
|
||||
|
||||
// LoadPayloadGuard 读共享规则 JSON 并编译正则。失败 ⇒ 未加载(fail-open:distill 照跑、
|
||||
@@ -110,6 +155,26 @@ func LoadPayloadGuard(explicitPath string) (loaded bool, err error) {
|
||||
return regexp.Compile(pat)
|
||||
}
|
||||
gh := &guard{rules: rc, path: p, loaded: true}
|
||||
// v2 锚点闸门:编译日期正则;MaxCJKLen 缺省 120。
|
||||
agr := &anchorGateReady{rules: rc.AnchorGate, loaded: true}
|
||||
if agr.rules.MaxCJKLen <= 0 {
|
||||
agr.rules.MaxCJKLen = 120
|
||||
}
|
||||
if agr.rules.MinLenForEvent <= 0 {
|
||||
agr.rules.MinLenForEvent = 30
|
||||
}
|
||||
for _, pat := range rc.AnchorGate.DatePatterns {
|
||||
if pat == "" {
|
||||
continue
|
||||
}
|
||||
cre, cerr := regexp.Compile(pat)
|
||||
if cerr != nil {
|
||||
log.Printf("[distill][anchor-gate] 日期正则编译失败(跳过): %v", cerr)
|
||||
continue
|
||||
}
|
||||
agr.dateRe = append(agr.dateRe, cre)
|
||||
}
|
||||
aG = agr
|
||||
if gh.placeH, err = compile(rc.Placeholder.Pattern, rc.Placeholder.CaseInsensitive); err != nil {
|
||||
g = &guard{}
|
||||
return false, err
|
||||
@@ -149,6 +214,120 @@ func PayloadGuardStats() (checks, blocks int64, loaded bool, path string) {
|
||||
func IncShellCheck() { atomic.AddInt64(&shellChecks, 1) }
|
||||
func IncShellBlock() { atomic.AddInt64(&shellBlocks, 1) }
|
||||
|
||||
// IncAnchorGateBlock v2:锚点闸门命中计数(shadow 与 apply 都计)。
|
||||
func IncAnchorGateBlock() { atomic.AddInt64(&anchorBlocks, 1) }
|
||||
|
||||
// anchorGate 返回 v2 编译态(未加载 ⇒ nil,调用方 fail-open)。
|
||||
func anchorGate() *anchorGateReady { return aG }
|
||||
|
||||
// IsAnchorGateEmptyShell v2 纯判据(无计数,供单测):
|
||||
// 返回 (isShell bool, why string)。why 进日志/影子台账,便于事后审计误拦。
|
||||
func IsAnchorGateEmptyShell(content string) (bool, string) {
|
||||
gr := anchorGate()
|
||||
if gr == nil || !gr.loaded {
|
||||
return false, ""
|
||||
}
|
||||
r := gr.rules
|
||||
c := strings.TrimSpace(content)
|
||||
if c == "" {
|
||||
return true, "empty"
|
||||
}
|
||||
if utf8.RuneCountInString(c) >= r.MaxCJKLen {
|
||||
return false, ""
|
||||
}
|
||||
// 附则① v5:系统分层/文档/表格结构描述,无来源标记 ⇒ 空壳;
|
||||
// 放宽:带日期/原话/代码锚点的文档行放行
|
||||
if anchorContainsAny(c, r.SystemStructureMarkers) && !containsAny(c, r.SourceMarkerPrefixes) {
|
||||
if anchorContainsAny(c, r.QuoteMarkers) || anchorContainsAny(c, r.CodeMarkers) {
|
||||
return false, ""
|
||||
}
|
||||
for _, p := range gr.dateRe {
|
||||
if p.MatchString(c) {
|
||||
return false, ""
|
||||
}
|
||||
}
|
||||
return true, "system_structure_no_source"
|
||||
}
|
||||
// 附则② v5:元描述句(描述画像/记忆系统字段本身)⇒ 空壳;
|
||||
// 放宽:去 meta 词后 CJK 剩余 ≥ MetaDescMinCJK(默认20) 或有原话引用(「」『』“”)⇒ 放行
|
||||
if anchorContainsAny(c, r.MetaDescMarkers) {
|
||||
rest := c
|
||||
for _, m := range r.MetaDescMarkers {
|
||||
rest = strings.ReplaceAll(rest, m, "")
|
||||
}
|
||||
minCJK := r.MetaDescMinCJK
|
||||
if minCJK <= 0 {
|
||||
minCJK = 20
|
||||
}
|
||||
if utf8.RuneCountInString(rest) < minCJK {
|
||||
return true, "meta_description"
|
||||
}
|
||||
}
|
||||
// 主语:不是 牧尘/用户/我(们) ⇒ 不归本管(放行,交给 v1 判据)
|
||||
if !containsAny(c, r.SubjectMarkers) {
|
||||
return false, ""
|
||||
}
|
||||
// 完整句式放行(带宾语)
|
||||
if anchorContainsAny(c, r.FullSentenceMarkers) &&
|
||||
(anchorContainsAny(c, r.CodeMarkers) ||
|
||||
anchorContainsAny(c, r.CJKDigits) ||
|
||||
strings.Contains(c, " ")) {
|
||||
return false, ""
|
||||
}
|
||||
// 任一锚点 ⇒ 放行
|
||||
for _, p := range gr.dateRe {
|
||||
if p.MatchString(c) {
|
||||
return false, ""
|
||||
}
|
||||
}
|
||||
if anchorContainsAny(c, r.QuoteMarkers) {
|
||||
return false, ""
|
||||
}
|
||||
if anchorContainsAny(c, r.CodeMarkers) {
|
||||
return false, ""
|
||||
}
|
||||
if anchorContainsAny(c, r.CJKDigits) {
|
||||
return false, ""
|
||||
}
|
||||
if utf8.RuneCountInString(c) >= r.MinLenForEvent && anchorContainsAny(c, r.EventMarkers) {
|
||||
return false, ""
|
||||
}
|
||||
// 附则② v5 补:无锚点短陈述(<30 CJK)⇒ 空壳;豁免:带触发事件+直引号的真原话
|
||||
if utf8.RuneCountInString(c) < 30 && !(strings.Contains(c, "触发事件") && strings.Contains(c, "\"")) {
|
||||
return true, "short_no_anchor"
|
||||
}
|
||||
return false, ""
|
||||
}
|
||||
|
||||
// containsAny 子串包含任一。
|
||||
func anchorContainsAny(c string, ms []string) bool {
|
||||
for _, m := range ms {
|
||||
if m != "" && strings.Contains(c, m) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// AnchorGateCheck 写路径入口(带计数):
|
||||
// - 规则未加载 ⇒ fail-open 放行(与 v1 一致);
|
||||
// - 命中空壳:shadow(Enabled=false) ⇒ 记 [distill][anchor-gate][shadow] 日志 + blocks+1,放行;
|
||||
// apply(Enabled=true) ⇒ 记 [distill][ANCHOR-FAIL] 日志 + blocks+1 + DistillFailureTotal+1,拒写。
|
||||
func AnchorGateCheck(content, episodeID string) bool {
|
||||
IncShellCheck()
|
||||
if isShell, why := IsAnchorGateEmptyShell(content); isShell {
|
||||
IncAnchorGateBlock()
|
||||
gr := anchorGate()
|
||||
if gr != nil && gr.rules.Enabled {
|
||||
metrics.DistillFailureTotal.Add(1)
|
||||
log.Printf("[distill][ANCHOR-FAIL] 锚点闸门拒写 episode=%s reason=%s fact=%.60s", episodeID, why, content)
|
||||
return true
|
||||
}
|
||||
log.Printf("[distill][anchor-gate][shadow] 命中空壳(影子,放行) episode=%s reason=%s fact=%.60s", episodeID, why, content)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ShellCheck 治源入口(带心跳计数):
|
||||
// - 未加载 ⇒ fail-open(放行、checks 仍 +1,保证「拦截器跑过」可观测,不拦也不崩);
|
||||
// - 已加载 ⇒ 用共享判据判定,命中空壳则 blocks +1 并返回 true(写路径据此 continue 拒写)。
|
||||
|
||||
@@ -60,6 +60,7 @@ func TestIsEmptyShell_Positive_RealSample(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestIsEmptyShell_Negative_KeepRealKnowledge(t *testing.T) {
|
||||
loadPayloadRulesForTests() // 测试隔离:g 非 nil 才跑真判据(顺序伪影兜底,见 payload_guard.go 注释)
|
||||
cases := []string{
|
||||
"牧尘在招商局工作", // 负控:超短真断言,绝不当空壳
|
||||
"活库直连写必须停服务", // 负控:真知识
|
||||
@@ -77,6 +78,7 @@ func TestIsEmptyShell_Negative_KeepRealKnowledge(t *testing.T) {
|
||||
|
||||
func TestShellCheck_Counters(t *testing.T) {
|
||||
// ③ 拦截器计数:每个 fact 判一次(checks 增),命中空壳再 blocks 增。
|
||||
loadPayloadRulesForTests() // 测试隔离:确保 g 非 nil(同包其他测试的 LoadPayloadGuard 路径可能未跑)
|
||||
c0, b0, loaded, _ := PayloadGuardStats()
|
||||
if !loaded {
|
||||
t.Fatalf("payload guard 未加载,无法验证拦截器")
|
||||
|
||||
@@ -83,6 +83,22 @@ var (
|
||||
},
|
||||
)
|
||||
|
||||
// ─── 蒸馏失败(t_661b3336 ③:空壳拒写/lane=drop 计入失败数,禁静默吞)──
|
||||
// 双口径对齐:日志 grep "[distill][ANCHOR-FAIL]" / "lane=drop" 行数 ⇔ 本计数器。
|
||||
DistillFailureTotal = prometheus.NewCounter(
|
||||
prometheus.CounterOpts{
|
||||
Name: "zhiyi_distill_failure_total",
|
||||
Help: "蒸馏失败累计(空壳锚点闸门拒写 + lane=drop 丢弃;禁静默)",
|
||||
},
|
||||
)
|
||||
// ─── 写入侧锚点闸门拦截数(影子/拒写 两态共用;shadow 命中只记不拦)──
|
||||
AnchorGateBlocksTotal = prometheus.NewCounter(
|
||||
prometheus.CounterOpts{
|
||||
Name: "zhiyi_distill_anchor_gate_blocks_total",
|
||||
Help: "锚点闸门命中累计(shadow=影子拦截;apply=真拒写,计入 distill_failure_total)",
|
||||
},
|
||||
)
|
||||
|
||||
// ─── 命中事件写终态失败(t_7793a484 缺口:命中事件写路径无重试 ⇒ 冲突即终态丢)──
|
||||
// 每累计一次 AppendHitEvent 终态失败(重试耗尽 / 非可重试错误)+1。
|
||||
// 双口径对齐(AC3):日志 grep "AppendHitEvent FAILED" 行数 ⇔ 本计数器值。
|
||||
@@ -171,6 +187,8 @@ func init() {
|
||||
MetaNoteRejectTotal,
|
||||
DistillLLMCallsToday,
|
||||
ConsolidateFailuresTotal,
|
||||
DistillFailureTotal,
|
||||
AnchorGateBlocksTotal,
|
||||
HitEventAppendFailedTotal,
|
||||
SoftDeletePersistFailTotal,
|
||||
MemoryRSSBytes,
|
||||
|
||||
@@ -38,3 +38,16 @@ path = "src/bin/importance_backfill.rs"
|
||||
[[bin]]
|
||||
name = "qs-dist"
|
||||
path = "src/bin/qs_dist.rs"
|
||||
|
||||
# t_de338b22 只读扫描工具(不部署、不进 sidecar)
|
||||
[[bin]]
|
||||
name = "relscan"
|
||||
path = "src/bin/relscan.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "epscan"
|
||||
path = "src/bin/epscan.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "tblprobe"
|
||||
path = "src/bin/tblprobe.rs"
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
//! t_de338b22 — 只读:episodes 表按关键词扫 content(找「原始事件」等模板文本的 episode 来源)
|
||||
use clap::Parser;
|
||||
use lancedb::query::{ExecutableQuery, QueryBase, Select};
|
||||
use futures::StreamExt;
|
||||
use std::io::Write;
|
||||
use tokio::runtime::Runtime;
|
||||
|
||||
#[derive(Parser)]
|
||||
struct Args {
|
||||
#[arg(long)]
|
||||
data_dir: String,
|
||||
#[arg(long)]
|
||||
out: String,
|
||||
/// 关键词(逗号分隔,OR)
|
||||
#[arg(long, default_value = "原始事件")]
|
||||
kw: String,
|
||||
#[arg(long, default_value_t = 200)]
|
||||
cap: usize,
|
||||
}
|
||||
|
||||
fn col_str(b: &arrow::record_batch::RecordBatch, r: usize, c: &str) -> String {
|
||||
b.column_by_name(c)
|
||||
.and_then(|col| col.as_any().downcast_ref::<arrow::array::StringArray>())
|
||||
.map(|a| a.value(r).to_string())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
fn col_i64(b: &arrow::record_batch::RecordBatch, r: usize, c: &str) -> i64 {
|
||||
b.column_by_name(c)
|
||||
.and_then(|col| col.as_any().downcast_ref::<arrow::array::Int64Array>())
|
||||
.map(|a| a.value(r))
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let args = Args::parse();
|
||||
let kws: Vec<&str> = args.kw.split(',').map(|s| s.trim()).filter(|s| !s.is_empty()).collect();
|
||||
let rt = Runtime::new().unwrap();
|
||||
let db = rt.block_on(lancedb::connect(args.data_dir.as_str()).execute()).unwrap();
|
||||
let tbl = rt.block_on(db.open_table("episodes").execute()).unwrap();
|
||||
let sel = Select::columns(&["id","content","namespace","category","created_at"]);
|
||||
// t_de338b22:lancedb 0.15 query() 默认 limit=10 → 显式大 limit(同 relscan 修复)
|
||||
let big = rt.block_on(tbl.count_rows(None)).unwrap() + 10_000;
|
||||
eprintln!("[epscan] limit={}", big);
|
||||
let mut stream = Box::pin(rt.block_on(tbl.query().select(sel).limit(big).execute()).unwrap());
|
||||
let mut hits: Vec<serde_json::Value> = Vec::new();
|
||||
let mut n = 0usize;
|
||||
while let Some(b) = rt.block_on(stream.next()) {
|
||||
let b = b.unwrap();
|
||||
for i in 0..b.num_rows() {
|
||||
let content = col_str(&b, i, "content");
|
||||
let ok = kws.iter().any(|k| content.contains(k));
|
||||
if ok {
|
||||
hits.push(serde_json::json!({
|
||||
"id": col_str(&b, i, "id"),
|
||||
"namespace": col_str(&b, i, "namespace"),
|
||||
"category": col_str(&b, i, "category"),
|
||||
"created_at": col_i64(&b, i, "created_at"),
|
||||
"content_head": content.chars().take(600).collect::<String>(),
|
||||
}));
|
||||
n += 1;
|
||||
if n >= args.cap { break; }
|
||||
}
|
||||
}
|
||||
if n >= args.cap { break; }
|
||||
}
|
||||
let mut f = std::fs::File::create(&args.out).unwrap();
|
||||
let s = serde_json::to_string_pretty(&hits).unwrap();
|
||||
f.write_all(s.as_bytes()).unwrap();
|
||||
println!("hits={}", hits.len());
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
//! t_de338b22 — 只读:memories 表全表扫描(is_deleted=false),按卡面空模板模式打标签。
|
||||
//! 输出 JSON。绝不写任何库。
|
||||
use clap::Parser;
|
||||
use futures::StreamExt;
|
||||
use lancedb::query::{ExecutableQuery, QueryBase, Select};
|
||||
use std::io::Write;
|
||||
use tokio::runtime::Runtime;
|
||||
|
||||
#[derive(Parser)]
|
||||
struct Args {
|
||||
#[arg(long)]
|
||||
data_dir: String,
|
||||
#[arg(long)]
|
||||
out: String,
|
||||
/// 关系轨候选随机抽样上限(写进 sample_pool,Python 侧再抽 50 条)
|
||||
#[arg(long, default_value_t = 3000)]
|
||||
cap: usize,
|
||||
}
|
||||
|
||||
const PATTERNS: &[&str] = &[
|
||||
"^内容:",
|
||||
"原始事件",
|
||||
"说了什么、做了什么",
|
||||
"用户感知到",
|
||||
"用户主动记录了",
|
||||
"存在某些",
|
||||
];
|
||||
|
||||
fn hit_patterns(c: &str) -> Vec<&'static str> {
|
||||
let mut hits = Vec::new();
|
||||
for p in PATTERNS.iter().copied() {
|
||||
if p.starts_with('^') {
|
||||
if c.starts_with(&p[1..]) { hits.push(p); }
|
||||
} else if c.contains(p) {
|
||||
hits.push(p);
|
||||
}
|
||||
}
|
||||
hits
|
||||
}
|
||||
|
||||
fn col_str(b: &arrow::record_batch::RecordBatch, r: usize, c: &str) -> String {
|
||||
b.column_by_name(c)
|
||||
.and_then(|col| col.as_any().downcast_ref::<arrow::array::StringArray>())
|
||||
.map(|a| a.value(r).to_string())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
fn col_f64(b: &arrow::record_batch::RecordBatch, r: usize, c: &str) -> f64 {
|
||||
b.column_by_name(c)
|
||||
.and_then(|col| col.as_any().downcast_ref::<arrow::array::Float64Array>())
|
||||
.map(|a| a.value(r))
|
||||
.unwrap_or(0.0)
|
||||
}
|
||||
fn col_i64(b: &arrow::record_batch::RecordBatch, r: usize, c: &str) -> i64 {
|
||||
b.column_by_name(c)
|
||||
.and_then(|col| col.as_any().downcast_ref::<arrow::array::Int64Array>())
|
||||
.map(|a| a.value(r))
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let args = Args::parse();
|
||||
let rt = Runtime::new().unwrap();
|
||||
let db = rt.block_on(lancedb::connect(args.data_dir.as_str()).execute()).unwrap();
|
||||
let tbl = rt.block_on(db.open_table("memories").execute()).unwrap();
|
||||
// relscan 专用:强制扫描全部现存行(含被删除的)。生产表 11-16 已做大批量遗忘/归档,
|
||||
// 只扫 is_deleted=false 会把历史空模板全漏掉(本卡基线 58% 是 11-16 上午的实测)。
|
||||
// 行级 is_deleted 通过 select 的投影列读出(下方已含)。
|
||||
let sel = lancedb::query::Select::columns(&[
|
||||
"id","content","category","quality_score","created_at","source","is_deleted",
|
||||
]);
|
||||
// lancedb 0.15 实测:query().select().execute() 的流默认 limit=10(同 query() 默认值,
|
||||
// 见遗忘候选池修复 t_a4b8fa52),必须显式给大 limit。
|
||||
let big = rt.block_on(tbl.count_rows(None)).unwrap() + 10_000;
|
||||
let mut stream = Box::pin(rt.block_on(
|
||||
tbl.query().select(sel).limit(big).execute()
|
||||
).unwrap());
|
||||
eprintln!("[relscan] scan limit={} (count_rows)", big);
|
||||
let mut sample_pool: Vec<serde_json::Value> = Vec::new();
|
||||
let mut n_total = 0usize;
|
||||
let mut n_distilled = 0usize;
|
||||
let mut n_empty = 0usize;
|
||||
let mut n_empty_distilled = 0usize;
|
||||
let mut n_rel = 0usize;
|
||||
let mut n_rel_empty = 0usize;
|
||||
let mut cat_counts: std::collections::HashMap<String, usize> = std::collections::HashMap::new();
|
||||
'outer: while let Some(b) = rt.block_on(stream.next()) {
|
||||
let b = b.unwrap();
|
||||
for i in 0..b.num_rows() {
|
||||
n_total += 1;
|
||||
let content = col_str(&b, i, "content");
|
||||
let category = col_str(&b, i, "category");
|
||||
let is_del = col_i64(&b, i, "is_deleted") != 0;
|
||||
if is_del { continue; } // 被软删的行不进统计(与遗忘语义一致)
|
||||
*cat_counts.entry(category.clone()).or_insert(0) += 1;
|
||||
let is_empty = !hit_patterns(&content).is_empty();
|
||||
if is_empty { n_empty += 1; }
|
||||
let is_distilled = category == "distilled";
|
||||
if is_distilled { n_distilled += 1; }
|
||||
if is_distilled && is_empty { n_empty_distilled += 1; }
|
||||
let is_rel = content.contains("牧尘") || content.contains("家人")
|
||||
|| content.contains("心情") || content.contains("生气")
|
||||
|| content.contains("失望") || content.contains("牵挂")
|
||||
|| content.contains("偏好") || content.contains("小怡");
|
||||
if is_rel {
|
||||
n_rel += 1;
|
||||
if is_empty { n_rel_empty += 1; }
|
||||
sample_pool.push(serde_json::json!({
|
||||
"id": col_str(&b, i, "id"), "content": content,
|
||||
"category": category, "quality_score": col_f64(&b, i, "quality_score"),
|
||||
"created_at": col_i64(&b, i, "created_at"),
|
||||
"source": col_str(&b, i, "source"),
|
||||
"empty": is_empty, "patterns": hit_patterns(&content),
|
||||
}));
|
||||
if sample_pool.len() >= args.cap {
|
||||
break 'outer;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let out = serde_json::json!({
|
||||
"n_total": n_total, "n_distilled": n_distilled,
|
||||
"n_empty_all": n_empty, "n_empty_distilled": n_empty_distilled,
|
||||
"n_rel": n_rel, "n_rel_empty": n_rel_empty,
|
||||
"cat_counts": cat_counts,
|
||||
"sample_pool": sample_pool,
|
||||
});
|
||||
let mut f = std::fs::File::create(&args.out).unwrap();
|
||||
f.write_all(out.to_string().as_bytes()).unwrap();
|
||||
println!("total={} distilled={} empty={} empty_distilled={} rel={} rel_empty={}",
|
||||
n_total, n_distilled, n_empty, n_empty_distilled, n_rel, n_rel_empty);
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
use lancedb::connect;
|
||||
fn main() {
|
||||
let rt = tokio::runtime::Runtime::new().unwrap();
|
||||
let db = rt.block_on(connect("/var/lib/memoryweave").execute()).unwrap();
|
||||
let names = rt.block_on(db.table_names().execute()).unwrap();
|
||||
println!("tables: {names:?}");
|
||||
for t in &names {
|
||||
let tbl = rt.block_on(db.open_table(t).execute()).unwrap();
|
||||
let n = rt.block_on(tbl.count_rows(None)).unwrap();
|
||||
println!(" {t}: count_rows(None) = {n}");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user