- 审计误读订正:deprecated_per_day 是计数器(无分母),不给 _samples/_valid 是对的 - 新增 deprecated_total:区分「今天 0 条」与「从未统计过」(0 的二义性) - executor t8:原 Condition:0.1 从不生效(判据硬编码 >5)→ Condition:5.0 + 读 t.Condition,行为不变 - 单测钉住:RecordDeprecation 双计数、t8 阈值真的可配、计数器不带率键 - docs/metrics-semantics.md:counter/rate/mean 三种读法 + 0 的二义性铁律
1047 lines
34 KiB
Go
1047 lines
34 KiB
Go
// 织忆 MemoryWeave — 自优化引擎
|
||
package selfoptimize
|
||
|
||
import (
|
||
"encoding/json"
|
||
"math"
|
||
"os"
|
||
"strconv"
|
||
"sync"
|
||
"sync/atomic"
|
||
"time"
|
||
|
||
"github.com/xiaoxue/memoryweave/internal/storage"
|
||
)
|
||
|
||
// Redis persistence key
|
||
const dashboardRedisKey = "zhiyi:dashboard"
|
||
|
||
// dashboardSchemaV2 缺口计数器版本标记(t_e5997ca4,三选一改 (b):计数器版本化重置)。
|
||
//
|
||
// 依据:历史 ClosedGaps/TotalGaps 被事件式自增污染(线上实测 closed_gaps=9 vs
|
||
// total_gaps=2,Redis HASH zhiyi:dashboard),且无法对账还原「哪 9 次关闭对应哪 2 个
|
||
// 缺口」——(a) min() 对账是伪造数据,(c) 等 TotalGaps 追上的时长不可估且期间指标恒 -1。
|
||
// 做法:首次加载见到 marker 缺失/非 v2 ⇒ 一次性把 ClosedGaps/TotalGaps 重置为 0 并回写
|
||
// marker;此后加载保真(v2 起按「缺口个数」口径从头累计)。
|
||
// 代价(写明):丢失 gap 双计数器的历史累计值——其本身就是不可信数据(虚增产物),
|
||
// 不掩盖。其他计数器(useful/cascade/auto/distill)口径无事件污染,一律不动。
|
||
const dashboardSchemaV2 = "dashboard_schema_v2"
|
||
|
||
// dashboardLocalPath 本地 JSON 落盘点 —— Redis 不可用时的兜底(2026-09-27 加)
|
||
//
|
||
// 背景:本机无 Redis(无 redis-cli/redis-server、6379 无监听、单元无 REDIS_ADDR/URL)
|
||
// ⇒ storage.GetRedisClient() 返回 nil ⇒ redisPersistence 恒为 nil
|
||
// ⇒ 原 persist() 第一行就 return:7 项指标只活在进程内,**重启即归零**,
|
||
//
|
||
// 而 tuning / executor / distributed 都在读它做自优化决策。
|
||
//
|
||
// 可用 ZHIYI_DASHBOARD_PATH 覆盖(测试用)。
|
||
func dashboardLocalPath() string {
|
||
if p := os.Getenv("ZHIYI_DASHBOARD_PATH"); p != "" {
|
||
return p
|
||
}
|
||
return "/var/lib/memoryweave/dashboard.json"
|
||
}
|
||
|
||
// dashboardState 可序列化快照 —— Dashboard 内含 sync.RWMutex,不能直接 Marshal/整块复制
|
||
type dashboardState struct {
|
||
UsefulCount int `json:"useful_count"`
|
||
NotUsefulCount int `json:"not_useful_count"`
|
||
TotalRecalls int `json:"total_recalls"`
|
||
HitCount int `json:"hit_count"`
|
||
ClosedGaps int `json:"closed_gaps"`
|
||
TotalGaps int `json:"total_gaps"`
|
||
CascadeFixedTotal int `json:"cascade_fixed_total"`
|
||
TotalFixes int `json:"total_fixes"`
|
||
DeprecatedToday int `json:"deprecated_today"`
|
||
// DeprecatedTotal 累计淘汰条数(不跨天归零)。
|
||
// 2026-10-26:加它的唯一理由是把「今天 0 条」和「从来没统计过」分开 ——
|
||
// 与 pattern 召回那次的教训同型:0 永远有两种含义,指标必须能自证是哪一种。
|
||
DeprecatedTotal int `json:"deprecated_total"`
|
||
DeprecatedDate string `json:"deprecated_date"` // deprecated_today 的归属日期(跨天归零用)
|
||
DistillLossSum float64 `json:"distill_loss_sum"`
|
||
DistillLossCount int `json:"distill_loss_count"`
|
||
DistillSuccessCount int `json:"distill_success_count"`
|
||
AutoResolvedConflicts int `json:"auto_resolved_conflicts"`
|
||
TotalConflicts int `json:"total_conflicts"`
|
||
// t_e5997ca4:计数器版本标记(见 dashboardSchemaV2 注释)。
|
||
SchemaVersion string `json:"dashboard_schema_version"`
|
||
}
|
||
|
||
func todayStr() string { return time.Now().Format("2006-01-02") }
|
||
|
||
// redisPersistence 是否开启了 Redis 持久化(由 server.go 在启动时设置)
|
||
var redisPersistence *storage.RedisClient
|
||
|
||
// EnableRedisPersistence 启动 Dashboard 持久化(Redis 优先;Redis 不可用则转本地 JSON)
|
||
func EnableRedisPersistence() {
|
||
rc := storage.GetRedisClient()
|
||
if rc != nil {
|
||
redisPersistence = rc
|
||
_ = loadDashboard()
|
||
return
|
||
}
|
||
_ = loadDashboardLocal()
|
||
}
|
||
|
||
// loadDashboardLocal 从本地 JSON 恢复指标;文件不存在/损坏 → 静默保持零值(= 老行为)
|
||
func loadDashboardLocal() error {
|
||
b, err := os.ReadFile(dashboardLocalPath())
|
||
if err != nil {
|
||
return err
|
||
}
|
||
var st dashboardState
|
||
if err := json.Unmarshal(b, &st); err != nil {
|
||
return err
|
||
}
|
||
Dash.mu.Lock()
|
||
defer Dash.mu.Unlock()
|
||
Dash.UsefulCount = st.UsefulCount
|
||
Dash.NotUsefulCount = st.NotUsefulCount
|
||
Dash.TotalRecalls = st.TotalRecalls
|
||
Dash.HitCount = st.HitCount
|
||
Dash.ClosedGaps = st.ClosedGaps
|
||
Dash.TotalGaps = st.TotalGaps
|
||
Dash.CascadeFixedTotal = st.CascadeFixedTotal
|
||
Dash.TotalFixes = st.TotalFixes
|
||
Dash.DeprecatedToday = st.DeprecatedToday
|
||
Dash.DeprecatedTotal = st.DeprecatedTotal
|
||
if st.DeprecatedDate != todayStr() { // "今日"计数跨天归零(进程跨零点不重置,下次加载时纠正)
|
||
Dash.DeprecatedToday = 0
|
||
}
|
||
Dash.DistillLossSum = st.DistillLossSum
|
||
Dash.DistillLossCount = st.DistillLossCount
|
||
Dash.DistillSuccessCount = st.DistillSuccessCount
|
||
Dash.AutoResolvedConflicts = st.AutoResolvedConflicts
|
||
Dash.TotalConflicts = st.TotalConflicts
|
||
// t_e5997ca4 (b):版本化一次性重置。本地 JSON 无 v2 标记(旧版/缺失)⇒ 丢弃被
|
||
// 事件式自增污染的 gap 双计数器,回写标记;v2 标记在位 ⇒ 保真(新口径累计值)。
|
||
if st.SchemaVersion != dashboardSchemaV2 {
|
||
Dash.ClosedGaps = 0
|
||
Dash.TotalGaps = 0
|
||
Dash.persist() // 回写 v2 标记(调用方已持 Dash.mu ✓;Redis 不可用则落本地 JSON)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// persistLocalLocked 原子写本地 JSON(best-effort:任何失败都静默 → 行为退回"仅内存")
|
||
// 注意:调用方必须已持有 d.mu 锁
|
||
func (d *Dashboard) persistLocalLocked() {
|
||
st := dashboardState{
|
||
UsefulCount: d.UsefulCount,
|
||
NotUsefulCount: d.NotUsefulCount,
|
||
TotalRecalls: d.TotalRecalls,
|
||
HitCount: d.HitCount,
|
||
ClosedGaps: d.ClosedGaps,
|
||
TotalGaps: d.TotalGaps,
|
||
CascadeFixedTotal: d.CascadeFixedTotal,
|
||
TotalFixes: d.TotalFixes,
|
||
DeprecatedToday: d.DeprecatedToday,
|
||
DeprecatedTotal: d.DeprecatedTotal,
|
||
DeprecatedDate: todayStr(),
|
||
DistillLossSum: d.DistillLossSum,
|
||
DistillLossCount: d.DistillLossCount,
|
||
DistillSuccessCount: d.DistillSuccessCount,
|
||
AutoResolvedConflicts: d.AutoResolvedConflicts,
|
||
TotalConflicts: d.TotalConflicts,
|
||
SchemaVersion: dashboardSchemaV2, // t_e5997ca4:v2 常驻回写(新安装/旧版升级都自动打标)
|
||
}
|
||
b, err := json.Marshal(st)
|
||
if err != nil {
|
||
return
|
||
}
|
||
p := dashboardLocalPath()
|
||
tmp := p + ".tmp"
|
||
if err := os.WriteFile(tmp, b, 0o600); err != nil {
|
||
return
|
||
}
|
||
_ = os.Rename(tmp, p)
|
||
}
|
||
|
||
// persist 持久化 Dashboard:Redis 可用→写 Redis HASH;否则→写本地 JSON
|
||
// 注意:调用方必须已持有 d.mu 锁(RLock 或 Lock)
|
||
func (d *Dashboard) persist() {
|
||
if redisPersistence == nil {
|
||
d.persistLocalLocked()
|
||
return
|
||
}
|
||
_ = redisPersistence.HSet(dashboardRedisKey, "useful_count", strconv.Itoa(d.UsefulCount))
|
||
_ = redisPersistence.HSet(dashboardRedisKey, "not_useful_count", strconv.Itoa(d.NotUsefulCount))
|
||
_ = redisPersistence.HSet(dashboardRedisKey, "total_recalls", strconv.Itoa(d.TotalRecalls))
|
||
_ = redisPersistence.HSet(dashboardRedisKey, "hit_count", strconv.Itoa(d.HitCount))
|
||
_ = redisPersistence.HSet(dashboardRedisKey, "closed_gaps", strconv.Itoa(d.ClosedGaps))
|
||
_ = redisPersistence.HSet(dashboardRedisKey, "total_gaps", strconv.Itoa(d.TotalGaps))
|
||
_ = redisPersistence.HSet(dashboardRedisKey, "cascade_fixed_total", strconv.Itoa(d.CascadeFixedTotal))
|
||
_ = redisPersistence.HSet(dashboardRedisKey, "total_fixes", strconv.Itoa(d.TotalFixes))
|
||
_ = redisPersistence.HSet(dashboardRedisKey, "deprecated_today", strconv.Itoa(d.DeprecatedToday))
|
||
_ = redisPersistence.HSet(dashboardRedisKey, "distill_loss_sum", strconv.FormatFloat(d.DistillLossSum, 'f', 4, 64))
|
||
_ = redisPersistence.HSet(dashboardRedisKey, "distill_loss_count", strconv.Itoa(d.DistillLossCount))
|
||
_ = redisPersistence.HSet(dashboardRedisKey, "distill_success_count", strconv.Itoa(d.DistillSuccessCount))
|
||
_ = redisPersistence.HSet(dashboardRedisKey, "auto_resolved_conflicts", strconv.Itoa(d.AutoResolvedConflicts))
|
||
_ = redisPersistence.HSet(dashboardRedisKey, "total_conflicts", strconv.Itoa(d.TotalConflicts))
|
||
_ = redisPersistence.HSet(dashboardRedisKey, "dashboard_schema_version", dashboardSchemaV2) // t_e5997ca4:v2 标记常驻回写
|
||
_ = redisPersistence.Expire(dashboardRedisKey, 90*24*time.Hour) // 90 天 TTL
|
||
}
|
||
|
||
// loadDashboard 从 Redis 加载 Dashboard 数据
|
||
func loadDashboard() error {
|
||
if redisPersistence == nil {
|
||
return nil
|
||
}
|
||
data, err := redisPersistence.HGetAll(dashboardRedisKey)
|
||
if err != nil || len(data) == 0 {
|
||
return err
|
||
}
|
||
Dash.mu.Lock()
|
||
defer Dash.mu.Unlock()
|
||
|
||
if v, err := strconv.Atoi(data["useful_count"]); err == nil {
|
||
Dash.UsefulCount = v
|
||
}
|
||
if v, err := strconv.Atoi(data["not_useful_count"]); err == nil {
|
||
Dash.NotUsefulCount = v
|
||
}
|
||
if v, err := strconv.Atoi(data["total_recalls"]); err == nil {
|
||
Dash.TotalRecalls = v
|
||
}
|
||
if v, err := strconv.Atoi(data["hit_count"]); err == nil {
|
||
Dash.HitCount = v
|
||
}
|
||
if v, err := strconv.Atoi(data["closed_gaps"]); err == nil {
|
||
Dash.ClosedGaps = v
|
||
}
|
||
if v, err := strconv.Atoi(data["total_gaps"]); err == nil {
|
||
Dash.TotalGaps = v
|
||
}
|
||
if v, err := strconv.Atoi(data["cascade_fixed_total"]); err == nil {
|
||
Dash.CascadeFixedTotal = v
|
||
}
|
||
if v, err := strconv.Atoi(data["total_fixes"]); err == nil {
|
||
Dash.TotalFixes = v
|
||
}
|
||
if v, err := strconv.Atoi(data["deprecated_today"]); err == nil {
|
||
Dash.DeprecatedToday = v
|
||
}
|
||
if v, err := strconv.ParseFloat(data["distill_loss_sum"], 64); err == nil {
|
||
Dash.DistillLossSum = v
|
||
}
|
||
if v, err := strconv.Atoi(data["distill_loss_count"]); err == nil {
|
||
Dash.DistillLossCount = v
|
||
}
|
||
if v, err := strconv.Atoi(data["distill_success_count"]); err == nil {
|
||
Dash.DistillSuccessCount = v
|
||
}
|
||
if v, err := strconv.Atoi(data["auto_resolved_conflicts"]); err == nil {
|
||
Dash.AutoResolvedConflicts = v
|
||
}
|
||
if v, err := strconv.Atoi(data["total_conflicts"]); err == nil {
|
||
Dash.TotalConflicts = v
|
||
}
|
||
// t_e5997ca4 (b):版本化一次性重置(Redis 路径)。HASH 无 v2 标记(旧版遗留/缺失)⇒
|
||
// 丢弃被事件式自增污染的 gap 双计数器(线上 9 vs 2 实例),回写标记;v2 在位 ⇒ 保真。
|
||
if data["dashboard_schema_version"] != dashboardSchemaV2 {
|
||
Dash.ClosedGaps = 0
|
||
Dash.TotalGaps = 0
|
||
Dash.persist() // 回写 v2 标记(调用方已持 Dash.mu ✓)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// ─── 自优化仪表盘 ────────────────────────────────────────
|
||
|
||
type Dashboard struct {
|
||
mu sync.RWMutex
|
||
UsefulCount int `json:"useful_count"`
|
||
NotUsefulCount int `json:"not_useful_count"`
|
||
TotalRecalls int `json:"total_recalls"`
|
||
HitCount int `json:"hit_count"`
|
||
ClosedGaps int `json:"closed_gaps"`
|
||
TotalGaps int `json:"total_gaps"`
|
||
CascadeFixedTotal int `json:"cascade_fixed_total"`
|
||
TotalFixes int `json:"total_fixes"`
|
||
DeprecatedToday int `json:"deprecated_today"`
|
||
// DeprecatedTotal 累计淘汰条数(不跨天归零)—— 与「今天 0 条」区分开,
|
||
// 因为 0 有两种含义:今天没淘汰 / 从来没统计。指标必须能自证是哪一种。
|
||
DeprecatedTotal int `json:"deprecated_total"`
|
||
DistillLossSum float64 `json:"distill_loss_sum"`
|
||
DistillLossCount int `json:"distill_loss_count"`
|
||
DistillSuccessCount int `json:"distill_success_count"`
|
||
AutoResolvedConflicts int `json:"auto_resolved_conflicts"`
|
||
TotalConflicts int `json:"total_conflicts"`
|
||
}
|
||
|
||
var Dash = &Dashboard{}
|
||
|
||
// metricNA 是「无数据」哨兵值。
|
||
//
|
||
// 2026-10-14(牧尘批准):比率型指标在**分母退化**时(样本数为 0,或负样本通道
|
||
// 恒空)此前返回 1 —— 被下游读成「100% 好用/100% 命中」。那不是满分,是失明:
|
||
// NotUseful=0 只说明「没人说过不好」,不说明「全都好」;
|
||
// HitCount/TotalRecalls 的分子来自 core.go 的 `len(results) > 0`,
|
||
// 量的是「召回有没有返回东西」,不是「召回对不对」。
|
||
//
|
||
// 现在统一返回 -1:真实比率必落在 [0,1],-1 不可能被误读成任何一档成绩。
|
||
// 消费方规则(配套键,同 map 返回):
|
||
//
|
||
// <name>_valid = 1 表示该比率可解读;0 表示值必为 -1、**禁止**当成绩用
|
||
// <name>_samples = 该比率的分母样本数(0 时即退化)
|
||
//
|
||
// 已有消费方均按 key 存在性取值(SyncFromDashboard / 各 JSON 端点),
|
||
// 本改动**只增键、不改键名**,故不破坏任何调用方。
|
||
const metricNA = -1.0
|
||
|
||
// rateOrNA 分母 > 0 才算比率(保留两位),否则返回 metricNA。
|
||
func rateOrNA(num, den int) float64 {
|
||
if den <= 0 {
|
||
return metricNA
|
||
}
|
||
return math.Round(float64(num)/float64(den)*100) / 100
|
||
}
|
||
|
||
// IsNA 判定指标值是否为「无数据」哨兵(metricNA < 0)。
|
||
//
|
||
// ⚠️ 消费者铁律:触发器 / 调优建议 / 面板解读等所有消费点在拿比率值跟阈值
|
||
// 比较之前,必须先排除 N/A。-1 是哨兵(样本不足 或 越界被拦),不是「低分」,
|
||
// 若直接 `v < 阈值` 会恒为 true ⇒ 每轮误触发(本卡 t_c70f385c 事故面:
|
||
// gap_closure_rate=-1 让「降低 gap 阈值」触发器每轮 fire,与旧 recall_hit_rate 恒=1
|
||
// 是同类判定失真、方向相反)。故:v < 0 ⇒ 跳过 / 不触发。
|
||
func IsNA(v float64) bool { return v < 0 }
|
||
|
||
// minRateSamples 是率类指标的最小样本门。
|
||
//
|
||
// 样本太少(< 20)时,即使比率为真值也**统计上不可解读** —— 例如 auto_resolve_rate
|
||
// 曾仅 2 个样本即判 valid=1 满分。现在样本不足一律按退化处理(metricNA + valid=0),
|
||
// 不参与面板解读,避免「小样本满分」误导。
|
||
const minRateSamples = 20
|
||
|
||
// b2f 布尔转 0/1,用于 *_valid 键。
|
||
func b2f(b bool) float64 {
|
||
if b {
|
||
return 1
|
||
}
|
||
return 0
|
||
}
|
||
|
||
// Metrics 返回核心指标。
|
||
//
|
||
// 比率项在分母退化时返回 metricNA(-1) 而非 1;每项比率都配 *_valid / *_samples。
|
||
//
|
||
// 计数项(deprecated_per_day / deprecated_total)**不配** *_samples / *_valid:
|
||
// 它们没有分母,「样本不足」对它们无定义(2026-10-26 更正:旧注释写作「与均值项
|
||
// avg_distill_loss 同规则」是错的 —— 均值项有分母,故有 samples/valid;审计脚本
|
||
// 曾据此对计数器报「0 样本」告警,属探针模型错配,不是指标缺陷)。
|
||
func (d *Dashboard) Metrics() map[string]float64 {
|
||
d.mu.RLock()
|
||
defer d.mu.RUnlock()
|
||
|
||
// 负样本通道为空 ⇒ 无从判断好坏(详见 metricNA 注释)。
|
||
usefulRate := metricNA
|
||
usefulSamples := d.UsefulCount + d.NotUsefulCount
|
||
if d.NotUsefulCount > 0 {
|
||
usefulRate = rateOrNA(d.UsefulCount, usefulSamples)
|
||
}
|
||
|
||
// recall_hit_rate 已于 2026-10-17 退役:其分子原本是 core.go 的 `len(results) > 0`,
|
||
// 量的是「召回有没有返回东西」而非「返回的对不对」,恒为 1、无区分度。
|
||
// 「返回且被使用」的语义已由 recall_usefulness_rate 承担(198 样本 0.98)。
|
||
// 禁止保留恒等于 1 的指标挂在面板上误导人 ⇒ 从 Metrics() 移除该键。
|
||
gapRate := rateOrNA(d.ClosedGaps, d.TotalGaps)
|
||
cascadeRate := rateOrNA(d.CascadeFixedTotal, d.TotalFixes)
|
||
autoRate := rateOrNA(d.AutoResolvedConflicts, d.TotalConflicts)
|
||
|
||
// gap_closure_rate 断言:ClosedGaps/TotalGaps 统一为「缺口个数」口径后,比率必落 [0,1]。
|
||
// 若写入侧再次把「关闭事件数」混进来导致越界,断言拦截 → -1 + valid=0(禁止越界值上板)。
|
||
if gapRate > 1 {
|
||
gapRate = metricNA
|
||
}
|
||
|
||
// 均值项:零样本的均值同样是退化,按同一规则返回 metricNA。
|
||
avgLoss := metricNA
|
||
if d.DistillLossCount > 0 {
|
||
avgLoss = math.Round(d.DistillLossSum/float64(d.DistillLossCount)*100) / 100
|
||
}
|
||
|
||
// 最小样本门:samples < minRateSamples ⇒ 值 = -1 且 valid = 0,不参与面板解读。
|
||
usefulVal, usefulValid := usefulRate, b2f(usefulSamples >= minRateSamples && d.NotUsefulCount > 0)
|
||
gapVal, gapValid := gapRate, b2f(d.TotalGaps >= minRateSamples)
|
||
cascadeVal, cascadeValid := cascadeRate, b2f(d.TotalFixes >= minRateSamples)
|
||
autoVal, autoValid := autoRate, b2f(d.TotalConflicts >= minRateSamples)
|
||
lossVal, lossValid := avgLoss, b2f(d.DistillLossCount >= minRateSamples)
|
||
if usefulSamples < minRateSamples || d.NotUsefulCount == 0 {
|
||
usefulVal = metricNA
|
||
}
|
||
if d.TotalGaps < minRateSamples || gapVal == metricNA {
|
||
// 样本不足 或 越界断言触发 ⇒ 值 -1 且 valid=0(值与 flag 保持同步)。
|
||
gapVal = metricNA
|
||
gapValid = 0
|
||
}
|
||
if d.TotalFixes < minRateSamples {
|
||
cascadeVal = metricNA
|
||
}
|
||
if d.TotalConflicts < minRateSamples {
|
||
autoVal = metricNA
|
||
}
|
||
if d.DistillLossCount < minRateSamples {
|
||
lossVal = metricNA
|
||
}
|
||
|
||
return map[string]float64{
|
||
"recall_usefulness_rate": usefulVal,
|
||
"recall_usefulness_samples": float64(usefulSamples),
|
||
"recall_usefulness_negative": float64(d.NotUsefulCount),
|
||
"recall_usefulness_positive": float64(d.UsefulCount),
|
||
"recall_usefulness_valid": usefulValid,
|
||
|
||
"gap_closure_rate": gapVal,
|
||
"gap_closure_samples": float64(d.TotalGaps),
|
||
"gap_closure_valid": gapValid,
|
||
|
||
"cascade_fix_rate": cascadeVal,
|
||
"cascade_fix_samples": float64(d.TotalFixes),
|
||
"cascade_fix_valid": cascadeValid,
|
||
|
||
"auto_resolve_rate": autoVal,
|
||
"auto_resolve_samples": float64(d.TotalConflicts),
|
||
"auto_resolve_valid": autoValid,
|
||
|
||
// counter 语义:无分母 ⇒ 不给 _samples/_valid(给了会诱导下游当率用)。
|
||
"deprecated_per_day": float64(d.DeprecatedToday),
|
||
"deprecated_total": float64(d.DeprecatedTotal),
|
||
|
||
"distill_success_count": float64(d.DistillSuccessCount),
|
||
"avg_distill_loss": lossVal,
|
||
"avg_distill_loss_samples": float64(d.DistillLossCount),
|
||
"avg_distill_loss_valid": lossValid,
|
||
}
|
||
}
|
||
|
||
func (d *Dashboard) RecordRecall(hit bool) {
|
||
d.mu.Lock()
|
||
defer d.mu.Unlock()
|
||
d.TotalRecalls++
|
||
if hit {
|
||
d.HitCount++
|
||
}
|
||
d.persist()
|
||
}
|
||
|
||
func (d *Dashboard) RecordFeedback(useful bool) {
|
||
d.mu.Lock()
|
||
defer d.mu.Unlock()
|
||
if useful {
|
||
d.UsefulCount++
|
||
} else {
|
||
d.NotUsefulCount++
|
||
}
|
||
d.persist() // 2026-09-27 补:此处漏了 persist() ⇒ useful/not_useful 从不落盘、重启即丢
|
||
}
|
||
|
||
// RecordUseful 记录有用反馈
|
||
func (d *Dashboard) RecordUseful() {
|
||
d.RecordFeedback(true)
|
||
d.persist()
|
||
}
|
||
|
||
// RecordNotUseful 记录无用反馈
|
||
func (d *Dashboard) RecordNotUseful() {
|
||
d.RecordFeedback(false)
|
||
d.persist()
|
||
}
|
||
|
||
// RecordDistillSuccess 记录蒸馏成功次数(独立于用户反馈,2026-10-14 t_c06ad4d0 解污染)。
|
||
// 原先蒸馏回调误调 RecordUseful() ⇒ 把「蒸馏成功次数」灌进 recall_usefulness_rate 的分子,
|
||
// 与分母(用户 not-useful 反馈)不同源,比率在数学上不成立、恒被蒸馏数抬高到 ~1.0。
|
||
// 现拆成独立计数 distill_success_count,recall_usefulness_rate 只由 [useful|not-useful] 反馈驱动。
|
||
func (d *Dashboard) RecordDistillSuccess() {
|
||
d.mu.Lock()
|
||
defer d.mu.Unlock()
|
||
d.DistillSuccessCount++
|
||
d.persist()
|
||
}
|
||
|
||
// QualityScore 计算当前质量分数
|
||
func (d *Dashboard) QualityScore() float64 {
|
||
d.mu.RLock()
|
||
defer d.mu.RUnlock()
|
||
total := d.UsefulCount + d.NotUsefulCount
|
||
if total == 0 {
|
||
return 0.5
|
||
}
|
||
return float64(d.UsefulCount) / float64(total)
|
||
}
|
||
|
||
// ─── 知识缺口检测 ────────────────────────────────────────
|
||
|
||
type GapType string
|
||
|
||
const (
|
||
GapUnknown GapType = "A" // 真不知道
|
||
GapSynonym GapType = "B" // 同义词不匹配
|
||
GapRecallFailed GapType = "C" // 召回失败
|
||
GapFragmented GapType = "D" // 碎片化
|
||
)
|
||
|
||
type Gap struct {
|
||
Topic string `json:"topic"`
|
||
Type GapType `json:"type"`
|
||
MissCount int `json:"miss_count"`
|
||
CreatedAt time.Time `json:"created_at"`
|
||
Closed bool `json:"closed"`
|
||
}
|
||
|
||
type GapDetector struct {
|
||
mu sync.RWMutex
|
||
gaps map[string]*Gap
|
||
misses map[string]int
|
||
|
||
// 向量比较引擎(用于缺口分类)
|
||
embedder *storage.Embedder
|
||
ldb storage.LanceDB
|
||
|
||
// Redis 持久化
|
||
redisClient *storage.RedisClient
|
||
}
|
||
|
||
// gapRedisKey 缺口持久化 Redis key
|
||
const gapRedisKey = "zhiyi:gaps"
|
||
const gapMissRedisKey = "zhiyi:gap_misses"
|
||
|
||
// EnableGapRedisPersistence 启动缺口 Redis 持久化
|
||
func (gd *GapDetector) EnableGapRedisPersistence() {
|
||
rc := storage.GetRedisClient()
|
||
if rc == nil {
|
||
return
|
||
}
|
||
gd.redisClient = rc
|
||
gd.loadFromRedis()
|
||
}
|
||
|
||
func (gd *GapDetector) loadFromRedis() {
|
||
if gd.redisClient == nil {
|
||
return
|
||
}
|
||
// 加载 gaps
|
||
data, err := gd.redisClient.HGetAll(gapRedisKey)
|
||
if err == nil && len(data) > 0 {
|
||
for topic, jsonStr := range data {
|
||
var gap Gap
|
||
if err := json.Unmarshal([]byte(jsonStr), &gap); err == nil {
|
||
gd.gaps[topic] = &gap
|
||
}
|
||
}
|
||
}
|
||
// 加载 misses
|
||
missData, err := gd.redisClient.HGetAll(gapMissRedisKey)
|
||
if err == nil && len(missData) > 0 {
|
||
for topic, countStr := range missData {
|
||
if count, err := strconv.Atoi(countStr); err == nil {
|
||
gd.misses[topic] = count
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
func (gd *GapDetector) persistGaps() {
|
||
if gd.redisClient == nil {
|
||
return
|
||
}
|
||
for topic, gap := range gd.gaps {
|
||
data, _ := json.Marshal(gap)
|
||
gd.redisClient.HSet(gapRedisKey, topic, string(data))
|
||
}
|
||
gd.redisClient.Expire(gapRedisKey, 90*24*time.Hour)
|
||
}
|
||
|
||
func (gd *GapDetector) persistMisses() {
|
||
if gd.redisClient == nil {
|
||
return
|
||
}
|
||
for topic, count := range gd.misses {
|
||
gd.redisClient.HSet(gapMissRedisKey, topic, strconv.Itoa(count))
|
||
}
|
||
gd.redisClient.Expire(gapMissRedisKey, 90*24*time.Hour)
|
||
}
|
||
|
||
func NewGapDetector(emb *storage.Embedder, ldb storage.LanceDB) *GapDetector {
|
||
return &GapDetector{
|
||
gaps: make(map[string]*Gap),
|
||
misses: make(map[string]int),
|
||
embedder: emb,
|
||
ldb: ldb,
|
||
}
|
||
}
|
||
|
||
// RecordMiss 记录一次召回失败
|
||
// gapThreshold 是「同一 topic 累计几次 miss 才生成 gap」的阈值。
|
||
// 由 routes 的调参器经 SetGapThreshold 推送;安全边界 [2,10]:
|
||
// 下界 2 防「一次偶然即成缺口」把缺口表灌爆,上界 10 防阈值大到永不生成(闭环静默死掉)。
|
||
var gapThreshold atomic.Int64
|
||
|
||
func init() { gapThreshold.Store(3) }
|
||
|
||
// SetGapThreshold 由调参器推送新的缺口阈值(越界值一律忽略)。
|
||
func SetGapThreshold(n int) {
|
||
if n < 2 || n > 10 {
|
||
return
|
||
}
|
||
gapThreshold.Store(int64(n))
|
||
}
|
||
|
||
// GapThreshold 返回当前缺口阈值。
|
||
func GapThreshold() int { return int(gapThreshold.Load()) }
|
||
|
||
func (gd *GapDetector) RecordMiss(topic string) *Gap {
|
||
gd.mu.Lock()
|
||
defer gd.mu.Unlock()
|
||
|
||
gd.misses[topic]++
|
||
gd.persistMisses()
|
||
// 2026-10-14:阈值改读调参器推送值(默认 3,与原硬编码一致 ⇒ 行为不变)。
|
||
// 此前 Tuner.GapThreshold 没有任何读者 ⇒ 调参改了也不生效。
|
||
if gd.misses[topic] >= GapThreshold() {
|
||
if _, exists := gd.gaps[topic]; !exists {
|
||
gap := &Gap{
|
||
Topic: topic,
|
||
Type: gd.classifyGap(topic),
|
||
MissCount: gd.misses[topic],
|
||
CreatedAt: time.Now(),
|
||
}
|
||
gd.gaps[topic] = gap
|
||
gd.persistGaps()
|
||
Dash.TotalGaps++ // 同步仪表盘
|
||
return gap
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// classifyGap 基于向量比较的缺口分类(与设计文档一致)
|
||
//
|
||
// sim > 0.85 且 category 相近 → Type C(召回失败)
|
||
// sim > 0.75 但 entity 名称不同 → Type B(同义词不匹配)
|
||
// max sim < 0.3(全聚类 centroid)→ Type A(真未知)
|
||
// 多个 L1 记忆各自部分覆盖 → Type D(碎片化)
|
||
func (gd *GapDetector) classifyGap(topic string) GapType {
|
||
if gd.embedder == nil || gd.ldb == nil {
|
||
// 降级:无 embedder 时用简单启发式
|
||
// 英文大写开头 → 可能是同义词未收录
|
||
for _, r := range topic {
|
||
if r >= 'A' && r <= 'Z' {
|
||
return GapSynonym
|
||
}
|
||
}
|
||
// 中文:检查是否含常见同义词标记(又称、别名、aka、aka)
|
||
if containsAny(topic, []string{"又称", "别名", "也就是", "aka", "AKA", "同义词"}) {
|
||
return GapSynonym
|
||
}
|
||
// 含中文字符但无同义词标记 → 真未知
|
||
return GapUnknown
|
||
}
|
||
|
||
vec, err := gd.embedder.EncodeSingle(topic)
|
||
if err != nil {
|
||
return GapUnknown
|
||
}
|
||
|
||
// ANN 搜索 top-5 最近记忆
|
||
results, err := gd.ldb.Search("memories", vec, 5, "")
|
||
if err != nil || len(results) == 0 {
|
||
return GapUnknown
|
||
}
|
||
|
||
maxSim := results[0].QualityScore // 复用 quality_score 字段存向量相似度
|
||
if maxSim > 0.85 {
|
||
// 存在高度相似 → 召回失败(应调整 top_k/diversity)
|
||
return GapRecallFailed
|
||
}
|
||
if maxSim > 0.75 {
|
||
// 有相似但名称不同 → 同义词/别称问题
|
||
return GapSynonym
|
||
}
|
||
|
||
// 检查多个 L1 是否有部分覆盖 → 碎片化
|
||
if len(results) >= 3 {
|
||
partialCover := 0
|
||
for _, r := range results {
|
||
if r.QualityScore > 0.5 && r.QualityScore < 0.7 {
|
||
partialCover++
|
||
}
|
||
}
|
||
if partialCover >= 2 {
|
||
return GapFragmented
|
||
}
|
||
}
|
||
|
||
// 最大相似度 < 0.3 → 真未知
|
||
if maxSim < 0.3 {
|
||
return GapUnknown
|
||
}
|
||
|
||
// 兜底
|
||
return GapUnknown
|
||
}
|
||
|
||
func (gd *GapDetector) List() []*Gap {
|
||
gd.mu.RLock()
|
||
defer gd.mu.RUnlock()
|
||
var result []*Gap
|
||
for _, g := range gd.gaps {
|
||
result = append(result, g)
|
||
}
|
||
return result
|
||
}
|
||
|
||
// Close 关闭 topic 对应的缺口,返回本次调用是否真的发生了 open→closed 迁移。
|
||
//
|
||
// t_e5997ca4 写侧统一:ClosedGaps/TotalGaps 口径为「缺口个数」(同一缺口至多计 1 次)。
|
||
// +1 的唯一入口在本迁移点(g.Closed false→true 时),重复调用/多入口重复关闭永不虚增;
|
||
// 外部调用方(routes / server 循环)禁止再做无条件 RecordGapClosed 式事件自增
|
||
// ——历史 9 vs 2 不一致(closed_gaps=9 / total_gaps=2)正是那样污染的。
|
||
// 返回值语义:true = 首次关闭(仪表盘计数已随迁移 +1);
|
||
// false = 已关闭或未知 topic(no-op,调用方不得再动计数)。
|
||
func (gd *GapDetector) Close(topic string) bool {
|
||
gd.mu.Lock()
|
||
defer gd.mu.Unlock()
|
||
if g, ok := gd.gaps[topic]; ok && !g.Closed {
|
||
g.Closed = true
|
||
gd.persistGaps()
|
||
Dash.recordGapClosed() // 迁移内 +1(唯一入口)
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
// RecordHit 记录一次召回命中,自动关闭该 topic 的 open gap。
|
||
// 返回语义与 Close 一致:true = 本次命中真的发生了 open→closed 迁移。
|
||
func (gd *GapDetector) RecordHit(topic string) bool {
|
||
gd.mu.Lock()
|
||
defer gd.mu.Unlock()
|
||
if g, ok := gd.gaps[topic]; ok && !g.Closed {
|
||
g.Closed = true
|
||
gd.persistGaps()
|
||
Dash.recordGapClosed() // 迁移内 +1(唯一入口)
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
// ClearMisses 清除某个 topic 的 miss 计数(recall 命中后调用)
|
||
func (gd *GapDetector) ClearMisses(topic string) {
|
||
gd.mu.Lock()
|
||
defer gd.mu.Unlock()
|
||
delete(gd.misses, topic)
|
||
gd.persistMisses()
|
||
}
|
||
|
||
// ─── 全局 GapDetector 访问(供 routes 包调用)───────────────
|
||
|
||
var globalGapDetector *GapDetector
|
||
|
||
func SetGlobalGapDetector(gd *GapDetector) { globalGapDetector = gd }
|
||
|
||
func GetGlobalGapDetector() *GapDetector { return globalGapDetector }
|
||
|
||
// ─── 因果追踪 ────────────────────────────────────────────
|
||
|
||
type TraceEntry struct {
|
||
MemoryID string `json:"memory_id"`
|
||
Version int `json:"version"`
|
||
Content string `json:"content"`
|
||
Source string `json:"source"` // muchen_oral / config_parse / agent_infer / llm_distill
|
||
Trigger string `json:"trigger"` // 什么触发了这次修改
|
||
UpdatedAt time.Time `json:"updated_at"`
|
||
}
|
||
|
||
type CausalTracker struct {
|
||
mu sync.RWMutex
|
||
entries map[string][]*TraceEntry // memory_id → version history
|
||
deps map[string][]string // memory_id → depends_on[]
|
||
|
||
// Redis 持久化
|
||
redisClient *storage.RedisClient
|
||
}
|
||
|
||
const causalRedisKey = "zhiyi:causal_entries"
|
||
|
||
// EnableCausalRedisPersistence 启动因果追踪 Redis 持久化
|
||
func (ct *CausalTracker) EnableCausalRedisPersistence() {
|
||
rc := storage.GetRedisClient()
|
||
if rc == nil {
|
||
return
|
||
}
|
||
ct.redisClient = rc
|
||
ct.loadCausalFromRedis()
|
||
}
|
||
|
||
func (ct *CausalTracker) loadCausalFromRedis() {
|
||
if ct.redisClient == nil {
|
||
return
|
||
}
|
||
data, err := ct.redisClient.HGetAll(causalRedisKey)
|
||
if err != nil || len(data) == 0 {
|
||
return
|
||
}
|
||
// entries: memory_id → JSON数组
|
||
// deps: memory_id → JSON数组
|
||
ct.mu.Lock()
|
||
defer ct.mu.Unlock()
|
||
for key, jsonStr := range data {
|
||
if len(key) > 5 && key[:5] == "dep_:" {
|
||
// dep_:memory_id → [dependency_ids]
|
||
memID := key[5:]
|
||
var deps []string
|
||
if json.Unmarshal([]byte(jsonStr), &deps) == nil {
|
||
ct.deps[memID] = deps
|
||
}
|
||
} else {
|
||
// memory_id → [TraceEntry]
|
||
var entries []*TraceEntry
|
||
if json.Unmarshal([]byte(jsonStr), &entries) == nil {
|
||
ct.entries[key] = entries
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
func (ct *CausalTracker) persistCausal() {
|
||
if ct.redisClient == nil {
|
||
return
|
||
}
|
||
for memID, entries := range ct.entries {
|
||
data, _ := json.Marshal(entries)
|
||
ct.redisClient.HSet(causalRedisKey, memID, string(data))
|
||
}
|
||
for memID, deps := range ct.deps {
|
||
data, _ := json.Marshal(deps)
|
||
ct.redisClient.HSet(causalRedisKey, "dep_:"+memID, string(data))
|
||
}
|
||
ct.redisClient.Expire(causalRedisKey, 90*24*time.Hour)
|
||
}
|
||
|
||
func NewCausalTracker() *CausalTracker {
|
||
return &CausalTracker{
|
||
entries: make(map[string][]*TraceEntry),
|
||
deps: make(map[string][]string),
|
||
}
|
||
}
|
||
|
||
// RecordVersion 记录版本变更
|
||
func (ct *CausalTracker) RecordVersion(memoryID, content, source, trigger string) {
|
||
ct.mu.Lock()
|
||
defer ct.mu.Unlock()
|
||
|
||
entry := &TraceEntry{
|
||
MemoryID: memoryID,
|
||
Version: len(ct.entries[memoryID]) + 1,
|
||
Content: content,
|
||
Source: source,
|
||
Trigger: trigger,
|
||
UpdatedAt: time.Now(),
|
||
}
|
||
ct.entries[memoryID] = append(ct.entries[memoryID], entry)
|
||
ct.persistCausal()
|
||
}
|
||
|
||
// AddDependency A depends_on B
|
||
func (ct *CausalTracker) AddDependency(a, b string) {
|
||
ct.mu.Lock()
|
||
defer ct.mu.Unlock()
|
||
ct.deps[a] = append(ct.deps[a], b)
|
||
ct.persistCausal()
|
||
}
|
||
|
||
// GetAffected 当 memoryID 被修正时,返回所有依赖它的记忆
|
||
func (ct *CausalTracker) GetAffected(memoryID string, visited map[string]bool) []string {
|
||
ct.mu.RLock()
|
||
defer ct.mu.RUnlock()
|
||
|
||
if visited == nil {
|
||
visited = make(map[string]bool)
|
||
}
|
||
if visited[memoryID] {
|
||
return nil
|
||
}
|
||
visited[memoryID] = true
|
||
|
||
var affected []string
|
||
for dependent, deps := range ct.deps {
|
||
for _, d := range deps {
|
||
if d == memoryID && !visited[dependent] {
|
||
affected = append(affected, dependent)
|
||
affected = append(affected, ct.GetAffected(dependent, visited)...)
|
||
}
|
||
}
|
||
}
|
||
return affected
|
||
}
|
||
|
||
// SourceTrust 来源信任度
|
||
func SourceTrust(source string) float64 {
|
||
switch source {
|
||
case "muchen_oral":
|
||
return 1.0
|
||
case "muchen_feishu":
|
||
return 0.95
|
||
case "config_parse":
|
||
return 0.7
|
||
case "agent_infer":
|
||
return 0.5
|
||
case "llm_distill":
|
||
return 0.4
|
||
default:
|
||
return 0.3
|
||
}
|
||
}
|
||
|
||
// IsVolatile 判断一条记忆是否易变(频繁修正)
|
||
func (ct *CausalTracker) IsVolatile(memoryID string) bool {
|
||
ct.mu.RLock()
|
||
defer ct.mu.RUnlock()
|
||
return len(ct.entries[memoryID]) >= 3
|
||
}
|
||
|
||
// Entries 返回全部版本历史(只读)
|
||
func (ct *CausalTracker) Entries() map[string][]*TraceEntry {
|
||
ct.mu.RLock()
|
||
defer ct.mu.RUnlock()
|
||
cp := make(map[string][]*TraceEntry, len(ct.entries))
|
||
for k, v := range ct.entries {
|
||
cp[k] = v
|
||
}
|
||
return cp
|
||
}
|
||
|
||
// ─── 记忆预取 ────────────────────────────────────────────
|
||
|
||
type PrefetchGraph struct {
|
||
mu sync.RWMutex
|
||
coOccurs map[string]map[string]int // A → {B: count}
|
||
}
|
||
|
||
func NewPrefetchGraph() *PrefetchGraph {
|
||
return &PrefetchGraph{coOccurs: make(map[string]map[string]int)}
|
||
}
|
||
|
||
func (pg *PrefetchGraph) RecordCoAccess(a, b string) {
|
||
pg.mu.Lock()
|
||
defer pg.mu.Unlock()
|
||
if pg.coOccurs[a] == nil {
|
||
pg.coOccurs[a] = make(map[string]int)
|
||
}
|
||
pg.coOccurs[a][b]++
|
||
}
|
||
|
||
// GetPrefetch 获取某个 query 的预取候选项
|
||
// RecordDeprecation 记录废弃操作
|
||
func (d *Dashboard) RecordDeprecation() {
|
||
d.mu.Lock()
|
||
defer d.mu.Unlock()
|
||
d.DeprecatedToday++
|
||
d.DeprecatedTotal++
|
||
d.persist()
|
||
}
|
||
|
||
// RecordCorrection 记录用户修正
|
||
func (d *Dashboard) RecordCorrection(source string) {
|
||
d.mu.Lock()
|
||
defer d.mu.Unlock()
|
||
d.TotalFixes++
|
||
if source == "muchen_correction" {
|
||
d.CascadeFixedTotal++
|
||
}
|
||
d.persist()
|
||
}
|
||
|
||
// recordGapClosed 记录缺口关闭(+1 ClosedGaps 并持久化)。
|
||
//
|
||
// ⚠️ t_e5997ca4 起降为包内(小写):+1 的唯一合法入口是 GapDetector 的
|
||
// open→closed 迁移点(Close / RecordHit 内部)。跨包事件式无条件调用
|
||
// (旧 gaps.go:44 / server.go:1950 那种)会让同一缺口被无限计数、
|
||
// ClosedGaps 越过 TotalGaps(历史 9 vs 2 的根因)⇒ 用包边界从编译期堵死。
|
||
// 需要「本次是否真的关闭」语义的外部调用方,看 Close/RecordHit 的 bool 返回值。
|
||
func (d *Dashboard) recordGapClosed() {
|
||
d.mu.Lock()
|
||
defer d.mu.Unlock()
|
||
d.ClosedGaps++
|
||
d.persist()
|
||
}
|
||
|
||
// ClosedGapsCount 只读快照访问器(跨包审计/测试用;调用方自持锁语义不暴露)。
|
||
func (d *Dashboard) ClosedGapsCount() int {
|
||
d.mu.RLock()
|
||
defer d.mu.RUnlock()
|
||
return d.ClosedGaps
|
||
}
|
||
|
||
// TotalGapsCount 只读快照访问器(跨包审计/测试用)。
|
||
func (d *Dashboard) TotalGapsCount() int {
|
||
d.mu.RLock()
|
||
defer d.mu.RUnlock()
|
||
return d.TotalGaps
|
||
}
|
||
|
||
// RecordConflictResolved 记录冲突解决
|
||
func (d *Dashboard) RecordConflictResolved(auto bool) {
|
||
d.mu.Lock()
|
||
defer d.mu.Unlock()
|
||
d.TotalConflicts++
|
||
if auto {
|
||
d.AutoResolvedConflicts++
|
||
}
|
||
d.persist()
|
||
}
|
||
|
||
// RecordDistillLoss 记录蒸馏质量损失
|
||
func (d *Dashboard) RecordDistillLoss(loss float64) {
|
||
d.mu.Lock()
|
||
defer d.mu.Unlock()
|
||
d.DistillLossSum += loss
|
||
d.DistillLossCount++
|
||
d.persist()
|
||
}
|
||
|
||
func (pg *PrefetchGraph) GetPrefetch(query string) []string {
|
||
pg.mu.RLock()
|
||
defer pg.mu.RUnlock()
|
||
|
||
related := pg.coOccurs[query]
|
||
if related == nil {
|
||
return nil
|
||
}
|
||
|
||
var result []string
|
||
for topic, count := range related {
|
||
total := 0
|
||
for _, c := range pg.coOccurs[query] {
|
||
total += c
|
||
}
|
||
prob := float64(count) / float64(total)
|
||
if prob > 0.6 {
|
||
result = append(result, topic)
|
||
}
|
||
}
|
||
return result
|
||
}
|
||
|
||
// containsAny 检查字符串是否包含任意一个子串(无 strings 依赖)
|
||
func containsAny(s string, substrs []string) bool {
|
||
for _, sub := range substrs {
|
||
if len(sub) == 0 {
|
||
continue
|
||
}
|
||
for i := 0; i <= len(s)-len(sub); i++ {
|
||
match := true
|
||
for j := 0; j < len(sub); j++ {
|
||
if s[i+j] != sub[j] {
|
||
match = false
|
||
break
|
||
}
|
||
}
|
||
if match {
|
||
return true
|
||
}
|
||
}
|
||
}
|
||
return false
|
||
}
|