Files
memoryweave/go/internal/distill/rules.go
T

113 lines
3.2 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// 织忆 MemoryWeave — 硬规则过滤
// 蒸馏前筛选:跳过闲聊、纯知识问答、低密度内容
package distill
import (
"strings"
"unicode/utf8"
)
// HardRulesPass 硬规则过滤 → true = 通过,可进入蒸馏队列
func HardRulesPass(content string) bool {
// 规则1: 无用户消息(纯系统消息) → 跳过
if strings.TrimSpace(content) == "" {
return false
}
// 规则2: 太短的消息 (< 10字符) → 跳过
// 例外:人侧结构化短事实(画像/心迹/牵挂/偏好/习惯/目标)天然短但 100% 是用户事实,
// 不因字数被丢弃——否则"牧尘不在招商局工作"(9 字) 这类关键事实永远到不了 memories。
if utf8.RuneCountInString(content) < 10 {
if isSoulfulShortFact(content) {
return true
}
return false
}
// 规则3: 纯闲聊/问候 → 跳过
chitchatPatterns := []string{
"你好", "在吗", "在?", "好的", "谢谢", "不客气",
"ok", "OK", "嗯", "哦", "知道了",
}
lower := strings.ToLower(strings.TrimSpace(content))
for _, pattern := range chitchatPatterns {
if strings.EqualFold(lower, pattern) || lower == pattern {
return false
}
}
// 规则4: 纯知识问答(如 "什么是...") → 跳过
qaPrefixes := []string{"什么是", "怎么定义", "解释一下", "介绍一下"}
for _, prefix := range qaPrefixes {
if strings.HasPrefix(content, prefix) {
return false
}
}
// 规则5: 内容密度检查 (信息密度 < 0.3 → 跳过)
density := contentDensity(content)
if density < 0.3 {
return false
}
// 规则6: 长度适中(太长可能是粘贴/日志,跳过蒸馏,交给 episode 保留)
if utf8.RuneCountInString(content) > 5000 {
return false
}
return true
}
// contentDensity 内容信息密度 = 有意义字符 / 总字符
// 过滤空白、标点、重复字符
func contentDensity(text string) float64 {
if len(text) == 0 {
return 0
}
meaningful := 0
spaces := 0
prev := rune(0)
repeats := 0
for _, r := range text {
if r == ' ' || r == '\t' || r == '\n' || r == '\r' {
spaces++
} else if r == ',' || r == '。' || r == '!' || r == '?' ||
r == '.' || r == '!' || r == '?' || r == ';' || r == ';' {
// 标点,不计入 meaningful
} else if r == prev {
repeats++
} else {
meaningful++
}
prev = r
}
// 有空格和重复扣分
total := len(text)
adjusted := meaningful - spaces/2 - repeats/3
if adjusted < 0 {
adjusted = 0
}
return float64(adjusted) / float64(total)
}
// ─── 深度整合触发条件 ────────────────────────────────────
// ShouldDeepConsolidate 深度整合触发条件
// 条件: (新增蒸馏 > 50 且 距上次 > 24h) 或 (距上次 > 48h)
func ShouldDeepConsolidate(distilledSinceLast int, hoursSinceLast float64) bool {
return (distilledSinceLast > 50 && hoursSinceLast > 24) || hoursSinceLast > 48
}
// ─── 批量蒸馏触发条件 ────────────────────────────────────
// ShouldBatchDistill 批量蒸馏触发条件
// 条件: 队列 >= 10 或 距上次 > 5 分钟
func ShouldBatchDistill(queueLen int, secondsSinceLast float64) bool {
return queueLen >= 10 || secondsSinceLast > 300
}