1. PageRank 三重循环 O(47k×374k×20≈3.5e11) → 入边表 O(V+E): 单次 full 整合 4h41m → 95.2s,内存峰值 3.8G → 237MB(实测) 2. 图谱修剪 merge 边 id 加 relation + INSERT OR REPLACE: 修复 09-18 起 UNIQUE constraint 静默失败 → 主库首次跑消 9.8万冗余边 3. LanceDB 写入去抖(server.go + validator.go): 被动验证只写 Confidence 实际变化的记录;原先每输入重写全库 6000 条 → 版本快照 O(n²),每日 ~1.9万版本/十几G 膨胀的根因 4. 新增 scripts/lance_vacuum_online.py:在线 VACUUM(不停服务)+ 24h 版本窗口(可回滚)+ 行数门禁(减少才报警);替代停服务版 每日 vacuum → cron 已改为在线版 实测:侧车 RSS 1.09G→69MB;memories 分片 5383→1;服务全程在线
234 lines
6.3 KiB
Go
234 lines
6.3 KiB
Go
// 织忆 MemoryWeave — 被动验证器 (PassiveValidator) P1/P2/P3 三层匹配
|
|
package selfoptimize
|
|
|
|
import (
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
// ─── 被动验证器 ──────────────────────────────────────────
|
|
|
|
// ValidationLevel 验证层级
|
|
type ValidationLevel int
|
|
|
|
const (
|
|
P1_ExactMatch ValidationLevel = 1 // 精确匹配:牧尘再次提到同一事实
|
|
P2_PartialMatch ValidationLevel = 2 // 部分匹配:子串/同义词
|
|
P3_ImpliedMatch ValidationLevel = 3 // 隐含验证:牧尘基于该记忆做出的决策成功
|
|
)
|
|
|
|
// ValidationRecord 单条验证记录
|
|
type ValidationRecord struct {
|
|
MemoryID string `json:"memory_id"`
|
|
Level ValidationLevel `json:"level"`
|
|
MatchedBy string `json:"matched_by"` // 匹配到的内容片段
|
|
Confidence float64 `json:"confidence"` // 当前信任度
|
|
LastSeen time.Time `json:"last_seen"`
|
|
PassiveHits int `json:"passive_hits"` // 被动匹配次数
|
|
Changed bool `json:"changed"` // 本次验证是否实际改变了 Confidence(用于写入去抖)
|
|
}
|
|
|
|
// PassiveValidator 被动验证引擎
|
|
type PassiveValidator struct {
|
|
mu sync.RWMutex
|
|
records map[string]*ValidationRecord // memory_id → record
|
|
boostP1 float64 // P1 信任度提升 (default 0.1)
|
|
boostP2 float64 // P2 信任度提升 (default 0.05)
|
|
boostP3 float64 // P3 信任度提升 (default 0.15)
|
|
maxConf float64 // 最大信任度 (default 0.99)
|
|
}
|
|
|
|
var Validator = NewPassiveValidator()
|
|
|
|
func NewPassiveValidator() *PassiveValidator {
|
|
return &PassiveValidator{
|
|
records: make(map[string]*ValidationRecord),
|
|
boostP1: 0.10,
|
|
boostP2: 0.05,
|
|
boostP3: 0.15,
|
|
maxConf: 0.99,
|
|
}
|
|
}
|
|
|
|
// Validate 牧尘的新输入到达时,检查是否验证了已有记忆
|
|
func (pv *PassiveValidator) Validate(userInput string, existingMemories []MemoryForValidation) []*ValidationRecord {
|
|
pv.mu.Lock()
|
|
defer pv.mu.Unlock()
|
|
|
|
var validated []*ValidationRecord
|
|
|
|
for _, mem := range existingMemories {
|
|
level := pv.checkMatch(userInput, mem.Content)
|
|
if level == 0 {
|
|
continue
|
|
}
|
|
|
|
record, exists := pv.records[mem.ID]
|
|
if !exists {
|
|
record = &ValidationRecord{
|
|
MemoryID: mem.ID,
|
|
Confidence: mem.QualityScore,
|
|
PassiveHits: 0,
|
|
}
|
|
pv.records[mem.ID] = record
|
|
}
|
|
|
|
record.Level = level
|
|
record.MatchedBy = extractMatchFragment(userInput, mem.Content)
|
|
record.LastSeen = time.Now()
|
|
record.PassiveHits++
|
|
|
|
// 按层级提升信任度
|
|
oldConf := record.Confidence
|
|
switch level {
|
|
case P1_ExactMatch:
|
|
record.Confidence = minConf(record.Confidence+pv.boostP1, pv.maxConf)
|
|
case P2_PartialMatch:
|
|
record.Confidence = minConf(record.Confidence+pv.boostP2, pv.maxConf)
|
|
case P3_ImpliedMatch:
|
|
record.Confidence = minConf(record.Confidence+pv.boostP3, pv.maxConf)
|
|
}
|
|
// 记录本次是否实际改变:调用方(api/server.go)据此决定是否写库。
|
|
// 背景(2026-09-20):值未变也写会为每条记忆产生一个 LanceDB 版本快照(O(n²)),
|
|
// 导致版本数每日 ~1.9 万、占用每日涨十几 G。
|
|
record.Changed = record.Confidence != oldConf
|
|
|
|
validated = append(validated, record)
|
|
}
|
|
return validated
|
|
}
|
|
|
|
// checkMatch 三层匹配检测
|
|
func (pv *PassiveValidator) checkMatch(userInput, memoryContent string) ValidationLevel {
|
|
// P1: 精确匹配 — memoryContent 是 userInput 的子串(或相反)
|
|
if strings.Contains(userInput, memoryContent) || strings.Contains(memoryContent, userInput) {
|
|
return P1_ExactMatch
|
|
}
|
|
|
|
// P2: 部分匹配 — 关键词重叠 > 60%
|
|
overlap := keywordOverlap(userInput, memoryContent)
|
|
if overlap > 0.6 {
|
|
return P2_PartialMatch
|
|
}
|
|
|
|
// P3: 隐含匹配 — 共享实体引用
|
|
entities1 := extractEntities(userInput)
|
|
entities2 := extractEntities(memoryContent)
|
|
if sharedEntities(entities1, entities2) {
|
|
return P3_ImpliedMatch
|
|
}
|
|
|
|
return 0 // 无匹配
|
|
}
|
|
|
|
// GetConfidence 获取某条记忆的被动验证信任度
|
|
func (pv *PassiveValidator) GetConfidence(memoryID string) float64 {
|
|
pv.mu.RLock()
|
|
defer pv.mu.RUnlock()
|
|
if record, ok := pv.records[memoryID]; ok {
|
|
return record.Confidence
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// GetRecords 获取所有验证记录(按 confidence 降序)
|
|
func (pv *PassiveValidator) GetRecords() []*ValidationRecord {
|
|
pv.mu.RLock()
|
|
defer pv.mu.RUnlock()
|
|
var records []*ValidationRecord
|
|
for _, r := range pv.records {
|
|
records = append(records, r)
|
|
}
|
|
if records == nil {
|
|
return []*ValidationRecord{}
|
|
}
|
|
return records
|
|
}
|
|
|
|
// ─── 辅助函数 ──────────────────────────────────────────
|
|
|
|
type MemoryForValidation struct {
|
|
ID string
|
|
Content string
|
|
QualityScore float64
|
|
}
|
|
|
|
func keywordOverlap(a, b string) float64 {
|
|
wordsA := strings.Fields(strings.ToLower(a))
|
|
wordsB := strings.Fields(strings.ToLower(b))
|
|
setA := make(map[string]bool, len(wordsA))
|
|
for _, w := range wordsA {
|
|
setA[w] = true
|
|
}
|
|
overlap := 0
|
|
for _, w := range wordsB {
|
|
if setA[w] {
|
|
overlap++
|
|
}
|
|
}
|
|
if len(wordsA) == 0 || len(wordsB) == 0 {
|
|
return 0
|
|
}
|
|
return float64(overlap) / float64(maxInt(len(wordsA), len(wordsB)))
|
|
}
|
|
|
|
func extractEntities(text string) []string {
|
|
// 简单启发式:提取大写开头的词和中文专有名词
|
|
words := strings.Fields(text)
|
|
var entities []string
|
|
for _, w := range words {
|
|
if len(w) > 1 && (w[0] >= 'A' && w[0] <= 'Z') {
|
|
entities = append(entities, w)
|
|
}
|
|
}
|
|
return entities
|
|
}
|
|
|
|
func sharedEntities(a, b []string) bool {
|
|
set := make(map[string]bool, len(a))
|
|
for _, e := range a {
|
|
set[e] = true
|
|
}
|
|
for _, e := range b {
|
|
if set[e] {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func extractMatchFragment(userInput, memoryContent string) string {
|
|
// 返回重叠部分
|
|
if idx := strings.Index(userInput, memoryContent); idx >= 0 {
|
|
return memoryContent
|
|
}
|
|
// 返回共享关键实体
|
|
entities := extractEntities(memoryContent)
|
|
if len(entities) > 0 {
|
|
return strings.Join(entities, ", ")
|
|
}
|
|
return memoryContent[:minInt(50, len(memoryContent))]
|
|
}
|
|
|
|
func minConf(a, b float64) float64 {
|
|
if a < b {
|
|
return a
|
|
}
|
|
return b
|
|
}
|
|
|
|
func maxInt(a, b int) int {
|
|
if a > b {
|
|
return a
|
|
}
|
|
return b
|
|
}
|
|
|
|
func minInt(a, b int) int {
|
|
if a < b {
|
|
return a
|
|
}
|
|
return b
|
|
}
|