273 lines
8.0 KiB
Go
273 lines
8.0 KiB
Go
// 织忆 MemoryWeave — Consolidation 流水线 (Step 1-4)
|
||
package routes
|
||
|
||
import (
|
||
"fmt"
|
||
"log"
|
||
"strings"
|
||
"time"
|
||
|
||
"github.com/xiaoxue/memoryweave/internal/consolidate"
|
||
"github.com/xiaoxue/memoryweave/internal/governance"
|
||
"github.com/xiaoxue/memoryweave/internal/storage"
|
||
)
|
||
|
||
// ConsolidationPipeline 完整整合流水线
|
||
// 优先调用 Rust sidecar（DBSCAN + 衰减校准 + 质量回溯）
|
||
// Rust 不可用时降级为 Go 启发式
|
||
type ConsolidationPipeline struct {
|
||
ldb storage.LanceDB
|
||
graph governance.GraphStore
|
||
graphUpdater *governance.AutoGraphUpdater
|
||
conflicts *governance.ConflictDetector
|
||
// Rust IPC 路径
|
||
dataDir string
|
||
sqlitePath string
|
||
}
|
||
|
||
func NewConsolidationPipeline(ldb storage.LanceDB, g governance.GraphStore, cd *governance.ConflictDetector) *ConsolidationPipeline {
|
||
return &ConsolidationPipeline{
|
||
ldb: ldb,
|
||
graph: g,
|
||
graphUpdater: governance.NewAutoGraphUpdater(g),
|
||
conflicts: cd,
|
||
}
|
||
}
|
||
|
||
// SetDataDir 设置 Rust IPC 所需的路径（不设置则只走 Go 启发式）
|
||
func (cp *ConsolidationPipeline) SetDataDir(dataDir, sqlitePath string) {
|
||
cp.dataDir = dataDir
|
||
cp.sqlitePath = sqlitePath
|
||
}
|
||
|
||
// Run 执行全流程
|
||
// 优先调 Rust zhiyi-consolidate（DBSCAN + 衰减校准 + 质量回溯）
|
||
// Rust 不可用 → 降级为 Go 启发式
|
||
func (cp *ConsolidationPipeline) Run() (*ConsolidationReport, error) {
|
||
// ─── 尝试 Rust sidecar ──────────────────────────────
|
||
if cp.dataDir != "" && cp.sqlitePath != "" {
|
||
if rustReport, err := consolidate.Run(cp.dataDir, cp.sqlitePath, "full"); err == nil {
|
||
report := &ConsolidationReport{
|
||
StartedAt: time.Now(),
|
||
FinishedAt: time.Now(),
|
||
Duration: "rust_sidecar",
|
||
Merged: rustReport.Clusters,
|
||
ConflictsFound: 0,
|
||
Patterns: []string{fmt.Sprintf("decay_rates=%v", rustReport.DecayRates)},
|
||
GraphPruned: rustReport.Noise,
|
||
}
|
||
if rustReport.Quality != nil {
|
||
report.Patterns = append(report.Patterns,
|
||
fmt.Sprintf("quality_score=%.2f low_info=%d hallucinations=%d",
|
||
rustReport.Quality.Score, rustReport.Quality.LowInfo, rustReport.Quality.Hallucinations))
|
||
}
|
||
log.Printf("[consolidation] Rust sidecar 完成: clusters=%d noise=%d", rustReport.Clusters, rustReport.Noise)
|
||
PushConsolidationDone(fmt.Sprintf("rust: merged=%d conflicts=%d patterns=%d pruned=%d",
|
||
report.Merged, report.ConflictsFound, len(report.Patterns), report.GraphPruned))
|
||
return report, nil
|
||
}
|
||
log.Printf("[consolidation] Rust sidecar 不可用，降级为 Go 启发式")
|
||
}
|
||
|
||
return cp.runGoFallback()
|
||
}
|
||
|
||
// runGoFallback Go 启发式整合（Rust 不可用时的降级方案）
|
||
func (cp *ConsolidationPipeline) runGoFallback() (*ConsolidationReport, error) {
|
||
report := &ConsolidationReport{StartedAt: time.Now()}
|
||
|
||
// Step 1: 合并相似记忆
|
||
merged, err := cp.mergeSimilar()
|
||
if err != nil {
|
||
report.Errors = append(report.Errors, "merge: "+err.Error())
|
||
} else {
|
||
report.Merged = merged
|
||
}
|
||
|
||
// Step 2: 扫描冲突
|
||
found, err := cp.scanConflicts()
|
||
if err != nil {
|
||
report.Errors = append(report.Errors, "conflicts: "+err.Error())
|
||
} else {
|
||
report.ConflictsFound = found
|
||
}
|
||
|
||
// Step 3: 模式挖掘
|
||
patterns, err := cp.minePatterns()
|
||
if err != nil {
|
||
report.Errors = append(report.Errors, "patterns: "+err.Error())
|
||
} else {
|
||
report.Patterns = patterns
|
||
}
|
||
|
||
// Step 4: 图谱更新
|
||
pruned, err := cp.updateGraph()
|
||
if err != nil {
|
||
report.Errors = append(report.Errors, "graph: "+err.Error())
|
||
} else {
|
||
report.GraphPruned = pruned
|
||
}
|
||
|
||
report.FinishedAt = time.Now()
|
||
report.Duration = report.FinishedAt.Sub(report.StartedAt).String()
|
||
|
||
// 推送完成事件
|
||
PushConsolidationDone(fmt.Sprintf("merged=%d conflicts=%d patterns=%d pruned=%d",
|
||
report.Merged, report.ConflictsFound, len(report.Patterns), report.GraphPruned))
|
||
|
||
return report, nil
|
||
}
|
||
|
||
// RunMerge 公开合并步骤（供触发器 merge 单独使用）
|
||
func (cp *ConsolidationPipeline) RunMerge() (int, error) {
|
||
return cp.mergeSimilar()
|
||
}
|
||
|
||
// mergeSimilar 合并相似记忆（向量相似度 > 0.8 → 保留最新）
|
||
func (cp *ConsolidationPipeline) mergeSimilar() (int, error) {
|
||
// 获取全部 distilled 记忆
|
||
memories, err := cp.ldb.GetCandidatesForForgetting()
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
|
||
merged := 0
|
||
// 简单启发式：相同 category + 高内容重叠 → 合并
|
||
for i := 0; i < len(memories); i++ {
|
||
for j := i + 1; j < len(memories); j++ {
|
||
catI := strVal(memories[i]["category"])
|
||
catJ := strVal(memories[j]["category"])
|
||
if catI != catJ {
|
||
continue
|
||
}
|
||
contentI := strVal(memories[i]["content"])
|
||
contentJ := strVal(memories[j]["content"])
|
||
if overlap := contentOverlap(contentI, contentJ); overlap > 0.8 {
|
||
// 保留较新的
|
||
idI := strVal(memories[i]["id"])
|
||
idJ := strVal(memories[j]["id"])
|
||
_ = cp.ldb.SoftDelete(idJ, fmt.Sprintf("merged_into_%s", idI))
|
||
merged++
|
||
}
|
||
}
|
||
}
|
||
return merged, nil
|
||
}
|
||
|
||
// Step 2: 扫描所有待解决冲突
|
||
func (cp *ConsolidationPipeline) scanConflicts() (int, error) {
|
||
// 获取最近 100 条记忆
|
||
memories, err := cp.ldb.GetCandidatesForForgetting()
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
|
||
count := 0
|
||
for i := 0; i < len(memories); i++ {
|
||
content := strVal(memories[i]["content"])
|
||
entities := extractEntities(content)
|
||
if len(entities) == 0 {
|
||
continue
|
||
}
|
||
// 对比已有内容
|
||
existing := make([]map[string]interface{}, 0)
|
||
for j := 0; j < len(memories) && j < 50; j++ {
|
||
if i != j {
|
||
existing = append(existing, memories[j])
|
||
}
|
||
}
|
||
conflicts := cp.conflicts.Scan(content, entities, existing)
|
||
if len(conflicts) > 0 {
|
||
count += len(conflicts)
|
||
// 自动裁决：来源信任差异 > 0.5
|
||
for _, c := range conflicts {
|
||
if c.Strategy == "latest_wins" {
|
||
cp.conflicts.AutoResolve(c)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
return count, nil
|
||
}
|
||
|
||
// Step 3: 模式挖掘（连续 3+ 同 category 记忆 → 提取 pattern）
|
||
func (cp *ConsolidationPipeline) minePatterns() ([]string, error) {
|
||
memories, err := cp.ldb.GetCandidatesForForgetting()
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
catCount := make(map[string]int)
|
||
for _, mem := range memories {
|
||
cat := strVal(mem["category"])
|
||
catCount[cat]++
|
||
}
|
||
|
||
var patterns []string
|
||
for cat, count := range catCount {
|
||
if count >= 3 {
|
||
patterns = append(patterns, fmt.Sprintf("pattern:%s (count=%d)", cat, count))
|
||
}
|
||
}
|
||
return patterns, nil
|
||
}
|
||
|
||
// Step 4: 图谱修剪
|
||
func (cp *ConsolidationPipeline) updateGraph() (int, error) {
|
||
before, _, _ := cp.graph.Stats()
|
||
cp.graph.Prune(0.1) // 删除权重 < 0.1 的边
|
||
after, _, _ := cp.graph.Stats()
|
||
return before - after, nil
|
||
}
|
||
|
||
// ─── 报告 ─────────────────────────────────────────────
|
||
|
||
type ConsolidationReport struct {
|
||
StartedAt time.Time `json:"started_at"`
|
||
FinishedAt time.Time `json:"finished_at"`
|
||
Duration string `json:"duration"`
|
||
Merged int `json:"merged"`
|
||
ConflictsFound int `json:"conflicts_found"`
|
||
Patterns []string `json:"patterns"`
|
||
GraphPruned int `json:"graph_pruned"`
|
||
Errors []string `json:"errors,omitempty"`
|
||
}
|
||
|
||
// ─── 辅助 ─────────────────────────────────────────────
|
||
|
||
func contentOverlap(a, b string) float64 {
|
||
wordsA := strings.Fields(strings.ToLower(a))
|
||
wordsB := strings.Fields(strings.ToLower(b))
|
||
setA := make(map[string]bool, len(wordsA))
|
||
for _, w := range wordsA {
|
||
setA[w] = true
|
||
}
|
||
overlap := 0
|
||
for _, w := range wordsB {
|
||
if setA[w] {
|
||
overlap++
|
||
}
|
||
}
|
||
if len(wordsA) == 0 || len(wordsB) == 0 {
|
||
return 0
|
||
}
|
||
return float64(overlap) / float64(minInt2(len(wordsA), len(wordsB)))
|
||
}
|
||
|
||
func extractEntities(text string) []string {
|
||
words := strings.Fields(text)
|
||
var entities []string
|
||
for _, w := range words {
|
||
if len(w) > 1 && w[0] >= 'A' && w[0] <= 'Z' {
|
||
entities = append(entities, w)
|
||
}
|
||
}
|
||
return entities
|
||
}
|
||
|
||
func minInt2(a, b int) int {
|
||
if a < b { return a }
|
||
return b
|
||
}
|
||
|