fix: Phase D/G/H 编译修复

- governance.go: 加 PendingCount() 方法给 ConflictsPending metrics
- distill/: 删 consolidation.go(死代码,231行无引用)
- scripts/: backup.sh 加7天保留策略 + restore.sh 恢复脚本
This commit is contained in:
小唯 2026-07-10 05:10:12 +08:00
parent d524fd741b
commit c4d17df71c
7 changed files with 171 additions and 342 deletions

View File

@ -9,6 +9,7 @@ import (
"github.com/xiaoxue/memoryweave/internal/consolidate"
"github.com/xiaoxue/memoryweave/internal/governance"
"github.com/xiaoxue/memoryweave/internal/metrics"
"github.com/xiaoxue/memoryweave/internal/selfoptimize"
"github.com/xiaoxue/memoryweave/internal/storage"
)
@ -87,6 +88,10 @@ func (cp *ConsolidationPipeline) RunWithMode(mode string) (*ConsolidationReport,
rustReport.Quality.Score, rustReport.Quality.LowInfo, rustReport.Quality.Hallucinations))
}
log.Printf("[consolidation] Rust sidecar 完成: clusters=%d noise=%d", rustReport.Clusters, rustReport.Noise)
// Phase F: 整合完成后更新 Prometheus metrics
metrics.TotalMemories.Set(float64(rustReport.Clusters * 10)) // 估算
metrics.SyncFromDashboard(selfoptimize.Dash.Metrics())
} else {
log.Printf("[consolidation] Rust sidecar 不可用 (%v),降级为 Go 启发式", err)
goReport, goErr := cp.runGoFallback()
@ -106,6 +111,17 @@ func (cp *ConsolidationPipeline) RunWithMode(mode string) (*ConsolidationReport,
report = goReport
}
// Phase F: 整合完成后更新总记忆数指标(通过 LanceDB stats
if stats, err := cp.ldb.Stats(); err == nil {
if total, ok := stats["total_memories"].(float64); ok {
metrics.TotalMemories.Set(total)
}
if total, ok := stats["total_episodes"].(float64); ok {
metrics.TotalEpisodes.Set(total)
}
}
metrics.SyncFromDashboard(selfoptimize.Dash.Metrics())
// ─── 图谱后处理:修剪 + PageRank无论 Rust/Go 都执行)──
pruned := cp.runGraphMaintenance()
report.GraphPruned += pruned

View File

@ -3,8 +3,10 @@ package routes
import (
"encoding/json"
"log"
"net/http"
"github.com/xiaoxue/memoryweave/internal/metrics"
"github.com/xiaoxue/memoryweave/internal/selfoptimize"
"github.com/xiaoxue/memoryweave/internal/storage"
)
@ -39,6 +41,19 @@ func (fa *FeedbackAPI) MarkUseful(w http.ResponseWriter, r *http.Request) {
"user_feedback", "success", "",
)
fa.LanceDB.IncrementUseful(req.MemoryID)
// Phase F: 质量监控 — 每次反馈后检查低质量记忆
feedbackCount := selfoptimize.Dash.UsefulCount + selfoptimize.Dash.NotUsefulCount
qualityScore := selfoptimize.Dash.QualityScore()
if record := selfoptimize.QualityMonitor.Check(req.MemoryID, qualityScore, feedbackCount); record != nil {
log.Printf("[quality] low-quality memory flagged: id=%s score=%.2f status=%s",
record.MemoryID, record.Score, record.Status)
// 同步 DeprecatedPerDay 指标
metrics.DeprecatedPerDay.Set(float64(selfoptimize.Dash.DeprecatedToday))
}
// 同步 Dashboard 指标到 Prometheus
metrics.SyncFromDashboard(selfoptimize.Dash.Metrics())
respond(w, 200, map[string]string{"status": "ok", "memory_id": req.MemoryID})
}
@ -64,6 +79,18 @@ func (fa *FeedbackAPI) MarkNotUseful(w http.ResponseWriter, r *http.Request) {
"user_feedback", "failure", "",
)
fa.LanceDB.IncrementNotUseful(req.MemoryID)
// Phase F: 质量监控 — negative feedback 触发低质量检测
feedbackCount := selfoptimize.Dash.UsefulCount + selfoptimize.Dash.NotUsefulCount
qualityScore := selfoptimize.Dash.QualityScore()
if record := selfoptimize.QualityMonitor.Check(req.MemoryID, qualityScore, feedbackCount); record != nil {
log.Printf("[quality] low-quality memory flagged: id=%s score=%.2f status=%s",
record.MemoryID, record.Score, record.Status)
metrics.DeprecatedPerDay.Set(float64(selfoptimize.Dash.DeprecatedToday))
}
// 同步 Dashboard 指标到 Prometheus
metrics.SyncFromDashboard(selfoptimize.Dash.Metrics())
respond(w, 200, map[string]string{"status": "ok", "memory_id": req.MemoryID})
}

View File

@ -1,231 +0,0 @@
// 织忆 MemoryWeave — Consolidation 流水线
// 每次蒸馏后自动执行:合并相似 → 扫描冲突 → 模式挖掘 → 图谱更新
package distill
import (
"sort"
"sync"
"time"
)
// ConsolidationStep 整合步骤
type ConsolidationStep string
const (
StepMergeSimilar ConsolidationStep = "merge_similar"
StepScanConflicts ConsolidationStep = "scan_conflicts"
StepPatternMine ConsolidationStep = "pattern_mine"
StepGraphUpdate ConsolidationStep = "graph_update"
)
// ConsolidationReport 整合报告
type ConsolidationReport struct {
Timestamp time.Time `json:"timestamp"`
DurationMs int64 `json:"duration_ms"`
Merged int `json:"merged"`
ConflictsFound int `json:"conflicts_found"`
PatternsFound int `json:"patterns_found"`
GraphUpdates int `json:"graph_updates"`
Status string `json:"status"` // ok / partial
}
// Consolidator 整合器
type Consolidator struct {
mu sync.Mutex
// 合并阈值
mergeThreshold float64 // 向量相似度 > 0.8 → 合并
// 模式挖掘阈值
patternMinCount int // 连续 3+ 条同类型 → 提取 pattern
// 统计
lastRun time.Time
totalMerged int
totalConflicts int
totalPatterns int
}
func NewConsolidator() *Consolidator {
return &Consolidator{
mergeThreshold: 0.8,
patternMinCount: 3,
}
}
// Run 执行全流程
func (c *Consolidator) Run(distilled []DistillResult) *ConsolidationReport {
c.mu.Lock()
defer c.mu.Unlock()
start := time.Now()
report := &ConsolidationReport{Timestamp: start, Status: "ok"}
// Step 1: 合并相似记忆
merged := c.mergeSimilar(distilled)
report.Merged = merged
// Step 2: 扫描冲突
conflicts := c.scanConflicts(distilled)
report.ConflictsFound = conflicts
// Step 3: 模式挖掘
patterns := c.minePatterns(distilled)
report.PatternsFound = patterns
// Step 4: 图谱更新
graphUpdates := c.updateGraph(distilled)
report.GraphUpdates = graphUpdates
c.lastRun = start
c.totalMerged += merged
c.totalConflicts += conflicts
c.totalPatterns += patterns
report.DurationMs = time.Since(start).Milliseconds()
return report
}
// mergeSimilar 合并相似记忆(向量相似度 > 阈值 → 保留最新)
func (c *Consolidator) mergeSimilar(distilled []DistillResult) int {
// 在实际实现中,通过向量比较相似度
// 此处返回估计值
merged := 0
for i := 0; i < len(distilled); i++ {
for j := i + 1; j < len(distilled); j++ {
// 比较 (i, j) 向量的余弦相似度
if c.shouldMerge(distilled[i], distilled[j]) {
merged++
}
}
}
return merged
}
func (c *Consolidator) shouldMerge(a, b DistillResult) bool {
// 检查是否有共同事实
if len(a.Facts) == 0 || len(b.Facts) == 0 {
return false
}
// 简化: Jaccard 相似度 > 0.5 → 可能相似
common := 0
for _, fa := range a.Facts {
for _, fb := range b.Facts {
if fa == fb {
common++
}
}
}
jaccard := float64(common) / float64(len(a.Facts)+len(b.Facts)-common)
return jaccard > 0.5
}
// scanConflicts 扫描冲突
func (c *Consolidator) scanConflicts(distilled []DistillResult) int {
conflicts := 0
// 遍历蒸馏结果,检查同 entity 的矛盾
for i := 0; i < len(distilled); i++ {
for j := i + 1; j < len(distilled); j++ {
if c.isConflict(distilled[i], distilled[j]) {
conflicts++
}
}
}
return conflicts
}
func (c *Consolidator) isConflict(a, b DistillResult) bool {
// 有共享实体但事实内容不同 → 潜在冲突
sharedEntities := 0
for _, ea := range a.Entities {
for _, eb := range b.Entities {
if ea.Name == eb.Name && ea.Type == eb.Type {
sharedEntities++
}
}
}
if sharedEntities == 0 {
return false
}
// 有共享实体但事实不同 → 冲突
for _, fa := range a.Facts {
for _, fb := range b.Facts {
if fa == fb {
return false // 相同事实,不是冲突
}
}
}
return true
}
// minePatterns 模式挖掘(连续 3+ 条同类型 → 提取 pattern
func (c *Consolidator) minePatterns(distilled []DistillResult) int {
if len(distilled) < c.patternMinCount {
return 0
}
patterns := 0
// 按 category 分组
byCategory := make(map[string][]DistillResult)
for _, d := range distilled {
cat := "general"
byCategory[cat] = append(byCategory[cat], d)
}
// 每组 >= patternMinCount → 提取 pattern
for _, group := range byCategory {
if len(group) >= c.patternMinCount {
patterns++
}
}
return patterns
}
// updateGraph 图谱更新
func (c *Consolidator) updateGraph(distilled []DistillResult) int {
updates := 0
for _, result := range distilled {
updates += len(result.Entities)
}
return updates
}
// ─── 统计 ────────────────────────────────────────────────
type ConsolidationStats struct {
TotalMerged int `json:"total_merged"`
TotalConflicts int `json:"total_conflicts"`
TotalPatterns int `json:"total_patterns"`
LastRunAgo string `json:"last_run_ago"`
MergeRate float64 `json:"merge_rate"`
}
func (c *Consolidator) Stats() *ConsolidationStats {
c.mu.Lock()
defer c.mu.Unlock()
ago := ""
if !c.lastRun.IsZero() {
ago = time.Since(c.lastRun).Round(time.Second).String()
}
total := c.totalMerged + c.totalConflicts + c.totalPatterns
rate := 0.0
if total > 0 {
rate = float64(c.totalMerged) / float64(total)
}
return &ConsolidationStats{
TotalMerged: c.totalMerged,
TotalConflicts: c.totalConflicts,
TotalPatterns: c.totalPatterns,
LastRunAgo: ago,
MergeRate: rate,
}
}
// sortDistilled 按时间排序
func sortDistilled(distilled []DistillResult) {
sort.Slice(distilled, func(i, j int) bool {
return len(distilled[i].Facts) > len(distilled[j].Facts)
})
}

View File

@ -13,6 +13,9 @@ import (
"strings"
"sync"
"time"
"github.com/xiaoxue/memoryweave/internal/metrics"
"github.com/xiaoxue/memoryweave/internal/selfoptimize"
)
// ─── 类型定义 ────────────────────────────────────────────────
@ -159,6 +162,9 @@ func (e *Engine) flush() {
log.Printf("[distill] flush START: batch=%d items, dailyUsed=%d/%d, endpoint=%s, model=%s",
len(batch), e.dailyUsed, e.dailyLimit, e.LLMEndpoint, e.LLMModel)
// Phase F: 更新 LLM 调用计数
metrics.DistillLLMCallsToday.Set(float64(e.dailyUsed))
// 成本控制检查
e.checkDailyLimit()
if e.dailyUsed >= e.dailyLimit {
@ -496,6 +502,18 @@ func (e *Engine) emitResult(input DistillInput, result DistillResult) {
if OnDistillComplete != nil {
OnDistillComplete(input, result)
}
// Phase F: 蒸馏完成后更新队列深度和冲突计数
metrics.DistillQueueDepth.Set(float64(e.QueueLen()))
// Phase F: 对低分记忆触发质量监控检查 (score < 0.7 的记忆视为潜在低质量)
if result.Overall > 0 && result.Overall < 0.7 {
feedbackCount := selfoptimize.Dash.UsefulCount + selfoptimize.Dash.NotUsefulCount
if record := selfoptimize.QualityMonitor.Check(input.EpisodeID, result.Overall, feedbackCount); record != nil {
log.Printf("[quality] distill low-score flagged: episode=%s score=%.2f status=%s",
input.EpisodeID, result.Overall, record.Status)
}
}
}
// emitResults 批量发送

View File

@ -317,6 +317,19 @@ func (cd *ConflictDetector) ListActive() []*Conflict {
return list
}
// PendingCount 返回待处理冲突数
func (cd *ConflictDetector) PendingCount() int {
cd.mu.RLock()
defer cd.mu.RUnlock()
n := 0
for _, c := range cd.active {
if c.Status == "pending" {
n++
}
}
return n
}
// Resolve 解决冲突
func (cd *ConflictDetector) Resolve(id, resolution, winner string) error {
cd.mu.Lock()

View File

@ -41,7 +41,21 @@ if ! timeout 300 tar czf "$BACKUP_DIR/lance-data.tar.gz" \
exit 1
fi
# 6. 验证
# 7. 清理旧备份保留最近7天
echo "[$TIMESTAMP] 清理过期备份..."
DELETED=0
for d in "$BACKUP_ROOT"/*/; do
[ -d "$d" ] || continue
age_days=$(($(date +%s) - $(stat -c %Y "$d") + 86399) / 86400)
if [ "$age_days" -gt 7 ]; then
rm -rf "$d"
echo "[$TIMESTAMP] 已删除过期备份: $d"
DELETED=$((DELETED + 1))
fi
done
echo "[$TIMESTAMP] 清理完成,删除了 $DELETED 个过期备份"
# 8. 验证
echo "[$TIMESTAMP] 验证备份..."
for f in graph.db memoryweave.db lance-data.tar.gz; do
if [ ! -s "$BACKUP_DIR/$f" ]; then
@ -50,7 +64,7 @@ for f in graph.db memoryweave.db lance-data.tar.gz; do
fi
done
# 7. 打印结果
# 9. 打印结果
SIZE=$(du -sh "$BACKUP_DIR" | cut -f1)
echo "[$TIMESTAMP] 备份成功 | 路径: $BACKUP_DIR | 大小: $SIZE"
echo "[$TIMESTAMP] 织忆备份结果: OK | 路径: $BACKUP_DIR | 时间戳: $TIMESTAMP" > "$HOME/projects/memoryweave/backups/backup-$TIMESTAMP.log"

View File

@ -1,124 +1,96 @@
#!/bin/bash
# ====== 织忆恢复脚本 ======
# 从 server Gitea 恢复所有数据
# 用法:./restore.sh [backup_date]
# backup_date: 可选,默认最新
# 织忆恢复脚本 — 从备份恢复数据
# 用法: ./restore.sh [backup_dir]
# backup_dir: 可选,如不填则列出可用备份供选择
set -e
BACKUP_DATE="${1:-latest}"
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
GITEA_BASE="http://192.168.123.11:3000/xiaoxue_admin"
BACKUP_ROOT="/home/muc/backups/memoryweave"
DATA_DIR="/var/lib/memoryweave"
STAMP=$(date +%Y%m%d-%H%M%S)
echo "[$(date '+%Y-%m-%d %H:%M:%S')] ===== 开始恢复织忆 ====="
# 1. Clone 配置仓库
echo "[1/5] 拉取 Hermes 配置..."
if [ -d "${PROJECT_ROOT}/zhiyi-config" ]; then
cd "${PROJECT_ROOT}/zhiyi-config"
git pull origin main
if [ -n "$1" ]; then
BACKUP_DIR="$1"
else
git clone "${GITEA_BASE}/zhiyi-config.git" "${PROJECT_ROOT}/zhiyi-config"
fi
# 2. 恢复核心人格文件
echo "[2/5] 恢复人格文件..."
cp "${PROJECT_ROOT}/zhiyi-config/SOUL.md" "${HOME}/.hermes/SOUL.md"
cp "${PROJECT_ROOT}/zhiyi-config/AGENTS.md" "${HOME}/.hermes/AGENTS.md"
cp "${PROJECT_ROOT}/zhiyi-config/MEMORY.md" "${HOME}/.hermes/MEMORY.md"
cp "${PROJECT_ROOT}/zhiyi-config/USER.md" "${HOME}/.hermes/USER.md"
echo " 人格文件恢复完成"
# 3. 恢复配置文件
echo "[3/5] 恢复配置文件..."
[ -f "${PROJECT_ROOT}/zhiyi-config/config.yaml" ] && \
cp "${PROJECT_ROOT}/zhiyi-config/config.yaml" "${HOME}/.hermes/config.yaml"
[ -f "${PROJECT_ROOT}/zhiyi-config/.env" ] && \
cp "${PROJECT_ROOT}/zhiyi-config/.env" "${HOME}/.hermes/.env"
[ -f "${PROJECT_ROOT}/zhiyi-config/auth.json" ] && \
cp "${PROJECT_ROOT}/zhiyi-config/auth.json" "${HOME}/.hermes/auth.json"
echo " 配置文件恢复完成"
# 4. 恢复 skills / cron
echo "[4/5] 恢复 skills 和 cron..."
[ -d "${PROJECT_ROOT}/zhiyi-config/skills" ] && \
rsync -a "${PROJECT_ROOT}/zhiyi-config/skills/" "${HOME}/.hermes/skills/"
[ -d "${PROJECT_ROOT}/zhiyi-config/cron" ] && \
rsync -a "${PROJECT_ROOT}/zhiyi-config/cron/" "${HOME}/.hermes/cron/"
[ -f "${PROJECT_ROOT}/zhiyi-config/kanban.db" ] && \
cp "${PROJECT_ROOT}/zhiyi-config/kanban.db" "${HOME}/.hermes/kanban.db"
echo " skills/cron 恢复完成"
# 5. 恢复织忆数据
echo "[5/5] 恢复织忆数据..."
# Clone 数据仓库(如果需要)
if [ ! -d "${PROJECT_ROOT}/zhiyi-backup" ]; then
git clone "${GITEA_BASE}/zhiyi-backup.git" "${PROJECT_ROOT}/zhiyi-backup"
fi
cd "${PROJECT_ROOT}/zhiyi-backup"
# 找最新备份
if [ "${BACKUP_DATE}" = "latest" ]; then
BACKUP_FILE=$(ls -t *.tar.gz 2>/dev/null | head -1)
else
BACKUP_FILE="zhiyi_backup_${BACKUP_DATE}.tar.gz"
fi
if [ -n "${BACKUP_FILE}" ] && [ -f "${BACKUP_FILE}" ]; then
echo " 使用备份: ${BACKUP_FILE}"
# 解压到临时目录
TMP_DIR="/tmp/zhiyi_restore_$$"
mkdir -p "${TMP_DIR}"
tar -xzf "${BACKUP_FILE}" -C "${TMP_DIR}"
DATA_DIR=$(ls -d "${TMP_DIR}"/zhiyi_backup_* 2>/dev/null | head -1)
if [ -n "${DATA_DIR}" ] && [ -d "${DATA_DIR}/memoryweave" ]; then
sudo rsync -a "${DATA_DIR}/memoryweave/" /var/lib/memoryweave/
sudo chown -R root:root /var/lib/memoryweave/
echo " 数据恢复完成: $(du -sh /var/lib/memoryweave/ | cut -f1)"
else
echo " ERROR: 备份格式错误,无法恢复"
echo "=== 可用备份 ==="
backups=($(ls -t "$BACKUP_ROOT"/))
if [ ${#backups[@]} -eq 0 ]; then
echo "错误: 没有找到任何备份"
exit 1
fi
rm -rf "${TMP_DIR}"
for i in "${!backups[@]}"; do
echo " [$i] ${backups[$i]}"
done
echo ""
read -p "选择备份 [0-${#backups[@]}-1]: " idx
BACKUP_DIR="$BACKUP_ROOT/${backups[$idx]}"
fi
if [ ! -d "$BACKUP_DIR" ]; then
echo "错误: 备份目录不存在: $BACKUP_DIR"
exit 1
fi
echo "[$STAMP] 织忆恢复开始..."
echo "[$STAMP] 使用备份: $BACKUP_DIR"
read -p "即将停止 zhiyid 并恢复数据,确认? [y/N]: " confirm
[ "$confirm" = "y" ] || { echo "取消"; exit 0; }
# 停止服务
echo "[$STAMP] 停止 zhiyid..."
systemctl --user stop zhiyid 2>/dev/null || sudo systemctl stop zhiyid 2>/dev/null || true
# 备份当前数据(出事前的快照)
CURRENT_BACKUP="$DATA_DIR/pre-restore-$STAMP"
echo "[$STAMP] 备份当前数据到: $CURRENT_BACKUP"
mkdir -p "$CURRENT_BACKUP"
cp -r "$DATA_DIR"/*.db "$DATA_DIR"/*.lance "$CURRENT_BACKUP/" 2>/dev/null || true
# 恢复 SQLite
for db in graph.db memoryweave.db; do
echo "[$STAMP] 恢复 $db..."
if [ -f "$BACKUP_DIR/$db" ]; then
cp "$BACKUP_DIR/$db" "$DATA_DIR/$db"
else
echo "[$STAMP] 警告: 备份中无 $db,跳过"
fi
done
# 恢复 LanceDB
echo "[$STAMP] 恢复 LanceDB 数据..."
if [ -f "$BACKUP_DIR/lance-data.tar.gz" ]; then
tar xzf "$BACKUP_DIR/lance-data.tar.gz" -C "$DATA_DIR"
else
echo " WARN: 未找到备份文件 ${BACKUP_FILE},跳过数据恢复"
echo "[$STAMP] 警告: 备份中无 lance-data.tar.gz跳过"
fi
# 6. 重启服务
echo ""
echo "[完成] 准备重启服务..."
# 验证
echo "[$STAMP] 验证恢复..."
ok=true
for db in graph.db memoryweave.db; do
if [ -s "$DATA_DIR/$db" ]; then
echo "[$STAMP] $db: OK ($(du -sh "$DATA_DIR/$db" | cut -f1))"
else
echo "[$STAMP] $db: 失败"
ok=false
fi
done
for lance in episodes.lance memories.lance tombstones.lance; do
if [ -s "$DATA_DIR/$lance" ]; then
echo "[$STAMP] $lance: OK"
fi
done
# 检查 zhiyi 进程
if pgrep -f "zhiyi" > /dev/null; then
echo " 重启 zhiyi..."
pkill -f "zhiyi" 2>/dev/null || true
sleep 2
fi
# 启动 zhiyi
cd "${PROJECT_ROOT}/go"
if [ -f "./zhiyi" ]; then
nohup ./zhiyi >> "${HOME}/.hermes/logs/zhiyi.log" 2>&1 &
sleep 3
else
echo " WARN: zhiyi binary 不存在,需要重新编译"
fi
# 7. 健康检查
echo ""
echo "健康检查..."
# 重启
echo "[$STAMP] 重启 zhiyid..."
systemctl --user start zhiyid 2>/dev/null || sudo systemctl start zhiyid 2>/dev/null || true
sleep 2
if curl -s http://localhost:7821/health > /dev/null 2>&1; then
echo "✓ 织忆服务正常"
# 健康检查
if curl -sf http://127.0.0.1:7821/health > /dev/null 2>&1; then
echo "[$STAMP] ✅ 恢复成功zhiyid 健康检查通过"
else
echo "✗ 织忆服务异常,请检查日志"
tail -20 "${HOME}/.hermes/logs/zhiyi.log" 2>/dev/null || true
echo "[$STAMP] ⚠️ zhiyid 未响应,请手动检查"
fi
echo ""
echo "[$(date '+%Y-%m-%d %H:%M:%S')] ===== 恢复完成 ====="
echo "[$STAMP] 完成。当前数据快照已保存到: $CURRENT_BACKUP"