139 lines
5.7 KiB
Bash
139 lines
5.7 KiB
Bash
#!/bin/bash
|
||
# fix-state-db-restore.sh — state.db 最终修复（2026-09-03 深夜）
|
||
# 方案：19:26 healthy 备份为基底 + messages/system_prompts 增量补回 = 零丢失
|
||
# 背景：live state.db 所有表数据可读，但 messages 表 b-tree 内部节点 page 63262
|
||
# 双重引用（VACUUM/REINDEX/FTS rebuild 均无法修复，官方 .recover 缺 dbpage）
|
||
# sessions 两库完全一致(365)，messages 差 285 条(id>100564)，system_prompts 差 1 条
|
||
# 用法：从 gateway 外部/后台 shell 跑；脚本自管 gateway 启停
|
||
set -uo pipefail
|
||
DB=/home/muc/.hermes/state.db
|
||
BASE=/home/muc/.hermes/state.db.before-fix-deploy-20260903_192633 # healthy 基底
|
||
TS=$(date +%Y%m%d_%H%M%S)
|
||
LOG=/tmp/fix-state-db-restore-$TS.log
|
||
TMP=/tmp/restored-$TS.db
|
||
PROG=/tmp/state-db-fix-progress.md
|
||
|
||
log() { echo "[$(date '+%H:%M:%S')] $*" | tee -a "$LOG"; }
|
||
|
||
log "=== state.db 最终修复开始 (TS=$TS) ==="
|
||
|
||
# 0. 等 8 秒，确保发起方的消息已发出（本脚本可能由 gateway 内 agent 触发）
|
||
sleep 8
|
||
|
||
# 1. 停 gateway + 防 auto-restart
|
||
GW=$(systemctl --user show hermes-gateway -p MainPID --value 2>/dev/null)
|
||
if [ -n "$GW" ] && [ "$GW" != "0" ]; then
|
||
log "停止 gateway (PID $GW)..."
|
||
systemctl --user stop hermes-gateway
|
||
sleep 5
|
||
systemctl --user reset-failed hermes-gateway 2>/dev/null
|
||
fi
|
||
ACT=$(systemctl --user is-active hermes-gateway 2>/dev/null)
|
||
log "gateway 状态: $ACT"
|
||
if [ "$ACT" = "active" ] || [ "$ACT" = "activating" ]; then
|
||
log "❌ gateway 仍在跑/被拉起，中止修复避免抢 DB"
|
||
exit 1
|
||
fi
|
||
|
||
# 2. 备份 live（修复前最新状态，含 gateway 停前最后写入）
|
||
log "备份 live → $DB.pre-restore-$TS"
|
||
cp "$DB" "$DB.pre-restore-$TS"
|
||
log " 备份: $(du -h "$DB.pre-restore-$TS" | cut -f1)"
|
||
|
||
# 3. 从健康基底重建恢复库 + 增量补回
|
||
log "从 $BASE 重建恢复库 + 增量补回..."
|
||
cp "$BASE" "$TMP" || { log "❌ cp 基底失败"; exit 1; }
|
||
/usr/bin/python3 - "$TMP" "$DB" <<'PYEOF' 2>>"$LOG" || { log "❌ 增量补回失败"; exit 1; }
|
||
import sqlite3, sys
|
||
dst_path, src_path = sys.argv[1], sys.argv[2]
|
||
src = sqlite3.connect(f'file:{src_path}?mode=ro', uri=True, timeout=120)
|
||
dst = sqlite3.connect(dst_path, timeout=120)
|
||
src.execute("PRAGMA busy_timeout=60000"); dst.execute("PRAGMA busy_timeout=60000")
|
||
|
||
# 1) messages 增量（id > 基底 max）
|
||
max_id_dst = dst.execute("SELECT COALESCE(MAX(id),0) FROM messages").fetchone()[0]
|
||
cols = [d[0] for d in src.execute("SELECT * FROM messages LIMIT 1").description]
|
||
ph = ",".join("?"*len(cols))
|
||
rows = src.execute(f"SELECT * FROM messages WHERE id > ? ORDER BY id", (max_id_dst,)).fetchall()
|
||
for r in rows:
|
||
dst.execute(f"INSERT OR IGNORE INTO messages ({','.join(cols)}) VALUES ({ph})", r)
|
||
dst.commit()
|
||
print(f"messages 增量: {len(rows)} 条 (id {max_id_dst}+1 → {src.execute('SELECT MAX(id) FROM messages').fetchone()[0]})")
|
||
|
||
# 2) system_prompts 增量
|
||
hashes_dst = set(x[0] for x in dst.execute("SELECT hash FROM system_prompts"))
|
||
added = 0
|
||
for h, p in src.execute("SELECT hash, prompt FROM system_prompts"):
|
||
if h not in hashes_dst:
|
||
try:
|
||
dst.execute("INSERT INTO system_prompts (hash, prompt) VALUES (?,?)", (h,p)); added += 1
|
||
except Exception as e:
|
||
print(f" sp 跳过 {h}: {e}")
|
||
dst.commit()
|
||
print(f"system_prompts 增量: {added} 条")
|
||
|
||
# 3) FTS rebuild 对齐
|
||
dst.execute("INSERT INTO messages_fts(messages_fts) VALUES('rebuild')")
|
||
dst.commit()
|
||
try:
|
||
dst.execute("INSERT INTO messages_fts_trigram(messages_fts_trigram) VALUES('rebuild')")
|
||
dst.commit()
|
||
except Exception as e:
|
||
print(f"trigram rebuild 跳过: {e}")
|
||
dst.rollback()
|
||
|
||
# 4) 验证
|
||
ic = dst.execute("PRAGMA integrity_check").fetchall()
|
||
print(f"integrity_check: {ic}")
|
||
if ic != [('ok',)]:
|
||
print("❌ 恢复库不健康，中止")
|
||
sys.exit(1)
|
||
for t in ["messages","sessions","system_prompts","delivery_obligations"]:
|
||
print(f" {t}: {dst.execute(f'SELECT count(*) FROM {t}').fetchone()[0]}")
|
||
print(f" max_msg_id: {dst.execute('SELECT MAX(id) FROM messages').fetchone()[0]} (src: {src.execute('SELECT MAX(id) FROM messages').fetchone()[0]})")
|
||
src.close(); dst.close()
|
||
PYEOF
|
||
PY_EXIT=$?
|
||
if [ $PY_EXIT -ne 0 ]; then
|
||
log "❌ 恢复库构建失败 (exit=$PY_EXIT) — 保留 live DB 不动，可回滚备份"
|
||
exit 1
|
||
fi
|
||
log "✅ 恢复库 integrity=ok"
|
||
|
||
# 4. 替换（清旧 WAL/SHM 防串扰）
|
||
log "替换 state.db..."
|
||
rm -f "$DB-wal" "$DB-shm"
|
||
mv "$DB" "$DB.pre-replace-$TS" || { log "❌ mv live 失败"; exit 1; }
|
||
mv "$TMP" "$DB" || { log "❌ mv 恢复库失败，回滚"; mv "$DB.pre-replace-$TS" "$DB"; exit 1; }
|
||
log " 已替换（live 保留在 $DB.pre-replace-$TS）"
|
||
|
||
# 5. 启动 gateway
|
||
log "启动 hermes-gateway..."
|
||
systemctl --user start hermes-gateway
|
||
sleep 10
|
||
STATUS=$(systemctl --user is-active hermes-gateway 2>/dev/null)
|
||
NEW_PID=$(systemctl --user show hermes-gateway -p MainPID --value 2>/dev/null)
|
||
log " gateway: $STATUS PID=$NEW_PID"
|
||
|
||
# 6. 最终验证（gateway 已起，只读查）
|
||
IC=$(sqlite3 "$DB" "PRAGMA integrity_check;" 2>&1 | head -1)
|
||
M=$(sqlite3 "$DB" "SELECT count(*) FROM messages;" 2>&1)
|
||
S=$(sqlite3 "$DB" "SELECT count(*) FROM sessions;" 2>&1)
|
||
log " 最终 integrity: $IC | messages: $M | sessions: $S"
|
||
|
||
# 7. 进度文件（防失忆）
|
||
cat > "$PROG" <<EOF
|
||
# state.db 修复进度（更新 $TS）
|
||
- 方案: 19:26 healthy 备份 + messages/system_prompts 增量补回（零丢失）
|
||
- 基底: $BASE
|
||
- live 备份: $DB.pre-restore-$TS
|
||
- live 替换前保留: $DB.pre-replace-$TS
|
||
- 结果: integrity=$IC messages=$M sessions=$S
|
||
- gateway: $STATUS PID=$NEW_PID
|
||
- 完成时间: $(date '+%F %T')
|
||
- 下一步: journalctl --user -u hermes-gateway --since '5 min ago' | grep -i malformed （应无报错）
|
||
EOF
|
||
|
||
log "=== 修复完成 ==="
|
||
echo "--- 完整日志: $LOG ---"
|