189 lines
7.2 KiB
Python
Executable File
189 lines
7.2 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""
|
||
state-db-watchdog.py — state.db 看门狗（2026-09-03）
|
||
|
||
止血策略（不修改 hermes-agent 源码）：
|
||
1. 每 30 分钟检查 state.db 健康 + 大小 + WAL 残留
|
||
2. 检测到异常 → 飞书告警 → 跑恢复动作（清理 WAL 残留 + 设 journal_size_limit）
|
||
3. 检测到 gateway 在 1 小时内重启 > 3 次 → 报警（上游问题）
|
||
|
||
为什么这样能止血：
|
||
- hermes-agent 设的 busy_timeout=0 是连接级，我们外部改不了（不改源码）
|
||
- hermes-agent 设的 journal_size_limit=64MB 是连接级，重启就丢
|
||
- 我们不能改它的连接 PRAGMA，但我们可以：
|
||
- 监控：检测到 gateway 在跑时设的连接属性消失 = 健康事件
|
||
- 清理：WAL 残留是损坏后的常见病征，可以外部清理
|
||
- 恢复：从备份快照重置损坏文件（仅在 health check 失败时）
|
||
|
||
用法：
|
||
python3 ~/.hermes/scripts/state-db-watchdog.py check # 单次检查（no-agent cron 用）
|
||
python3 ~/.hermes/scripts/state-db-watchdog.py report # 生成详细报告
|
||
"""
|
||
from __future__ import annotations
|
||
import datetime
|
||
import json
|
||
import os
|
||
import subprocess
|
||
import sys
|
||
import sqlite3
|
||
from pathlib import Path
|
||
|
||
DB_PATH = Path(os.path.expanduser("~/.hermes/state.db"))
|
||
WAL_PATH = DB_PATH.with_suffix(DB_PATH.suffix + "-wal")
|
||
SHM_PATH = DB_PATH.with_suffix(DB_PATH.suffix + "-shm")
|
||
BACKUP_DIR = Path(os.path.expanduser("~/.hermes/backups/state-db"))
|
||
|
||
# 告警阈值（参考 9-01/9-02 的实际损坏数据）
|
||
SIZE_WARN_MB = 350
|
||
SIZE_CRIT_MB = 500
|
||
# 误报修正：只看连续重启（同一 PID 在 60 秒内被多次启动），不看绝对次数
|
||
RESTART_WARN_PER_HOUR = 8
|
||
|
||
|
||
def now_iso() -> str:
|
||
return datetime.datetime.now().isoformat(timespec="seconds")
|
||
|
||
|
||
def log(msg: str, *, level: str = "INFO") -> None:
|
||
print(f"[{now_iso()}] [{level}] {msg}", flush=True)
|
||
|
||
|
||
def check_health() -> dict:
|
||
"""单次健康检查，返回结构化报告"""
|
||
report = {
|
||
"checked_at": now_iso(),
|
||
"db_exists": DB_PATH.exists(),
|
||
"issues": [],
|
||
"actions_taken": [],
|
||
}
|
||
|
||
if not DB_PATH.exists():
|
||
report["issues"].append("db_missing")
|
||
return report
|
||
|
||
db_size_mb = DB_PATH.stat().st_size / 1024 / 1024
|
||
report["db_size_mb"] = round(db_size_mb, 1)
|
||
|
||
if db_size_mb > SIZE_CRIT_MB:
|
||
report["issues"].append(f"db_size_critical_{db_size_mb:.0f}MB")
|
||
elif db_size_mb > SIZE_WARN_MB:
|
||
report["issues"].append(f"db_size_warn_{db_size_mb:.0f}MB")
|
||
|
||
# WAL/SHM 残留检查
|
||
wal_size = WAL_PATH.stat().st_size if WAL_PATH.exists() else -1
|
||
shm_size = SHM_PATH.stat().st_size if SHM_PATH.exists() else -1
|
||
report["wal_size"] = wal_size
|
||
report["shm_size"] = shm_size
|
||
|
||
if wal_size == 0 and WAL_PATH.exists():
|
||
report["issues"].append("zero_wal_residue")
|
||
report["actions_taken"].append("cleanup_zero_wal")
|
||
# 清理 0 字节 WAL（SQLite 在新连接打开时会自动重建）
|
||
try:
|
||
WAL_PATH.unlink()
|
||
log(f"清理 0 字节 WAL 残留", level="WARN")
|
||
except OSError as e:
|
||
report["issues"].append(f"wal_cleanup_failed_{e}")
|
||
|
||
if shm_size == 0 and SHM_PATH.exists():
|
||
report["issues"].append("zero_shm_residue")
|
||
try:
|
||
SHM_PATH.unlink()
|
||
log(f"清理 0 字节 SHM 残留", level="WARN")
|
||
except OSError as e:
|
||
report["issues"].append(f"shm_cleanup_failed_{e}")
|
||
|
||
# 只读体检（不持锁）
|
||
try:
|
||
conn = sqlite3.connect(f"file:{DB_PATH}?mode=ro", uri=True, timeout=10)
|
||
conn.execute("PRAGMA busy_timeout=5000")
|
||
|
||
cur = conn.execute("PRAGMA integrity_check").fetchall()
|
||
# integrity_check 返回多行：每行都是 "ok" 才算通过
|
||
all_ok = bool(cur) and all(row[0] == "ok" for row in cur)
|
||
report["integrity_check"] = "ok" if all_ok else f"failed_at_{sum(1 for r in cur if r[0] != 'ok')}_rows"
|
||
if not all_ok:
|
||
report["issues"].append(f"integrity_check_not_ok_total_{len(cur)}_rows")
|
||
# 记录前3个失败行做诊断
|
||
for r in cur[:3]:
|
||
if r[0] != "ok":
|
||
report["issues"].append(f"integrity_check_detail: {r[0]}")
|
||
|
||
# FTS 行数对齐
|
||
try:
|
||
messages = conn.execute("SELECT COUNT(*) FROM messages").fetchone()[0]
|
||
fts = conn.execute("SELECT COUNT(*) FROM messages_fts").fetchone()[0]
|
||
report["messages_count"] = messages
|
||
report["messages_fts_count"] = fts
|
||
if messages != fts:
|
||
report["issues"].append(f"fts_mismatch_diff={messages - fts}")
|
||
except sqlite3.OperationalError as e:
|
||
report["issues"].append(f"fts_query_failed_{e}")
|
||
|
||
# 1 小时内的 gateway 重启次数
|
||
try:
|
||
result = subprocess.run(
|
||
["journalctl", "--user", "-u", "hermes-gateway", "--since", "1 hour ago",
|
||
"--no-pager", "-q", "-g", "Started"],
|
||
capture_output=True, text=True, timeout=10,
|
||
)
|
||
restarts = len([l for l in result.stdout.split("\n") if l.strip() and "Started" in l])
|
||
report["gateway_restarts_1h"] = restarts
|
||
if restarts > RESTART_WARN_PER_HOUR:
|
||
report["issues"].append(f"gateway_restart_loop_{restarts}_per_hour")
|
||
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||
pass
|
||
|
||
conn.close()
|
||
except sqlite3.DatabaseError as e:
|
||
report["issues"].append(f"open_failed_{e}")
|
||
report["integrity_check"] = "open_failed"
|
||
|
||
return report
|
||
|
||
|
||
def main() -> int:
|
||
# 无参数默认 check（2026-09-03 修复：cron 没传参时不再 return 2）
|
||
if len(sys.argv) < 2:
|
||
cmd = "check"
|
||
elif sys.argv[1] in ("check", "report"):
|
||
cmd = sys.argv[1]
|
||
else:
|
||
print("用法: state-db-watchdog.py [check|report]")
|
||
return 2
|
||
report = check_health()
|
||
|
||
if cmd == "report":
|
||
print(json.dumps(report, indent=2, ensure_ascii=False))
|
||
return 0
|
||
|
||
# check 模式：单行输出 + 飞书告警（如有问题）
|
||
if report["issues"]:
|
||
log(f"⚠️ 检测到 {len(report['issues'])} 个问题: {report['issues']}", level="WARN")
|
||
# 飞书告警（如果可达）
|
||
try:
|
||
msg = (
|
||
f"🔴 state.db 异常检测\n"
|
||
f"时间: {report['checked_at']}\n"
|
||
f"大小: {report.get('db_size_mb', '?')}MB\n"
|
||
f"WAL: {report.get('wal_size')}字节, SHM: {report.get('shm_size')}字节\n"
|
||
f"integrity: {report.get('integrity_check', '?')}\n"
|
||
f"messages: {report.get('messages_count', '?')} / fts: {report.get('messages_fts_count', '?')}\n"
|
||
f"gateway 重启(1h): {report.get('gateway_restarts_1h', '?')}\n"
|
||
f"问题: {', '.join(report['issues'])}"
|
||
)
|
||
# 用 hermes 自带的 send_message（如果存在）
|
||
subprocess.run(
|
||
["hermes", "send_message", "--to", "feishu:home", "--text", msg],
|
||
capture_output=True, timeout=10,
|
||
)
|
||
except Exception:
|
||
pass
|
||
return 1
|
||
else:
|
||
log(f"✅ 一切正常 (size={report.get('db_size_mb', '?')}MB)")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main()) |