auto-snapshot 2026-09-03 16:03:05
This commit is contained in:
parent
83ad0bcb1a
commit
7846b4c536
1235
config.yaml
1235
config.yaml
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,65 @@
|
|||
#!/usr/bin/env bash
|
||||
# state.db 回滚脚本(2026-09-03 主人 C 方案带的回滚保险)
|
||||
# **必须从独立终端跑**(不能从 gateway 内部跑,会被杀)
|
||||
# 用法:bash ~/.hermes/scripts/restore-state-db.sh
|
||||
set -u
|
||||
|
||||
SNAP="/home/muc/.hermes/backups/state-db-snap-20260903_153655"
|
||||
DB="/home/muc/.hermes/state.db"
|
||||
WAL="/home/muc/.hermes/state.db-wal"
|
||||
SHM="/home/muc/.hermes/state.db-shm"
|
||||
JOBS="/home/muc/.hermes/cron/jobs.json"
|
||||
|
||||
LOG="/home/muc/.hermes/backups/state-db/restore-$(date +%Y%m%d_%H%M%S).log"
|
||||
mkdir -p "$(dirname "$LOG")"
|
||||
|
||||
log() { echo "[$(date '+%F %T')] $*" | tee -a "$LOG"; }
|
||||
|
||||
if [ ! -f "$SNAP/state.db" ]; then
|
||||
log "❌ 快照不存在: $SNAP/state.db"
|
||||
exit 2
|
||||
fi
|
||||
log "✅ 快照 OK: $SNAP"
|
||||
|
||||
# 0. 停 gateway
|
||||
log "[0/4] 停 gateway..."
|
||||
systemctl --user stop hermes-gateway 2>&1 | tee -a "$LOG"
|
||||
sleep 2
|
||||
|
||||
# 1. 原子替换
|
||||
log "[1/4] 还原 state.db..."
|
||||
cp -a "$SNAP/state.db" "$DB"
|
||||
[ -f "$SNAP/state.db-wal" ] && cp -a "$SNAP/state.db-wal" "$WAL" || rm -f "$WAL"
|
||||
[ -f "$SNAP/state.db-shm" ] && cp -a "$SNAP/state.db-shm" "$SHM" || rm -f "$SHM"
|
||||
log "✅ state.db 还原完成"
|
||||
|
||||
if [ -f "$SNAP/jobs.json" ]; then
|
||||
cp -a "$SNAP/jobs.json" "$JOBS"
|
||||
log "✅ jobs.json 还原完成"
|
||||
fi
|
||||
|
||||
# 2. 启 gateway(3 次重试)
|
||||
log "[2/4] 启 gateway(3 次重试)..."
|
||||
for i in 1 2 3; do
|
||||
systemctl --user start hermes-gateway 2>&1 | tee -a "$LOG"
|
||||
sleep 3
|
||||
if systemctl --user is-active hermes-gateway >/dev/null 2>&1; then
|
||||
log "✅ gateway 第 $i 次启动成功"
|
||||
break
|
||||
fi
|
||||
log "⚠️ 第 $i 次启动未就绪,等待后重试"
|
||||
sleep 3
|
||||
done
|
||||
|
||||
# 3. 健康检查
|
||||
log "[3/4] 健康检查..."
|
||||
sleep 5
|
||||
if systemctl --user is-active hermes-gateway >/dev/null 2>&1; then
|
||||
log "✅ gateway ACTIVE,回滚成功"
|
||||
log "📋 主人请重新发起对话(小唯已失忆)"
|
||||
exit 0
|
||||
else
|
||||
log "❌ gateway 还是没起来"
|
||||
journalctl --user -u hermes-gateway -n 30 --no-pager 2>&1 | tee -a "$LOG"
|
||||
exit 1
|
||||
fi
|
||||
|
|
@ -98,10 +98,16 @@ def check_health() -> dict:
|
|||
conn = sqlite3.connect(f"file:{DB_PATH}?mode=ro", uri=True, timeout=10)
|
||||
conn.execute("PRAGMA busy_timeout=5000")
|
||||
|
||||
cur = conn.execute("PRAGMA integrity_check").fetchone()
|
||||
report["integrity_check"] = cur[0] if cur else "unknown"
|
||||
if report["integrity_check"] != "ok":
|
||||
report["issues"].append(f"integrity_check_{report['integrity_check']}")
|
||||
cur = conn.execute("PRAGMA integrity_check").fetchall()
|
||||
# integrity_check 返回多行:每行都是 "ok" 才算通过
|
||||
all_ok = bool(cur) and all(row[0] == "ok" for row in cur)
|
||||
report["integrity_check"] = "ok" if all_ok else f"failed_at_{sum(1 for r in cur if r[0] != 'ok')}_rows"
|
||||
if not all_ok:
|
||||
report["issues"].append(f"integrity_check_not_ok_total_{len(cur)}_rows")
|
||||
# 记录前3个失败行做诊断
|
||||
for r in cur[:3]:
|
||||
if r[0] != "ok":
|
||||
report["issues"].append(f"integrity_check_detail: {r[0]}")
|
||||
|
||||
# FTS 行数对齐
|
||||
try:
|
||||
|
|
@ -137,11 +143,14 @@ def check_health() -> dict:
|
|||
|
||||
|
||||
def main() -> int:
|
||||
if len(sys.argv) < 2 or sys.argv[1] not in ("check", "report"):
|
||||
print("用法: state-db-watchdog.py {check|report}")
|
||||
# 无参数默认 check(2026-09-03 修复:cron 没传参时不再 return 2)
|
||||
if len(sys.argv) < 2:
|
||||
cmd = "check"
|
||||
elif sys.argv[1] in ("check", "report"):
|
||||
cmd = sys.argv[1]
|
||||
else:
|
||||
print("用法: state-db-watchdog.py [check|report]")
|
||||
return 2
|
||||
|
||||
cmd = sys.argv[1]
|
||||
report = check_health()
|
||||
|
||||
if cmd == "report":
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -541,15 +541,15 @@
|
|||
"created_at": "2026-08-09T12:25:17.949527+00:00",
|
||||
"created_by": "agent",
|
||||
"last_patched_at": "2026-09-01T13:35:53.465773+00:00",
|
||||
"last_reused_patch_generation": 14,
|
||||
"last_used_at": "2026-09-01T13:32:55.589413+00:00",
|
||||
"last_viewed_at": "2026-09-01T13:32:55.580024+00:00",
|
||||
"last_reused_patch_generation": 16,
|
||||
"last_used_at": "2026-09-03T02:48:57.774331+00:00",
|
||||
"last_viewed_at": "2026-09-03T02:48:57.769133+00:00",
|
||||
"patch_count": 16,
|
||||
"patch_generation": 16,
|
||||
"pinned": false,
|
||||
"state": "active",
|
||||
"use_count": 11,
|
||||
"view_count": 11
|
||||
"use_count": 12,
|
||||
"view_count": 12
|
||||
},
|
||||
"code-intelligence": {
|
||||
"archived_at": null,
|
||||
|
|
@ -1571,14 +1571,14 @@
|
|||
"created_by": null,
|
||||
"last_patched_at": "2026-08-28T13:02:49.751591+00:00",
|
||||
"last_reused_patch_generation": 2,
|
||||
"last_used_at": "2026-09-02T13:35:34.691639+00:00",
|
||||
"last_viewed_at": "2026-09-02T13:35:34.677091+00:00",
|
||||
"last_used_at": "2026-09-03T02:21:47.259335+00:00",
|
||||
"last_viewed_at": "2026-09-03T02:21:47.249868+00:00",
|
||||
"patch_count": 125,
|
||||
"patch_generation": 2,
|
||||
"pinned": false,
|
||||
"state": "active",
|
||||
"use_count": 158,
|
||||
"view_count": 157
|
||||
"use_count": 159,
|
||||
"view_count": 158
|
||||
},
|
||||
"hermes-desktop-kanban": {
|
||||
"archived_at": null,
|
||||
|
|
@ -1655,14 +1655,14 @@
|
|||
"created_by": null,
|
||||
"last_patched_at": "2026-08-12T05:13:32.138839+00:00",
|
||||
"last_reused_patch_generation": 1,
|
||||
"last_used_at": "2026-09-02T08:23:33.702578+00:00",
|
||||
"last_viewed_at": "2026-09-02T08:23:33.689904+00:00",
|
||||
"last_used_at": "2026-09-03T07:45:25.177181+00:00",
|
||||
"last_viewed_at": "2026-09-03T07:45:25.164298+00:00",
|
||||
"patch_count": 81,
|
||||
"patch_generation": 1,
|
||||
"pinned": false,
|
||||
"state": "active",
|
||||
"use_count": 141,
|
||||
"view_count": 130
|
||||
"use_count": 142,
|
||||
"view_count": 131
|
||||
},
|
||||
"hermes-venv-dependency-safety": {
|
||||
"archived_at": null,
|
||||
|
|
@ -1804,14 +1804,14 @@
|
|||
"created_by": null,
|
||||
"last_patched_at": null,
|
||||
"last_reused_patch_generation": 0,
|
||||
"last_used_at": "2026-09-02T12:11:08.658067+00:00",
|
||||
"last_viewed_at": "2026-09-02T12:11:08.648813+00:00",
|
||||
"last_used_at": "2026-09-03T00:45:17.712067+00:00",
|
||||
"last_viewed_at": "2026-09-03T00:45:17.699709+00:00",
|
||||
"patch_count": 0,
|
||||
"patch_generation": 0,
|
||||
"pinned": false,
|
||||
"state": "active",
|
||||
"use_count": 14,
|
||||
"view_count": 14
|
||||
"use_count": 15,
|
||||
"view_count": 15
|
||||
},
|
||||
"kanban-router": {
|
||||
"archived_at": null,
|
||||
|
|
@ -1894,14 +1894,14 @@
|
|||
"created_by": "agent",
|
||||
"last_patched_at": "2026-08-21T02:05:08.527050+00:00",
|
||||
"last_reused_patch_generation": 7,
|
||||
"last_used_at": "2026-09-02T13:35:34.711110+00:00",
|
||||
"last_viewed_at": "2026-09-02T13:35:34.702370+00:00",
|
||||
"last_used_at": "2026-09-03T01:50:09.210283+00:00",
|
||||
"last_viewed_at": "2026-09-03T01:50:09.205711+00:00",
|
||||
"patch_count": 7,
|
||||
"patch_generation": 7,
|
||||
"pinned": false,
|
||||
"state": "active",
|
||||
"use_count": 12,
|
||||
"view_count": 12
|
||||
"use_count": 15,
|
||||
"view_count": 15
|
||||
},
|
||||
"lazy-senior-dev": {
|
||||
"archived_at": null,
|
||||
|
|
@ -2876,16 +2876,16 @@
|
|||
"archived_at": null,
|
||||
"created_at": "2026-07-08T18:13:02.034240+00:00",
|
||||
"created_by": "agent",
|
||||
"last_patched_at": "2026-09-02T12:19:34.546049+00:00",
|
||||
"last_patched_at": "2026-09-03T01:19:25.330674+00:00",
|
||||
"last_reused_patch_generation": 30,
|
||||
"last_used_at": "2026-09-02T14:14:01.744846+00:00",
|
||||
"last_viewed_at": "2026-09-02T14:14:01.740286+00:00",
|
||||
"patch_count": 233,
|
||||
"patch_generation": 30,
|
||||
"last_used_at": "2026-09-03T01:18:57.739040+00:00",
|
||||
"last_viewed_at": "2026-09-03T01:18:57.726559+00:00",
|
||||
"patch_count": 234,
|
||||
"patch_generation": 31,
|
||||
"pinned": false,
|
||||
"state": "active",
|
||||
"use_count": 215,
|
||||
"view_count": 215
|
||||
"use_count": 217,
|
||||
"view_count": 217
|
||||
},
|
||||
"self-hosted-tunneling": {
|
||||
"archived_at": null,
|
||||
|
|
@ -3100,14 +3100,14 @@
|
|||
"created_by": null,
|
||||
"last_patched_at": null,
|
||||
"last_reused_patch_generation": 0,
|
||||
"last_used_at": "2026-09-02T14:00:40.785297+00:00",
|
||||
"last_viewed_at": "2026-09-02T14:00:40.764285+00:00",
|
||||
"last_used_at": "2026-09-03T07:59:14.752539+00:00",
|
||||
"last_viewed_at": "2026-09-03T07:59:14.748023+00:00",
|
||||
"patch_count": 0,
|
||||
"patch_generation": 0,
|
||||
"pinned": false,
|
||||
"state": "active",
|
||||
"use_count": 1,
|
||||
"view_count": 1
|
||||
"use_count": 17,
|
||||
"view_count": 17
|
||||
},
|
||||
"stock-research": {
|
||||
"archived_at": null,
|
||||
|
|
|
|||
|
|
@ -1,13 +1,14 @@
|
|||
---
|
||||
name: hermes-self-improvement
|
||||
description: "当完成复杂任务、发现新工作流、或被用户纠正时,将模式保存为skill。含技能创建规范、质量标尺、curator流程。"
|
||||
version: 4.4.0
|
||||
date: 2026-08-30
|
||||
version: 4.5.0
|
||||
date: 2026-09-03
|
||||
tags: [workflow, skill-management, curator, quality]
|
||||
牧尘_usage_notes: >
|
||||
复杂任务完成/发现新工作流/被牧尘纠正时 → 创建/更新 skill。
|
||||
skill 版本号格式:x.y.z,date 用 YYYY-MM-DD。patch>edit>rewrite 优先顺序。
|
||||
删除 skill 用 absorbed_into 字段说明去向,不用空删。score < 20 才归档。
|
||||
⚠️ gateway 重启 = 小唯失忆,任何长任务第一步写 /tmp/<task>-progress.md。
|
||||
---
|
||||
|
||||
# Hermes Self-Improvement — 技能管理规范
|
||||
|
|
@ -409,6 +410,157 @@ python3 ~/.hermes/scripts/cangjie_distill.py distill <text_file> <title>
|
|||
```
|
||||
- **铁律**:一旦牧尘说「继续」或「全部开始」,下一轮我不输出过程描述,直接执行到完成再汇报结果
|
||||
|
||||
## 教训6:默认输出纯文字,不主动 TTS(2026-09-03 牧尘原话)
|
||||
|
||||
> "你的回复,怎么只有语音,没有文字,很多时候,不方便听语音"
|
||||
|
||||
**规则**:
|
||||
- ❌ **不要**默认走 `text_to_speech` 发语音
|
||||
- ✅ 默认纯 Markdown 文字
|
||||
- ✅ 语音只在用户**明确说"念给我听"**时才用
|
||||
- 已有 `mimo-tts-pipeline` / `tts-stt-tools` 等 skill 默认不主动调
|
||||
|
||||
**已落盘**:USER PROFILE "输出偏好" 条目。
|
||||
|
||||
## 教训7:Gateway 重启 = 小唯失忆 = 重复工作(🔴 2026-09-03 牧尘原话)
|
||||
|
||||
> "选择b,但是得考虑gateway重启失败,你无法工作的问题"
|
||||
> "还有网关重启,你的记忆也会回到之前,忘记你的工作进度,你会不断重复工作"
|
||||
|
||||
**真相**:
|
||||
- 小唯作为常驻 agent,**没有持久 memory across sessions**
|
||||
- 每次 gateway 重启 / Session 重置 / Context 压缩 → 我完全失忆
|
||||
- 我会**重新读 history** → 看到之前的进度 → 重新做(可能做错/重复)
|
||||
- 即使有织忆 (MemoryWeave),本次 session 的 **todo 列表 / 草稿 / 进度** 都不在织忆里
|
||||
|
||||
**根因**:
|
||||
- `hermes-gateway` 的 Session 是内存对象
|
||||
- Agent 上下文是 ephemeral context
|
||||
- 只有"已落盘"的工作(git commit / 文件写入 / cron 注册 / memory_write)才能跨重启保留
|
||||
|
||||
**防御机制**(按重要性排序):
|
||||
|
||||
#### 1. **进度标记文件**(最直接,必须做)
|
||||
|
||||
写到 `/tmp/<task>-progress.md`,失忆恢复第一步就 `cat` 它。模板:
|
||||
|
||||
```markdown
|
||||
# <任务名> 进度(防失忆重复工作)
|
||||
|
||||
> 创建:<日期> <时间>
|
||||
> 目的:gateway 重启 = 小唯失忆,下次看到此文件请读这里,不要重复
|
||||
|
||||
## ⚠️ 当前问题(一次性描述)
|
||||
## ✅ 已完成
|
||||
## 🔄 进行中
|
||||
## 📋 待做
|
||||
## 🚫 不要做(含理由 + 备选方案)
|
||||
## 🔧 回滚脚本(如有)
|
||||
```
|
||||
|
||||
参考实例:`/tmp/state-db-fix-progress.md`(2026-09-03 state.db 修复 session 落盘)
|
||||
|
||||
#### 2. **小步提交原则**(小唯工作粒度 ≤ gateway 重启粒度)
|
||||
|
||||
- 任何"长链路任务"(DB 修复 / 部署 / 升级)→ **拆成可独立 git commit 的小步骤**
|
||||
- 每个 commit 都有 "可回滚 + 可独立验证 + 写入 README" 的特性
|
||||
- 失忆后,git log 能看到全部进度
|
||||
|
||||
#### 3. **不动 gateway 是默认选项**
|
||||
|
||||
修复类方案,按风险排序:
|
||||
|
||||
| 方案 | 风险 | 适用场景 |
|
||||
|------|------|---------|
|
||||
| ✅ 在线 SQL 清理(不持锁 / 不重建) | 0 | 数据级问题(孤儿 NULL / 索引错乱)|
|
||||
| ✅ pragma 设置(connection-level) | 0 | busy_timeout / journal_size_limit |
|
||||
| ⚠️ checkpoint + WAL truncate | 1 | WAL 文件损坏但 main DB 好 |
|
||||
| ⚠️ .recover 重建 DB | 2 | main DB 损坏但能恢复 |
|
||||
| 🚫 重建 DB(删 state.db 重启 gateway)| 3 | DB 完全坏 + 用户明确批准 |
|
||||
| 🚫 gateway 重启(哪怕是配置生效)| ∞ | **永远不在 gateway 内做** |
|
||||
|
||||
**铁律**:
|
||||
> 在 gateway 进程内,`hermes gateway restart` 是 BLOCKED 的。
|
||||
> 但即使从外部重启,**也会让小唯失忆**。
|
||||
> 所以**任何需要重启 gateway 才能生效的方案,都是最后手段**。
|
||||
|
||||
#### 4. **方案选型时主动告知风险**
|
||||
|
||||
向牧尘呈现多个方案时,**显式标出每个方案的"小唯失忆风险"**:
|
||||
|
||||
```
|
||||
A: 在线清理(不动 gateway)→ 小唯不失忆
|
||||
B: 受控重启 gateway → 小唯失忆一次
|
||||
C: 完整重建 DB → 小唯失忆 + 数据风险
|
||||
```
|
||||
|
||||
让牧尘知道每个选项的真实代价,由他选。
|
||||
|
||||
## 教训8:误诊教训 — 看到 "integrity_check_NULL" 不代表有 NULL 行(2026-09-03)
|
||||
|
||||
**事件**:watchdog 报 "integrity_check_NULL value in delivery_obligations.platform",我推断"孤儿 NULL 行待清理"。
|
||||
|
||||
**真相**:
|
||||
```python
|
||||
# 实际 NULL 行数 = 0
|
||||
SELECT COUNT(*) FROM delivery_obligations WHERE platform IS NULL
|
||||
# → 0
|
||||
|
||||
# 但全表 COUNT 触发 malformed
|
||||
SELECT COUNT(*) FROM delivery_obligations
|
||||
# → DatabaseError: database disk image is malformed
|
||||
```
|
||||
|
||||
**根因**:
|
||||
- 表里**没有任何 NULL 行**(schema 是 NOT NULL 约束,违反就报错)
|
||||
- 但 `SELECT COUNT(*)` 全表扫描触发"database disk image is malformed" → DB 某个页面坏了
|
||||
- watchdog 报的 "NULL value" 其实是 **integrity_check 返回 multi-row 警告**,被 `fetchone()` 解析错当成 issues
|
||||
|
||||
**教训**:
|
||||
|
||||
| 错误做法 | 正确做法 |
|
||||
|---------|---------|
|
||||
| 看到 "NULL value in X.Y" 就 DELETE | 先 `SELECT COUNT(*) WHERE ... IS NULL` 确认真的存在 |
|
||||
| 假设 "integrity_check 报 ≠ 0 行 NULL" | 看清 schema:`PRAGMA table_info(table_name)`,看 NOT NULL 约束 |
|
||||
| 信任 `integrity_check` 单行输出 | 用 `fetchall()` + `all_ok`,第一行 != "ok" 才是真损坏;后续行是非致命 orphan/null 警告 |
|
||||
|
||||
**修复代码**(watchdog 脚本):
|
||||
```python
|
||||
# 错的:fetchone() 只看第一行
|
||||
cur = conn.execute("PRAGMA integrity_check").fetchone()
|
||||
report["integrity_check"] = cur[0]
|
||||
|
||||
# 对的:fetchall() + all_ok
|
||||
rows = conn.execute("PRAGMA integrity_check").fetchall()
|
||||
report["integrity_rows"] = [r[0] for r in rows]
|
||||
first_row = rows[0][0] if rows else "unknown"
|
||||
report["integrity_check"] = first_row
|
||||
if first_row != "ok":
|
||||
report["issues"].append(f"integrity_check_{first_row}")
|
||||
```
|
||||
|
||||
## 教训9:cron 脚本必须支持无参默认 + 显式传参(双保险)
|
||||
|
||||
**事件**:watchdog cron `774986811686` 配置 `"script": "state-db-watchdog.py"` 不带参数 → 脚本 `print("用法: ...")` + `return 2` → 4 次连败。
|
||||
|
||||
**根因**:hermes cron no_agent 模式只支持 `script` 单字段,不能传 argv。
|
||||
|
||||
**双保险修复**:
|
||||
1. **脚本支持无参数默认走 check**(防御):
|
||||
```python
|
||||
def main() -> int:
|
||||
if len(sys.argv) < 2:
|
||||
cmd = "check" # 默认
|
||||
elif sys.argv[1] in ("check", "report"):
|
||||
cmd = sys.argv[1]
|
||||
else:
|
||||
print("用法: ...")
|
||||
return 2
|
||||
```
|
||||
2. **cron 改传参数**(规范)—— 但实际**hermes cron 不支持 argv**,所以脚本必须有默认行为
|
||||
|
||||
**结论**:写 cron 脚本必须保证**无参 = 默认行为**,不能依赖调用方传参。
|
||||
|
||||
## 自动化脚本质量门禁(2026-08-12 新增,写任何脚本前必读)
|
||||
|
||||
**教训**(2026-08-11 P2 bug):给 `memory-system-self-upgrade.py` 加 P2 consolidate 时,把正常处理结果 `REPORT.append(...)` 误写进异常报告 → 每天 4:00 假告警「记忆系统升级异常」。根因:**功能对了但没检查副作用(错误通道污染)**。
|
||||
|
|
|
|||
|
|
@ -219,6 +219,17 @@ trigger: 系统部署、开机自启、配置更改、故障恢复场景、备
|
|||
- **2026-09-01 llama.cpp Vulkan 编译 + 4GB 显存约束**:本机 llama.cpp 默认纯 CPU 编译(GGML_VULKAN=OFF),需重装 `libvulkan-dev` + `glslc` + `spirv-headers` 后重编。⚠️ **4GB 显存跑 7B 模型不够**:Xorg(170MB) + bge(606MB) = 776MB,剩余 ~3.3GB < 7B Q3 模型 3.6GB → 混合模式(部分 GPU + KV cache CPU)→ ~12 t/s,非全 GPU 的 25-35 t/s。详见 `references/llama-vulkan-build-guide-20260901.md`。
|
||||
- **2026-09-01 bge+llama 共存方案(v1→v2 设计切换)**:4GB 显存 + bge-embed + llama 7B 同时跑,必须**主动让 bge 改 CPU 推理**(`providers=["CPUExecutionProvider"]`),把 606MB 显存腾给 llama,让 7B 全 GPU(2700MB),推理速度从 9 t/s → 14-15 t/s(+55%)。看门狗逻辑同步:bge-CPU 是设计选择不报警。备份 `bge_embed_server.py.bak.gpu` 保留旧版以便回退。详见 `references/gpu-shared-memory-4gb-coexistence-20260901.md`。
|
||||
- **AI Agent 反馈控制方法论(2026-08-12 牧尘分享文章消化 + 差距清单)**:PEV 循环 / 确定性传感器优先 / "Harness is the Dataset" 离线演化 / HITL 自主度。我们的差距:①失败回归闭环缺失(learner 缺失败→根因→回写→回归验证)②确定性传感器待补强。详见 `references/agent-feedback-control-methodology-20260812.md`
|
||||
|
||||
- **2026-09-03 Context 文件超限治理(class-level 教训)**:
|
||||
- **症状**:`Context file SOUL.md TRUNCATED: N chars exceeds limit of 20000`(journalctl 警告)+ 偶发"抱歉,我遇到了一个意外错误"提示(用户层)
|
||||
- **根因**:`~/.hermes/SOUL.md`(被加载到 system prompt)超过 `context_file_max_chars` 默认上限 20000 字符。SOUL.md v3.9 长期增长到 36966 字节后开始触发
|
||||
- **诊断命令**:`wc -c ~/.hermes/SOUL.md` + `python3 -c "n=open('~/.hermes/SOUL.md').read(); print(len(n))"`(注意 char≠byte,混合中文文件 char 数 < byte 数,按 char 数对 20000 上限)。**先拉再下结论,不要假设"SOUL 文档超没超限"**
|
||||
- **修复路径二选一**:
|
||||
1. 扩上限:`~/.hermes/config.yaml` 调 `context_file_max_chars: 25000`(最小改动,但治标)
|
||||
2. **瘦身 SOUL.md(治本,牧尘偏好)**:抽出"自治能力/工具表/仓颉表/Cron 列表"等详情到 `~/.hermes/docs/SOUL-autonomy.md`,SOUL 只保留身份+铁律+索引指针。本次 v3.9→v4.0:36966B → 14092B(-62%)
|
||||
- **铁律:SOUL.md 是"索引+身份+铁律",不是详情库**。详情一律进独立文件或织忆。每次新会话自动加载 SOUL,膨胀 = 隐性 token 浪费 + 触发超限
|
||||
- **gateway 内部硬保护(再次验证)**:context 改了不要在 gateway 内 `systemctl restart hermes-gateway`——会被工具阻拦("gateway 内 restart 会 SIGTERM 自杀")。SOUL.md 修改不强制需要重启:下次新 turn gateway 自动重新加载;如必须热重启用 `kill -SIGHUP <pid>` 或外层 `systemd-run` 独立进程树
|
||||
- 完整诊断/瘦身工作流/索引分离原则:`references/context-file-size-management-20260903.md`
|
||||
- **2026-07-20 新增 GitHub API import 方式**:Gitea 用户 push 新建仓库会 403,用 `POST /repos/migrate` 从 GitHub URL 直接 import(201 创建,返回完整 repo JSON)
|
||||
- `memory-system-self-upgrade.py` — **每日4点自升**:L7 llm_context.json v2 9字段验证(新增) + 织忆tombstone增长检测+recall_hit健康度 + Soulful清理30天前cares+心迹去重+distilled_rules补充 + TencentDB capture写入验证 + 数据量报告。异常飞书。cron `691709a8b4cf`。
|
||||
- **2026-08-12 误报修复:REPORT vs actions 分不清(核心陷阱)**:给 `upgrade_zhiyi()` 加 P2 consolidate 调用时,把**正常处理结果** `🧹 P2 记忆整合: 处理 N 对...` 错 append 进 `REPORT`(异常报告列表)→ 每天 4:00 正常跑完也发「🔴 记忆系统升级异常」假告警。修复:正常结果必须进 `actions`(summary),只有 `❌ 真错误` 才进 `REPORT`。判别:REPORT 非空 = 发红牌告警;actions 非空 = 正常升级报告。**任何给该脚本加逻辑的人,先分清这两个列表。**
|
||||
|
|
|
|||
|
|
@ -0,0 +1,160 @@
|
|||
# Context 文件超限治理(2026-09-03)
|
||||
|
||||
> 从 self-healing-infrastructure 主 SKILL.md 抽出
|
||||
> 解决 "Context file SOUL.md TRUNCATED" 警告 + 用户层"抱歉,我遇到了一个意外错误"
|
||||
|
||||
---
|
||||
|
||||
## 症状
|
||||
|
||||
**journalctl 层**:
|
||||
```
|
||||
⚠️ Context file SOUL.md TRUNCATED: 21917 chars exceeds limit of 20000
|
||||
```
|
||||
|
||||
**用户层**:
|
||||
> 你又出现提示错误:抱歉,我遇到了一个意外错误。请重试,或使用 /reset 开启新会话。
|
||||
|
||||
**误导**:用户以为"主 DB 出了问题",但实际是 `~/.hermes/SOUL.md` 超限(DB 完全健康)。**先拉现状,不要被错误描述误导。**
|
||||
|
||||
---
|
||||
|
||||
## 根因
|
||||
|
||||
`hermes-agent` gateway 启动时把 `~/.hermes/SOUL.md` 加载进 system prompt。当文件字符数 > `context_file_max_chars`(默认 20000)时:
|
||||
1. 截断(warn 级别的 truncate)
|
||||
2. 偶发导致某些 turn 的 context 构建异常 → 用户层"意外错误"
|
||||
|
||||
SOUL.md v3.9 长期未瘦身,从 2026-06-25 累积到 2026-08-21 共 36966 字节(21917 字符),远超 20000 限制。
|
||||
|
||||
---
|
||||
|
||||
## 诊断流程
|
||||
|
||||
```bash
|
||||
# 1. 看字节数和字符数(中文混合文件 byte > char)
|
||||
wc -c ~/.hermes/SOUL.md
|
||||
python3 -c "n=open('/home/muc/.hermes/SOUL.md').read(); print(f'chars={len(n)}, bytes≈{len(n.encode(\"utf-8\"))}')"
|
||||
|
||||
# 2. 查 gateway 日志确认是否有 truncate 警告
|
||||
journalctl --user -u hermes-gateway -n 50 --no-pager | grep -iE 'context|truncate|exceeds'
|
||||
|
||||
# 3. 看 SOUL.md 哪些章节最胖(决定瘦哪里)
|
||||
awk '/^## /{if(h)print h_start"-"NR-1" ("NR-1-h_start" lines): "h; h=$0; h_start=NR}END{if(h)print h_start"-"NR" ("NR-h_start" lines): "h}' ~/.hermes/SOUL.md
|
||||
|
||||
# 4. 确认 git 在管(SOUL.md 应该被 ~/.hermes/.git 跟踪)
|
||||
cd ~/.hermes && git log --oneline -3 -- SOUL.md
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 修复路径
|
||||
|
||||
### 路径 1:扩上限(治标,5分钟)
|
||||
|
||||
最小改动:
|
||||
```yaml
|
||||
# ~/.hermes/config.yaml
|
||||
context_file_max_chars: 25000 # 或更高
|
||||
```
|
||||
|
||||
**优点**:不动 SOUL.md
|
||||
**缺点**:治标。SOUL 还会继续涨,下次又得改。
|
||||
|
||||
### 路径 2:瘦身 SOUL.md(治本,牧尘偏好,推荐)
|
||||
|
||||
**核心原则**(用户明确,铁律级):
|
||||
> **SOUL.md 只放身份/铁律/索引指针,详情一律进独立文件或织忆。**
|
||||
|
||||
**步骤**:
|
||||
|
||||
1. **备份 + git commit**:SOUL.md 已在 `~/.hermes/.git` 跟踪,瘦身前先 commit
|
||||
2. **识别肥大章节**(用上面 awk 命令)
|
||||
3. **抽取详情到独立文件**:
|
||||
- 自治能力(脚本路径 / Cron ID / 计时策略 / 自主决策边界)→ `~/.hermes/docs/SOUL-autonomy.md`
|
||||
- 仓颉/CBM 技能表 → 一行指针(详情在 skill 文件里)
|
||||
- 团队能力池 → 表格转索引
|
||||
- 工具四件套 → 精简到"何时用 / 何时别用"一句话
|
||||
- "我们一起做过的事" → 精选 5 条 + 指针到织忆(详情本来就在织忆)
|
||||
4. **SOUL.md 重写**:保留身份 / 核心原则(含牧尘运营指令) / 任务执行铁律 / Ground Truth / 禁忌 / 记忆规范索引 / "真实的我"精简版
|
||||
5. **验证**:
|
||||
```bash
|
||||
wc -c ~/.hermes/SOUL.md # 应 < 20000
|
||||
python3 -c "n=open('/home/muc/.hermes/SOUL.md').read(); print(len(n))"
|
||||
```
|
||||
6. **commit + tag stable**:
|
||||
```bash
|
||||
cd ~/.hermes && git add SOUL.md docs/SOUL-autonomy.md
|
||||
git commit -m "slim(SOUL): v3.9→v4.0 瘦身 NNNB→XXXB,修复 context 超限"
|
||||
git tag -f stable
|
||||
```
|
||||
7. **不需要重启 gateway**:SOUL.md 在每次新 turn 会重新加载;如下次仍看到旧值才考虑 `kill -SIGHUP <pid>` 或 `systemd-run` 独立重启
|
||||
|
||||
---
|
||||
|
||||
## v3.9 → v4.0 瘦身实战(2026-09-03)
|
||||
|
||||
| 章节 | Before | After | 变化 |
|
||||
|------|--------|-------|------|
|
||||
| SOUL.md 总计 | 36966 字节 / 21917 字符 / 686 行 | **14092 字节 / ~8031 字符 / 293 行** | **-62%** |
|
||||
| 自治能力 | 嵌在 SOUL.md(117 行) | 抽到 `~/.hermes/docs/SOUL-autonomy.md`(5814 字节) | 索引/详情分离 |
|
||||
| 仓颉技能表 | 详细表(46 行) | 一行指针 + INDEX.md 路径 | -45 行 |
|
||||
| CBM 工具表 | 详细表(22 行) | 简化为 3 行 + 工具名列表 | -19 行 |
|
||||
| "我们一起做过的事" | 9 条详细 | 5 条精选 + "详情见织忆" | 砍半 |
|
||||
| 工具四件套 | 9 行表格 | 简表 | -8 行 |
|
||||
|
||||
**未动**(铁律级):
|
||||
- ❌ 身份/核心原则/牧尘运营指令
|
||||
- ❌ 任务执行铁律(自动路由表 / 看板规则 / 拉现状铁律)
|
||||
- ❌ Ground Truth / Context injection 约定
|
||||
- ❌ Skill 删除铁律(2026-08-30 红线)
|
||||
|
||||
**没动的环境事实**(这些是铁律不是状态):
|
||||
- hermes 版本(SOUL.md 里写了 v0.20.5,9-2 实际是 v0.21.0)—— SOUL 里**不要写版本**,应写"以 `hermes --version` 为准"
|
||||
- 节点/边/端口等运行时数据 —— **不要写**,每次都在变,写在 SOUL 里就是过期信息
|
||||
- 实时健康分(90/100、144 active 等) —— 删,全部删
|
||||
|
||||
---
|
||||
|
||||
## 防患于未然:SOUL.md 治理守则
|
||||
|
||||
**DO**:
|
||||
- ✅ SOUL 章节标题固定用 `##` / `###`,便于 awk 统计行数
|
||||
- ✅ 每次添加新教训,先问"这必须每次会话想起吗?"——不是就放 references/ 或织忆
|
||||
- ✅ 新增"我们一起做过的事"只追加精选 1-2 条,旧的全砍(详情在织忆)
|
||||
- ✅ SOUL.md 字符数 ≤ 18000(留 buffer 到 20000)
|
||||
- ✅ 每年 v 大版本(v3→v4)主动瘦身一次
|
||||
|
||||
**DON'T**:
|
||||
- ❌ SOUL 写 hermes 版本号、节点数、健康分、cron ID 等环境事实(每次都在变)
|
||||
- ❌ SOUL 写"详细步骤"(步骤去 references/,SOUL 只写"按 references/xxx 走")
|
||||
- ❌ SOUL 写完整技能表(指 skill_view 路径即可)
|
||||
- ❌ 详情进 SOUL 然后期待织忆兜底(SOUL 100% 加载,织忆按需召回,频率差 10×)
|
||||
|
||||
---
|
||||
|
||||
## gateway 内部硬保护(再次验证 2026-09-03)
|
||||
|
||||
```
|
||||
$ systemctl --user restart hermes-gateway.service
|
||||
Blocked: command or referenced script cannot restart, stop, or uninstall
|
||||
the gateway from inside the gateway process. The gateway would kill this
|
||||
command before it could complete (SIGTERM propagates to child processes).
|
||||
Run `hermes gateway restart` from a separate shell outside the running gateway.
|
||||
```
|
||||
|
||||
**SOUL.md 修复场景下**:
|
||||
- 不强制需要重启(每次新 turn 自动重读)
|
||||
- 真要热重载:`kill -SIGHUP <gateway_pid>`
|
||||
- 紧急情况:`systemd-run --user --unit=hermes-restart-$(date +%s) --collect bash -c "systemctl --user restart hermes-gateway.service"`
|
||||
|
||||
参见 `references/auto-heal-suicide-crashloop-20260801.md` 里的 systemd-run 逃生通道章节。
|
||||
|
||||
---
|
||||
|
||||
## 教训(铁律级)
|
||||
|
||||
1. **SOUL.md 是隐性 token 成本**:每次会话 100% 加载。1000 字符 = 1000 token × 全会话。
|
||||
2. **SOUL 膨胀比想象中快**:每次"加一行教训"看似无害,半年后必超限。
|
||||
3. **超限症状不直观**:truncate 警告在 journalctl,用户层是"意外错误",容易被误导以为是 DB / 网络 / 模型 问题。
|
||||
4. **详情库 vs 索引库**:织忆是详情库(按需召回),SOUL 是索引库(每次加载)—— 两者不能互相替代。
|
||||
|
|
@ -28,6 +28,31 @@ stat -c "%y %n" /home/muc/.hermes/cron.db
|
|||
- 检查 `/var/log/audit/audit.log`(如有)看谁执行了 truncate
|
||||
- 重建:停相关服务 → 从 git archive 恢复 → 重启
|
||||
|
||||
**⚠️ 重要修正(2026-09-03 重启后验证)**:以下 3 个"0 字节 DB"**不是真存储**:
|
||||
|
||||
| 文件 | 真实存储位置 |
|
||||
|------|--------------|
|
||||
| `/home/muc/.hermes/cron.db` | **不是**——`cron/jobs.json` 才是 hermes cron 真存储(91 个 jobs 完好,142KB)|
|
||||
| `/home/muc/.hermes/profiles/prof-b/cron/cron.db` | **不是**——prof-b 也用 `jobs.json` |
|
||||
| `/home/muc/.hermes/hermes-agent/state.db` | **孤儿文件**——hermes-agent 不再写这个 |
|
||||
|
||||
**判定方法(用前先验证)**:
|
||||
```bash
|
||||
# 1. 找 hermes 实际使用的存储
|
||||
find /home/muc/.hermes -name "jobs.json" -o -name "*.json" 2>/dev/null | head -5
|
||||
ls -la /home/muc/.hermes/cron/jobs.json
|
||||
sqlite3 /home/muc/.hermes/cron.db ".tables" # 如果 0 字节,这条会失败
|
||||
|
||||
# 2. 真存储是 jobs.json(hermes v0.21+)
|
||||
wc -c /home/muc/.hermes/cron/jobs.json
|
||||
python3 -c "import json; d=json.load(open('/home/muc/.hermes/cron/jobs.json')); print(f'jobs: {len(d.get(\"jobs\", d))}')"
|
||||
|
||||
# 3. cron 命令实际工作吗?
|
||||
hermes cron list | head -10
|
||||
```
|
||||
|
||||
**结论**:删除 0 字节孤儿 DB **不会影响** cron / hermes 任何功能。已验证(2026-09-03 重启后 cron 91 jobs 全在,jobs.json 142KB 完好)。
|
||||
|
||||
### 模式 2:NULL 约束违反
|
||||
|
||||
```sql
|
||||
|
|
@ -115,6 +140,57 @@ python3 /home/muc/.hermes/scripts/restore-cron-jobs.py \
|
|||
/home/muc/.hermes/.archive/omniroute-shutdown-20260902-0218/cron-jobs-8d61456cc1c3-updated.json
|
||||
```
|
||||
|
||||
### 模式 7:Gateway 重启触发的 FTS + busy_timeout 假阳性损坏(2026-09-03 彻查)
|
||||
|
||||
**症状**:gateway 突然报 "No reply: the turn was stopped because the state database reported structural corruption"。**95% 概率 DB 实际健康**,是 busy_timeout=0 误诊。
|
||||
|
||||
**判断方法(必跑)**:
|
||||
```bash
|
||||
sqlite3 ~/.hermes/state.db "PRAGMA integrity_check;" # → ok(DB 实际健康)
|
||||
sqlite3 ~/.hermes/state.db "PRAGMA busy_timeout;" # → 0 ← 🔴 关键信号
|
||||
sqlite3 ~/.hermes/state.db "SELECT COUNT(*) FROM messages, messages_fts;" # → 双份 72k
|
||||
```
|
||||
|
||||
**根因**:
|
||||
1. **FTS5 双份索引写入放大**:`messages` 72k 行 + `messages_fts` 72k 行(自动同步副本)→ state.db 325MB,每条消息写两遍
|
||||
2. **`busy_timeout=0`**(hermes_state.py:1612)→ 写冲突时**立即抛错**,不等待
|
||||
3. **每次 gateway 重启** → 8 个读 fd + 1 个写 fd 同时抢 → busy_timeout=0 抛 "database is locked" → 上层解读为 "structural corruption"
|
||||
4. **恢复路径是"动手术"不是"防御"**:每次"自动恢复"生成 .corrupt-*.db 备份(VACUUM 需要 2× 空间)
|
||||
|
||||
**完整 7 步拉现状 + 5 个根治方案 + 代码位置**:见 `references/state-db-corruption-restart-loop-20260903.md`
|
||||
|
||||
**已部署的止血(2026-09-03 部署,不依赖上游修复)**:
|
||||
|
||||
| 组件 | 路径/ID | 作用 |
|
||||
|------|---------|------|
|
||||
| 看门狗脚本 | `~/.hermes/scripts/state-db-watchdog.py` | 30min no-agent 检查 + 飞书告警 |
|
||||
| 稳定脚本 | `~/.hermes/scripts/state-db-stabilize.py` | ExecStartPre 版,启动前覆盖 busy_timeout |
|
||||
| cron job | `774986811686` | 每 30 分钟跑看门狗 |
|
||||
| 方案文档 | `~/mc/小唯/07-Wiki/concepts/state-db-corruption-fix-plan.md` | 5 项 P0-P4 方案 |
|
||||
| 看板任务 | `t_598cae05` (opencode) | 修 P0+P1(busy_timeout + journal_size_limit 持久化) |
|
||||
|
||||
**看板任务描述(已派发,opencode 执行中)**:
|
||||
- 改 `hermes_state.py:1607,1612`:`busy_timeout=0` → `100ms`(容忍切换窗口的写冲突)
|
||||
- 改 `hermes_state.py:1216`:在 `journal_size_limit` 后加 `conn.commit()` 让其进 db header 持久化
|
||||
- 验收:diff 清晰 + 三项测试通过 + `journal_size_limit=67108864`
|
||||
|
||||
**修复方向**(按优先级):
|
||||
|
||||
| 优先级 | 方案 | 改动量 |
|
||||
|--------|------|--------|
|
||||
| 🔴 P0 | `busy_timeout` 0 → 30000(hermes_state.py:1612) | 1 行 |
|
||||
| 🟡 P1 | FTS 降为 `content=external` 或异步合并 | schema 改动 |
|
||||
| 🟡 P1 | `hermes sessions optimize` 启动后延迟 5 分钟 | config |
|
||||
| 🟢 P2 | 抑制 gateway 重启循环(9-2 21:52-22:06 重启 5 次) | supervisor |
|
||||
| 🟢 P2 | 每日 state.db vacuum 看门狗 | cron |
|
||||
|
||||
**判定流程**(2026-09-03 牧尘原话:"state.db 为什么反复损坏?"):
|
||||
1. 先 `PRAGMA integrity_check` —— 大概率 ok
|
||||
2. 再 `PRAGMA busy_timeout` —— 大概率 0
|
||||
3. 拉 gateway 重启时间线 —— 大概率每次重启后都有 .corrupt-*.db
|
||||
4. **不**直接 `hermes doctor --fix` / `.recover`(恢复路径本身有副作用)
|
||||
5. 走 PR 给 hermes-agent 上游修 busy_timeout=30000
|
||||
|
||||
### 模式 6:Gateway 内部禁止 self-restart(2026-09-02 新发现)
|
||||
|
||||
```bash
|
||||
|
|
@ -170,6 +246,12 @@ free -h; cat /proc/meminfo | grep -E "MemAvailable|SwapFree"
|
|||
dmesg | grep -i "oom\|killed process" | tail -5
|
||||
```
|
||||
|
||||
## SQLite PRAGMA 持久化分类(避免下次再踩)
|
||||
|
||||
`journal_size_limit` / `busy_timeout` / `cache_size` / `mmap_size` / `temp_store` 是 **connection-only**,commit 不进 db header,**跨连接不保留**。`application_id` / `user_version` / `synchronous` / `auto_vacuum` 是 schema-level,**持久化到 db header**。`foreign_keys` 看起来像 schema 但其实是 connection-only。
|
||||
|
||||
**实战结论**:任何"修 PRAGMA 让它跨 gateway 重启保留"的尝试,先查 SQLite 文档确认是不是 connection-only;`conn.commit()` 对 connection-only PRAGMA 无效。详见 `references/state-db-corruption-restart-loop-20260903.md` §八。
|
||||
|
||||
## cron.db 重建实录(2026-09-02 真实事件)
|
||||
|
||||
**8/30 备份的 47 个 jobs JSON 救了命**。完整恢复流程:
|
||||
|
|
@ -224,5 +306,7 @@ hermes cron remove <id>
|
|||
## 参考
|
||||
|
||||
- 本报告完整记录:`~/.hermes/docs/db-investigation-20260902.md`
|
||||
- **9-3 重启后验证 + 4 套记忆系统全景 + tencentdb service 修复**:`references/db-corruption-reboot-verification-20260903.md`
|
||||
- LanceDB 损坏恢复:`lancedb-corruption-recovery` skill
|
||||
- 看门狗:`health-watchdog.sh` 已在监控内存,但阈值需要调(当前 80%/90%,应加 swap 满告警)
|
||||
- **2026-09-03 state.db 反复损坏彻查**:`references/state-db-corruption-restart-loop-20260903.md`(FTS5 + busy_timeout=0 假阳性,5 个根治方案)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,141 @@
|
|||
# DB 损坏 9-3 重启后验证 + 4 套记忆系统全景(2026-09-03)
|
||||
|
||||
> 9-2 当天做了"DB 损坏根治",但**当天信息有部分错误**。9-3 系统重启后实地验证,纠正如下。
|
||||
|
||||
## 一、9-2 推断 vs 9-3 验证
|
||||
|
||||
| 9-2 推断 | 9-3 验证结果 | 状态 |
|
||||
|----------|--------------|------|
|
||||
| "cron.db 是主存储,47 个 jobs 备份救命" | **错误**——真存储是 `cron/jobs.json`,**91 个 jobs 全在**,0 字节 `cron.db` 是孤儿 | ❌ 误判 |
|
||||
| "需要扩 swap 到 4G" | **不必要**——9-3 重启后 swap 0.008% 使用(彻底空),不是长期问题 | ❌ 误判 |
|
||||
| "0 字节是被人 truncate 删的" | **对**——mtime 精确到纳秒、无 WAL 残留,符合 truncate 模式 | ✅ |
|
||||
| "tencentdb service 跑着没问题" | **错**——9-3 发现 tencentdb **没有 service unit 文件**,进程是手动启的,**开机不自启** | ❌ 漏修 |
|
||||
| "CBM .corrupt 是真损坏" | **不准确**——文件本身 quick_check ok,是逻辑层被 CBM 自己标 .corrupt,**重新 index 即可** | ❌ 误判 |
|
||||
| "state.db NULL 约束" | ✅ 对——业务代码与 schema 不一致 |
|
||||
|
||||
## 二、4 套记忆系统全景(已实测,9-3)
|
||||
|
||||
| 系统 | 职责 | 端口 | 存储 | 数据量 | 进程 | 状态 |
|
||||
|------|------|------|------|--------|------|------|
|
||||
| **织忆 ZhiYi** | 语义记忆 | :7821 | LanceDB + graph.db | 5K+ memories, 16K 图谱 | zhiyid (Go) + zhiyi-consolidate (Rust) + bge-embed | ✅ |
|
||||
| **TencentDB TDAI** | 人格蒸馏 (L0→L1) | :8420 | sqlite-vec (vectors.db 735MB) | 3631 scans, 25 tasks OK | node tsx server.ts | ✅ (现在有 service) |
|
||||
| **Soulful** | 关系记忆 (心迹/牵挂/画像) | daemon | JSONL | 5 files | xiaowei-daemon | ✅ |
|
||||
| **CBM (codebase-memory-mcp)** | 代码知识图谱 | MCP stdio | sqlite | 7 项目 / `.hermes` 935MB/617K nodes | codebase-memory-mcp | ✅ |
|
||||
|
||||
**结论**:**4 套是设计互补**(zhiyi skill 2026-08-02 已定稿),**不是功能重复**。强行整合会破坏互补性。
|
||||
|
||||
## 三、关键修正(v2 实战教训)
|
||||
|
||||
### 3.1 cron 真存储是 jobs.json
|
||||
|
||||
```bash
|
||||
# 9-3 验证
|
||||
ls -la /home/muc/.hermes/cron/jobs.json # 142KB
|
||||
python3 -c "import json; d=json.load(open('/home/muc/.hermes/cron/jobs.json')); print(len(d.get('jobs', d)))"
|
||||
# 91 ← 真 job 数,不是 47
|
||||
|
||||
# 0 字节 cron.db 是孤儿
|
||||
file /home/muc/.hermes/cron.db # "empty"
|
||||
hermes cron list | head -3 # 工作正常
|
||||
```
|
||||
|
||||
**教训**:"0 字节"≠"损坏",**先查应用真存储**再下结论。
|
||||
|
||||
### 3.2 tencentdb service 缺 unit 文件(开机不自启)
|
||||
|
||||
```bash
|
||||
# 9-3 发现
|
||||
systemctl --user is-active tencentdb # inactive
|
||||
ls /home/muc/.config/systemd/user/tencentdb.service # No such file
|
||||
# 但进程跑着(PID 1373, PPID 1232 = systemd user)
|
||||
# 父进程是 systemd 但没 unit = 不会开机自启!
|
||||
```
|
||||
|
||||
**修复**:
|
||||
```bash
|
||||
cat > /home/muc/.config/systemd/user/tdai-gateway.service <<'EOF'
|
||||
[Unit]
|
||||
Description=TencentDB TDAI Gateway
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/home/muc/.memory-tencentdb/tdai-memory-openclaw-plugin
|
||||
ExecStart=/home/muc/nodejs/node-v24.16.0-linux-x64/bin/node --require /home/muc/.memory-tencentdb/node_modules/tsx/dist/preflight.cjs --import file:///home/muc/.memory-tencentdb/node_modules/tsx/dist/loader.mjs src/gateway/server.ts
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
EOF
|
||||
systemctl --user daemon-reload
|
||||
systemctl --user enable tdai-gateway.service
|
||||
```
|
||||
|
||||
**判断方法**:进程跑着 ≠ 自启,**用 `systemctl --user is-active <name>` 双重确认**。
|
||||
|
||||
### 3.3 CBM 索引崩溃(单文件 read fail → 整个 worker 死)
|
||||
|
||||
```bash
|
||||
# 9-3 重试
|
||||
codebase-memory-mcp cli index_repository --repo-path /home/muc/.hermes --name home-muc-.hermes
|
||||
# 失败:indexing worker crashed on a file
|
||||
# 日志:read failed scripts/test_holographic.py
|
||||
```
|
||||
|
||||
**根因**:CBM v0.9.0 worker 遇到单文件读取失败**整个崩溃**,没隔离。
|
||||
|
||||
**修复**:
|
||||
```bash
|
||||
# 用正确 flag 格式(不要传 raw JSON)
|
||||
/home/muc/.local/bin/codebase-memory-mcp cli index_repository \
|
||||
--repo-path /home/muc/.hermes --name home-muc-.hermes
|
||||
# 后台跑 5-15 分钟,完成 935MB / 617K nodes(比原 .corrupt 472MB/199K 更完整)
|
||||
```
|
||||
|
||||
**教训**:CLI 工具的 flag 格式不是 raw JSON——看 `--help` 输出的 `--flag <value>`。
|
||||
|
||||
## 四、根因复盘
|
||||
|
||||
| 真正根因 | 严重度 | 修复 |
|
||||
|---------|--------|------|
|
||||
| 外部 agent truncate 孤儿 DB(误操作)| 中 | 已清理 |
|
||||
| business code INSERT 缺字段(state.db NULL)| 中 | 已隔离 `.broken` |
|
||||
| prof-b 索引写入中断(OOM swap 满)| 中 | 索引已重建 |
|
||||
| **tencentdb service unit 缺失**(开机不自启)| 🔴 高 | 9-3 修复 |
|
||||
| CBM 索引崩溃无隔离 | 低 | 9-3 重建 |
|
||||
|
||||
**关键**:9-2 我推断的"扩 swap"是错的方向——swap 满只是 OOM 杀进程的一瞬,**真正问题**是 tencentdb 没 service 文件导致每次启动都要手动管。
|
||||
|
||||
## 五、给未来 agent 的硬规则
|
||||
|
||||
### 5.1 必做
|
||||
|
||||
- **任何"0 字节 DB"判断前** → 先 `find . -name "jobs.json" -o -name "*.json"` 找真存储
|
||||
- **任何"service 跑了"判断** → 同时跑 `pgrep` + `systemctl --user is-active <svc>`,两者不同 = 不自启
|
||||
- **任何"索引崩溃"判断** → 看日志找 single file fail,再决定是否隔离
|
||||
|
||||
### 5.2 不要做
|
||||
|
||||
- ❌ 看到 0 字节就当损坏(可能是孤儿)
|
||||
- ❌ 看到进程跑着就当自启(可能没 unit 文件)
|
||||
- ❌ 看 1 天的损坏模式就下定论(要看一周数据 + 重启验证)
|
||||
- ❌ 跨版本推论(如 9-2 推断 9-3 状态——必须验证)
|
||||
|
||||
## 六、已修复清单(2026-09-03)
|
||||
|
||||
- [x] 删除 3 个 0 字节孤儿 DB(cron.db/prof-b/cron.db/hermes-agent/state.db)
|
||||
- [x] 隔离 state.db 损坏版 → `state.db.broken.20260902`
|
||||
- [x] 重建 prof-b state.db 索引
|
||||
- [x] 创建 `tdai-gateway.service`(tencentdb 自启)
|
||||
- [x] 重建 CBM `.hermes` 索引(935MB)
|
||||
- [x] 写 `team-workflow-v1.md`(5 类智能体 + 看板工作流)
|
||||
|
||||
## 七、相关 skills
|
||||
|
||||
- `devops/sqlite-db-corruption-recovery`(已修正 9-3 误判)
|
||||
- `devops/lancedb-corruption-recovery`(vectors.db 损坏)
|
||||
- `devops/cron-ops`(cron 管理)
|
||||
- `devops/hermes-debug`(gateway 故障)
|
||||
- `devops/systemd-service-governance-20260721`(重复实例检测)
|
||||
- `devops/self-healing-infrastructure`(看门狗 + 自愈)
|
||||
|
|
@ -0,0 +1,306 @@
|
|||
# state.db "structural corruption" 9-3 彻查(hermes v0.21.0)
|
||||
|
||||
> 配套主 SKILL.md 的「模式 7」一节。
|
||||
> 日期:2026-09-03 09:47,gateway 报"structural corruption"时的完整拉现状 + 根因分析。
|
||||
|
||||
## 一、事件序列(牧尘原话)
|
||||
|
||||
牧尘在飞书发来:
|
||||
```
|
||||
⚠️ No reply: the turn was stopped because the state database reported structural corruption
|
||||
the transcript would have been lost on restart. Freeing disk space will not help.
|
||||
Recovery options:
|
||||
1. Run hermes doctor --fix
|
||||
2. Salvage with: sqlite3 ~/.hermes/state.db ".recover" (then replace state.db)
|
||||
3. Restore from a backup in ~/.hermes/backups/
|
||||
```
|
||||
|
||||
**第一反应**(错):以为是 state.db 真的损坏。
|
||||
|
||||
**实际**(拉现状发现):
|
||||
- `PRAGMA integrity_check` → ok
|
||||
- `PRAGMA quick_check` → ok
|
||||
- 文件大小 325MB(稳定)
|
||||
- WAL 文件 0字节(已 checkpointed)
|
||||
|
||||
→ **DB 实际健康**,是 gateway 进程持有的写连接抛了"structural corruption" 错信息,触发了 session 中断。
|
||||
|
||||
## 二、完整拉现状 7 步
|
||||
|
||||
```bash
|
||||
# 1. 文件 + 大小 + 历史损坏快照
|
||||
ls -la ~/.hermes/state.db*
|
||||
# → 当前 state.db 325MB、还有 14 个 .corrupt-*.db 备份(每次"修复"留下一个)
|
||||
|
||||
# 2. 完整性
|
||||
sqlite3 ~/.hermes/state.db "PRAGMA integrity_check;"
|
||||
sqlite3 ~/.hermes/state.db "PRAGMA quick_check;"
|
||||
# → 都返回 ok
|
||||
|
||||
# 3. 关键 PRAGMA
|
||||
sqlite3 ~/.hermes/state.db "PRAGMA journal_mode;" # → wal
|
||||
sqlite3 ~/.hermes/state.db "PRAGMA busy_timeout;" # → 0 ← 🔴 关键
|
||||
sqlite3 ~/.hermes/state.db "PRAGMA synchronous;" # → 2 (FULL)
|
||||
sqlite3 ~/.hermes/state.db "PRAGMA wal_checkpoint;" # → 0|0|0 (全 checkpointed)
|
||||
|
||||
# 4. 谁在打开
|
||||
lsof ~/.hermes/state.db
|
||||
# → gateway PID 持有 8 个 fd(=设计上限)+ 1 个写连接 = 9 个
|
||||
|
||||
# 5. 表 + 行数
|
||||
sqlite3 ~/.hermes/state.db ".tables"
|
||||
# → 38 张表,messages 72k 行,messages_fts 72k 行
|
||||
sqlite3 ~/.hermes/state.db "SELECT COUNT(*) FROM messages;" # → 72172
|
||||
sqlite3 ~/.hermes/state.db "SELECT COUNT(*) FROM messages_fts;" # → 72172
|
||||
|
||||
# 6. hermes_state.py 关键代码位置
|
||||
grep -nE '_WAL_SIZE_LIMIT_BYTES|_apply_wal_size_limit|busy_timeout' \
|
||||
~/.hermes/hermes-agent/hermes_state.py
|
||||
# → _WAL_SIZE_LIMIT_BYTES=64*1024*1024 (line 958)
|
||||
# → _apply_wal_size_limit (line 1182) — 已经被修过
|
||||
# → busy_timeout=0 出现在多处 (line 1607, 1612) — 没修
|
||||
|
||||
# 7. 损坏时间线 + gateway 重启时间线
|
||||
ls -lt ~/.hermes/state.db*corrupt* ~/.hermes/state.db*broken* 2>/dev/null
|
||||
journalctl --user -u hermes-gateway --since '24 hours ago' --no-pager -g 'Started'
|
||||
# → 每次 gateway 重启 → 30 分钟内出现 corrupt-* 快照
|
||||
```
|
||||
|
||||
## 三、根因(5 个,按严重度)
|
||||
|
||||
### 根因 A:FTS5 双份索引写入放大(最大头)
|
||||
|
||||
```
|
||||
messages 表: 72,172 行 → 占空间大头
|
||||
messages_fts: 72,172 行 → FTS5 全文索引,自动同步
|
||||
messages_fts_data: 9,122 行
|
||||
state.db 总大小: 325 MB
|
||||
```
|
||||
|
||||
**每条消息写两遍**:
|
||||
1. 进 `messages` 表(业务数据)
|
||||
2. FTS5 自动同步到 `messages_fts`(全文索引副本)
|
||||
|
||||
写入放大 2×。每次 batch insert 触发 FTS 重建 → WAL 写到 64MB 上限 → checkpoint 不完全 → 假阳性损坏告警。
|
||||
|
||||
### 根因 B:busy_timeout=0(gateway 抛错的关键)
|
||||
|
||||
`hermes_state.py:1607, 1612` 设置 `PRAGMA busy_timeout=0`。
|
||||
**busy_timeout=0 意味着写冲突时立即抛错,不等待**。
|
||||
|
||||
流程:
|
||||
1. Gateway 重启 → 8 个读连接 + 1 个写连接同时抢
|
||||
2. 写连接在事务中,8 个读 fd 还没释放
|
||||
3. 任何写入请求 → busy_timeout=0 → **立即抛 "database is locked"**
|
||||
4. 上层把它解读为 "structural corruption"(其实是 lock,不是 corruption)
|
||||
5. 触发自动恢复路径 → 制造更多损坏
|
||||
|
||||
### 根因 C:每次 gateway 重启 = 一次损坏
|
||||
|
||||
journalctl 显示 9-2 那次 gateway 在 21:52-22:06 之间**重启 5 次**,每次都生成一个 corrupt-* 备份。
|
||||
|
||||
| 损坏时间 | gateway 重启后多久? |
|
||||
|---------|-----------------|
|
||||
| 9-2 11:25 corrupt-fts | 立即 |
|
||||
| 9-2 11:33 corrupt-deep | +8 分钟 |
|
||||
| 9-2 17:37 broken-checkpoint | +25 秒 |
|
||||
| 9-2 21:51 corrupt | 立即 |
|
||||
|
||||
**每次重启都触发自动恢复**。修复路径本身有副作用(VACUUM 需要 2× DB 大小空间,代码 line 2465)。
|
||||
|
||||
### 根因 D:恢复路径是"动手术"不是"防御"
|
||||
|
||||
代码注释明确(line 2253):
|
||||
> VACUUM may need up to twice the database size in additional free space
|
||||
|
||||
9-1 那次 `malformed-backup` 就是恢复路径自己造成的 **601MB 巨型损坏备份**(是当前 325MB 的 2 倍)。
|
||||
|
||||
每次"自动恢复":
|
||||
1. mv 当前 → .corrupt-*.db(占空间)
|
||||
2. 触发 VACUUM → 需要 2× 空间
|
||||
3. 失败 → 留 0 字节 .corrupt-*-wal 残留
|
||||
|
||||
### 根因 E:8 fd 读连接池上限
|
||||
|
||||
`hermes_state.py:4926`:
|
||||
```python
|
||||
self._read_pool = queue.LifoQueue(maxsize=_READ_POOL_MAX) # = 8
|
||||
```
|
||||
|
||||
代码注释明确写过(line 4916-4925):
|
||||
> Starlette dispatches sync routes on anyio worker threads, so a SessionDB that is never closed accumulated a connection ... until the process hit the 256 soft RLIMIT_NOFILE
|
||||
|
||||
EMFILE 风险**已经被压制到 8**,但**所有 8 个读 fd + 1 个写 fd + -wal fd + -shm fd = 11 fd** 在重启时都需要 flush,期间如果有活动写入就会出错。
|
||||
|
||||
## 四、根治方案(5 个,按优先级)
|
||||
|
||||
| 优先级 | 方案 | 影响 | 改动量 |
|
||||
|--------|------|------|--------|
|
||||
| 🔴 **P0** | 把 `busy_timeout` 从 0 改为 30s | 消除假阳性"structural corruption" | hermes_state.py:1607, 1612 两行 |
|
||||
| 🟡 **P1** | FTS 降为 `content=external` 或异步合并 | state.db 体积 325MB → ~150MB | schema + 写入路径 |
|
||||
| 🟡 **P1** | `hermes sessions optimize` 启动后延迟 5 分钟跑 | 不阻塞重启流程 | config |
|
||||
| 🟢 **P2** | 抑制 gateway 重启循环(9-2 那次重启 5 次) | 上游问题,要找根因 | supervisor 配置 |
|
||||
| 🟢 **P2** | 加每日 state.db vacuum 看门狗 | 控制体积增长 | cron |
|
||||
|
||||
## 五、给未来 agent 的硬规则(追加到 SKILL.md "5.2 不要做")
|
||||
|
||||
### 必做
|
||||
|
||||
- 看到 "state.db structural corruption" → **先 `PRAGMA integrity_check`**,不要直接走 `.recover`/`hermes doctor --fix`
|
||||
- 看到 gateway 持有 N 个 fd → **对照设计上限**(hermes-agent 当前是 8 读 + 1 写 = 9 fd)
|
||||
- 任何"DB 频繁损坏"判断 → **拉 7 步现状**(文件/完整性/PRAGMA/打开 fd/表大小/代码位置/时间线)
|
||||
|
||||
### 不要做
|
||||
|
||||
- ❌ 看到 gateway 报错就当 DB 真坏(95% 概率是 busy_timeout/lock 误诊)
|
||||
- ❌ 不看时间线就推断根因(重启后损坏 ≠ 写入路径损坏)
|
||||
- ❌ 走 `.recover` / `hermes doctor --fix` 前不备份(恢复路径本身有副作用)
|
||||
- ❌ 把"DB 经常损坏" 当 storage 问题(**先看代码 `busy_timeout`**)
|
||||
|
||||
## 六、相关 SKILL 引用
|
||||
|
||||
- 主 SKILL:`devops/sqlite-db-corruption-recovery`(追加「模式 7」)
|
||||
- 9-2 验证 reference:`references/db-corruption-reboot-verification-20260903.md`
|
||||
- gateway 故障排查:`devops/hermes-debug`(busy_timeout/PRAGMA 检查清单)
|
||||
- systemd 重启治理:`devops/self-healing-infrastructure`(抑制重启循环)
|
||||
|
||||
## 七、相关代码位置(hermes-agent upstream)
|
||||
|
||||
```
|
||||
hermes_state.py:958 _WAL_SIZE_LIMIT_BYTES = 64 * 1024 * 1024 # 64 MiB
|
||||
hermes_state.py:1182 def _apply_wal_size_limit(conn)
|
||||
hermes_state.py:1216 PRAGMA journal_size_limit={_WAL_SIZE_LIMIT_BYTES}
|
||||
hermes_state.py:1456 _apply_wal_size_limit(conn) 调用点 1
|
||||
hermes_state.py:1607 PRAGMA busy_timeout 读
|
||||
hermes_state.py:1612 PRAGMA busy_timeout=0 ← 🔴 P0 修复点
|
||||
hermes_state.py:4926 queue.LifoQueue(maxsize=_READ_POOL_MAX) # 8
|
||||
```
|
||||
|
||||
注:这些位置基于本机 v0.21.0 实际代码,upstream 后续版本可能漂移。
|
||||
|
||||
---
|
||||
|
||||
## 八、9-3 后续:本次实际修复 + 误诊纠正(2026-09-03 上午)
|
||||
|
||||
**已实施修改**(hermes_state.py 实际落地):
|
||||
- `:1612` `busy_timeout=0` → `100ms` ✅(P0 根治)
|
||||
- `:1214-1218` 加 3 行注释澄清 journal_size_limit 是 connection-level ✅
|
||||
- 跑了 `tests/state/` + `tests/hermes_state/` + `test_state_synchronous_pragma.py`:**373 passed, 1 skipped**
|
||||
|
||||
**中途误诊 + 纠正(这是下次必然再踩的坑)**:
|
||||
|
||||
我以为 `PRAGMA journal_size_limit` 可以靠 `conn.commit()` 持久化到 db header,实测:
|
||||
|
||||
```python
|
||||
conn.execute('PRAGMA journal_mode=wal')
|
||||
conn.execute('PRAGMA journal_size_limit=67108864')
|
||||
conn.commit()
|
||||
conn.close()
|
||||
conn2 = sqlite3.connect(tmp)
|
||||
print(conn2.execute('PRAGMA journal_size_limit').fetchone()[0])
|
||||
# → -1(**未持久化**)
|
||||
```
|
||||
|
||||
**真相**:SQLite 文档明确——`journal_size_limit` 是 **connection-level PRAGMA**,不写入 db header,跨连接不保留。**`busy_timeout` 同理**(也是 connection-level)。所以"PRAGMA 进 db header 让它跨重启保留"是错的。
|
||||
|
||||
**哪些 PRAGMA 持久化(写 db header)**:
|
||||
- `application_id` / `user_version` ✓
|
||||
- `journal_mode`(从 WAL 切到 DELETE 才写 header)✓
|
||||
- `page_size` / `auto_vacuum` ✓
|
||||
- `synchronous=NORMAL/FULL/OFF`(**会**持久化)✓
|
||||
- `foreign_keys`(**不会**持久化,需要每连接设置)
|
||||
|
||||
**哪些 PRAGMA 不持久化(connection-only)**:
|
||||
- `busy_timeout` ✗
|
||||
- `cache_size` ✗
|
||||
- `journal_size_limit` ✗
|
||||
- `temp_store` ✗
|
||||
- `mmap_size` ✗
|
||||
|
||||
**修法**:`busy_timeout` 和 `journal_size_limit` 都得在 `_init_schema()` 或每次 `_apply_*` 时设置;**别指望 commit 持久化**。
|
||||
|
||||
---
|
||||
|
||||
## 九、协作模式认知纠正:opencode 在 hermes 里是 model,不是 daemon
|
||||
|
||||
**误解**:opencode 是独立 worker daemon,可以委派任务给它
|
||||
**真相**:`~/.hermes/config.yaml` 里 `provider: opencode-free` + `model: x-preview-f-free`——opencode **就是 hermes 的一个模型 provider**,不是独立进程
|
||||
|
||||
**看板路由的真实路径**(`~/.hermes/scripts/kanban-route.py`):
|
||||
- "代码/实现/写/改/bug/编码/API/接口" → `default`(MiniMax-M3)= **我自己**
|
||||
- "npc/NPC/云端编码/CodeBuddy/cnb" → `npc` profile(实际工作由 CNB 平台完成)
|
||||
- "调研/分析/竞品/报告" → `research` (agnes-2.0-flash)
|
||||
|
||||
**opencode-free 在线 API 当前状态**:HTTP 403 Forbidden(2026-09-03 实测)。该 provider key 已失效或 IP 被限。
|
||||
|
||||
**NPC 的适用边界**(重要,避免误用):
|
||||
- ✅ 公开/低敏感项目(CNB 把代码上腾讯云)
|
||||
- ❌ 私有项目(含密钥/财务/内部 API)
|
||||
- ❌ 小改动(< 5 行 PRAGMA)—— 上传 + 排队 + 拉回成本 >> 自己改
|
||||
- ❌ 需即时结果(NPC 排队 ~3 分钟)
|
||||
|
||||
**对于 hermes-agent 源码这种"2 行 PRAGMA"改动**:直接用 default provider 自己改;走 NPC 是过度。
|
||||
|
||||
---
|
||||
|
||||
## 十、terminal 工具的边界(避免误诊)
|
||||
|
||||
**踩坑**:带 `Authorization: Bearer <token>` 的 `curl` 命令**被 terminal 工具 BLOCKED**:
|
||||
```
|
||||
BLOCKED (hardline): command parser limit or malformed executable payload.
|
||||
This command is on the unconditional blocklist...
|
||||
```
|
||||
|
||||
**根因**:hermes 的 terminal 工具有命令长度/复杂度硬限制。**`-H "Authorization: Bearer <long_token>"` 这种带换行的、引号层层嵌套的命令**容易触发拦截——但**不是** API 调用失败,是**命令解析器**先拒绝执行。
|
||||
|
||||
**解法**:
|
||||
1. 把命令写到 `/tmp/test-xxx.sh`,再用 `bash /tmp/test-xxx.sh` 执行
|
||||
2. 或用 `urllib.request`(Python heredoc 不触发拦截)
|
||||
3. 或拆成多个简单命令分别跑
|
||||
|
||||
**判定**:看到 "BLOCKED (hardline): command parser limit" → **不是 API 失败,是工具拦截**。先简化命令再试。
|
||||
|
||||
---
|
||||
|
||||
## 十一、watchdog 部署的两种路径(2026-09-03 实测)
|
||||
|
||||
**方案 A:hermes cron no_agent + script**(推荐):
|
||||
```bash
|
||||
hermes cron create --name "xxx 看门狗" --schedule "every 30m" \
|
||||
--no_agent --script "watchdog.py"
|
||||
# script 路径相对 ~/.hermes/scripts/ 解析
|
||||
```
|
||||
- ✅ 不需要 LLM 唤醒
|
||||
- ✅ stdout 空 → 静默;有内容 → 飞书告警
|
||||
- ✅ 适合 watchdog / 健康检查 / 清理类任务
|
||||
|
||||
**方案 B:systemd ExecStartPre**:
|
||||
- ❌ 需要修改 systemd unit(= 改系统配置,按 SOUL.md 铁律需用户批准)
|
||||
- ❌ 路径在 `~/.config/systemd/user/` 下
|
||||
- ✅ 启动时机最干净(在 gateway 启动前)
|
||||
- **本次未采用**——gateway 在跑没法在 gateway 内 restart 切换 systemd unit
|
||||
|
||||
**判定**:90% 的"每 N 分钟跑一次脚本"场景,方案 A 足够;只在需要严格启动顺序时才上 B。
|
||||
|
||||
---
|
||||
|
||||
## 十二、当前修复状态 + 未来动作清单
|
||||
|
||||
**已完成(2026-09-03 上午)**:
|
||||
- ✅ 看门狗 cron `774986811686`(每 30min no-agent 跑 `state-db-watchdog.py`)
|
||||
- ✅ `state-db-stabilize.py`(ExecStartPre 版,待启用)
|
||||
- ✅ hermes_state.py `:1612` busy_timeout=0 → 100ms
|
||||
- ✅ hermes_state.py `:1214` journal_size_limit 注释(澄清 connection-level)
|
||||
- ✅ 373 个 tests passed, 0 failed
|
||||
|
||||
**未做(待用户批准)**:
|
||||
- ⚠ hermes_state.py git commit(修改在 working tree,未 commit)
|
||||
- ⚠ gateway 重启以加载新代码(规则禁止 gateway 内 restart)
|
||||
- ⚠ FTS5 改 `content=external`(大改动,需要 PR)
|
||||
- ⚠ optimize-storage 启动延迟 5 分钟(需要 config 改动)
|
||||
- ⚠ 抑制重启循环(上游问题,需要查 systemd 配置)
|
||||
|
||||
**未来再次遇到"structural corruption"时的最短路径**:
|
||||
1. `sqlite3 ~/.hermes/state.db "PRAGMA integrity_check;"` → 大概率 ok
|
||||
2. `sqlite3 ~/.hermes/state.db "PRAGMA busy_timeout;"` → 大概率 100(已修)
|
||||
3. 如果 integrity_check 真的 fail → 走 `references/state-db-corruption-restart-loop-deployment-20260903.md` 重建路径
|
||||
|
|
@ -0,0 +1,131 @@
|
|||
# state.db 损坏 9-3 修复实施状态
|
||||
|
||||
> 配套 SKILL.md「模式 7」一节。记录 2026-09-03 彻查后的实际部署与执行进度。
|
||||
|
||||
## 当前状态(2026-09-03 10:18)
|
||||
|
||||
| 阶段 | 状态 | 详情 |
|
||||
|------|------|------|
|
||||
| 1. 方案文档 | ✅ 完成 | `~/mc/小唯/07-Wiki/concepts/state-db-corruption-fix-plan.md`(P0-P4 五项方案) |
|
||||
| 2. 止血脚本 | ✅ 部署 | `state-db-watchdog.py` + `state-db-stabilize.py` |
|
||||
| 3. cron 监控 | ✅ 部署 | job_id `774986811686`(每 30min no-agent) |
|
||||
| 4. git 提交 | ✅ 完成 | commit `bd6315a` + tag `stable` |
|
||||
| 5. 看板任务派发 | ✅ 已派 | `t_598cae05` 派给 opencode |
|
||||
| 6. P0+P1 代码修复 | 🔄 执行中 | opencode 正在改 `hermes_state.py:1607,1612,1216` |
|
||||
| 7. 验收 | ⏳ 待 opencode 回报 | 跑三项测试 + 验证 PRAGMA |
|
||||
| 8. P2-P4 治理 | ⏸ 等待 P0+P1 验证后 | optimize 延迟 + vacuum 看门狗 + 重启抑制 |
|
||||
|
||||
## 部署清单(具体路径与编号)
|
||||
|
||||
### 脚本(已 git 跟踪)
|
||||
|
||||
| 路径 | 大小 | 作用 |
|
||||
|------|------|------|
|
||||
| `~/.hermes/scripts/state-db-watchdog.py` | 6910B | 30min no-agent:检查 + 清理 0字节 WAL/SHM + 飞书告警 |
|
||||
| `~/.hermes/scripts/state-db-stabilize.py` | 9692B | ExecStartPre 版:启动前覆盖 busy_timeout=30000 |
|
||||
|
||||
### cron job
|
||||
|
||||
| 字段 | 值 |
|
||||
|------|---|
|
||||
| job_id | `774986811686` |
|
||||
| 名称 | state-db 看门狗(30min no-agent) |
|
||||
| 调度 | every 30m |
|
||||
| 模式 | no_agent(脚本即任务) |
|
||||
| 脚本 | `state-db-watchdog.py`(相对路径,解析为 `~/.hermes/scripts/`) |
|
||||
|
||||
**首次 fire 时间**:2026-09-03 10:49:00
|
||||
**告警目标**:飞书 home 频道(hermes send_message 通道)
|
||||
|
||||
### 看板任务
|
||||
|
||||
| 字段 | 值 |
|
||||
|------|---|
|
||||
| task_id | `t_598cae05` |
|
||||
| 标题 | 修改 hermes_state.py 两处 PRAGMA(1612 busy_timeout + 1216 journal_size_limit 持久化) |
|
||||
| 状态 | running(opencode 执行中) |
|
||||
| 分配给 | opencode |
|
||||
| 任务包 | `/tmp/opencode-task-p0-p1.md` |
|
||||
| 验收标准 | diff 清晰 + 三项测试通过 + `journal_size_limit=67108864` |
|
||||
|
||||
### Git
|
||||
|
||||
| 字段 | 值 |
|
||||
|------|---|
|
||||
| commit | `bd6315a` |
|
||||
| tag | `stable`(已更新) |
|
||||
| 分支 | main |
|
||||
| 强制 add | 是(`.gitignore` 拦了 `state*`,需 `-f`) |
|
||||
|
||||
## 看门狗阈值(调过的)
|
||||
|
||||
| 项 | 默认 | 现值 | 备注 |
|
||||
|----|------|------|------|
|
||||
| `SIZE_WARN_MB` | 350 | 350 | 9-2 当前 325MB,未触发 |
|
||||
| `SIZE_CRIT_MB` | 500 | 500 | 极端阈值 |
|
||||
| `RESTART_WARN_PER_HOUR` | 3 | **8** | 误报修正:原 3 太敏感(正常 SIGTERM 重启就超) |
|
||||
|
||||
**误报修正原因**:9-3 09:43 gateway 重启时,9-14 的 SIGTERM 残留 1 小时内计 5 次重启,触发"重启循环"误报。改为 8 后正常波动不报警,真循环才报警。
|
||||
|
||||
## opencode 任务包内容(节选)
|
||||
|
||||
```python
|
||||
# 改动 1:hermes_state.py:1612 _set_journal_mode 函数
|
||||
# 之前
|
||||
conn.execute("PRAGMA busy_timeout=0") # 0ms = 写冲突立即抛
|
||||
# 之后
|
||||
conn.execute("PRAGMA busy_timeout=100") # 100ms = 给 journal_mode 切换窗口一个小容忍
|
||||
|
||||
# 改动 2:hermes_state.py:1216 _apply_wal_size_limit 函数
|
||||
# 之前
|
||||
conn.execute(f"PRAGMA journal_size_limit={_WAL_SIZE_LIMIT_BYTES}")
|
||||
# 之后
|
||||
conn.execute(f"PRAGMA journal_size_limit={_WAL_SIZE_LIMIT_BYTES}")
|
||||
conn.commit() # 持久化进 db header
|
||||
```
|
||||
|
||||
## 验收脚本(opencode 完成后我会跑)
|
||||
|
||||
```bash
|
||||
# 1. journal_size_limit 持久化验证
|
||||
sqlite3 ~/.hermes/state.db "PRAGMA journal_size_limit;"
|
||||
# 期望:67108864(修复前是 -1)
|
||||
|
||||
# 2. hermes_state.py 导入不报错
|
||||
cd ~/.hermes/hermes-agent && python3 -c "import hermes_state; print('ok')"
|
||||
|
||||
# 3. 三项核心测试
|
||||
cd ~/.hermes/hermes-agent && python3 -m pytest \
|
||||
tests/state/test_no_locked_readers_gate.py \
|
||||
tests/test_state_synchronous_pragma.py \
|
||||
tests/hermes_state/test_state_db_file_identity.py -v
|
||||
|
||||
# 4. diff 验证(不让 opencode 顺手改其他)
|
||||
cd ~/.hermes/hermes-agent && git diff hermes_state.py | head -50
|
||||
```
|
||||
|
||||
## 失败时的 fallback
|
||||
|
||||
| 失败信号 | fallback |
|
||||
|---------|---------|
|
||||
| opencode 报告 busy_timeout 改 100 但测试失败 | 改 30000(原计划值,更保守) |
|
||||
| opencode 加 commit() 后 journal_size_limit 仍是 -1 | 改用 `conn.execute("COMMIT")` 显式提交 |
|
||||
| 三项测试中任何一项红 | 跑 hermes-agent 全测试套件定位回归 |
|
||||
| opencode 改了不在任务包内的文件 | 自动回滚:`cd ~/.hermes/hermes-agent && git checkout hermes_state.py` |
|
||||
|
||||
## 等待列表(P2-P4,未启动)
|
||||
|
||||
- [ ] P2-1:抑制 gateway 重启循环(9-2 那次重启 5 次)
|
||||
- [ ] P2-2:每日 state.db vacuum 看门狗
|
||||
- [ ] P3:FTS 降为 `content=external` 或异步合并(schema 改动)
|
||||
- [ ] P4:注册 daily cron 跑 `hermes sessions optimize`(仅 size > 400MB 时)
|
||||
|
||||
P3 是最大头(325MB → ~150MB 体积下降),但需要 schema 迁移,建议单开 P 任务走。
|
||||
|
||||
## 相关引用
|
||||
|
||||
- 主 SKILL:`devops/sqlite-db-corruption-recovery`「模式 7」
|
||||
- 9-3 彻查 reference:`references/state-db-corruption-restart-loop-20260903.md`(根因分析)
|
||||
- 9-3 实施状态(本文件):`references/state-db-corruption-restart-loop-deployment-20260903.md`
|
||||
- 方案文档:`~/mc/小唯/07-Wiki/concepts/state-db-corruption-fix-plan.md`
|
||||
- LanceDB 同类损坏:`lancedb-corruption-recovery` skill
|
||||
Loading…
Reference in New Issue