auto-snapshot 2026-09-02 03:00:29
This commit is contained in:
小唯 A06
parent
5d3b64de0c
commit
165c75c2fc
|
|
@ -38,7 +38,7 @@ check_mount() {
|
|||
log "⚠️ 服务器未挂载，尝试连接..."
|
||||
# -n: non-interactive，cron 环境无密码输入机会，立即失败不卡住
|
||||
# || true: 失败是预期路径（服务器不在线），由下方 mountpoint 判断处理，避免 set -e 提前退出
|
||||
sudo -n mount -t cifs //192.168.123.11/beifen /mnt/server-backup \
|
||||
sudo -n mount -t cifs //192.168.188.11/beifen /mnt/server-backup \
|
||||
-o username=administrator,password=xue.2538,uid=1000,gid=1000,iocharset=utf8,file_mode=0755,dir_mode=0755 2>/dev/null || true
|
||||
sleep 2
|
||||
if ! mountpoint -q /mnt/server-backup; then
|
||||
|
|
|
|||
|
|
@ -0,0 +1,129 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
NewAPI 观测脚本 — 持续采集主网关健康/延迟/路由数据
|
||||
每 6h 由 cron 触发（no_agent 模式）：
|
||||
- 正常时静默（数据追加到 JSONL 观测日志）
|
||||
- 异常时输出报警（cron 会自动推送）
|
||||
数据用途：监测 NewAPI + 9 个 NIM key 池健康度
|
||||
替代 omniroute-observe.py（OmniRoute 已于 2026-09-02 关停）
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import requests
|
||||
from datetime import datetime, timezone
|
||||
|
||||
API = "http://127.0.0.1:3000/v1"
|
||||
# api-test-token 是 NewAPI 唯一启用 token (user_id=1)
|
||||
KEY = "0ExNiLblJvIWBDpkS50fwOBw4MmqLyKdHJK5iQtlw9dOMWBP"
|
||||
STATE_DIR = os.path.expanduser("~/.hermes/newapi-observe")
|
||||
LOG = os.path.join(STATE_DIR, "observations.jsonl")
|
||||
STATE_FILE = os.path.join(STATE_DIR, "state.json")
|
||||
|
||||
# 测试用的模型组合 — 覆盖 NewAPI 真实 channel
|
||||
# - sensenova-free: 主 channel (sensenova deepseek-v4-flash)
|
||||
# - minimaxai/minimax-m3: NIM-k1 (NVIDIA 集成, 9 个 key 池)
|
||||
# - google/gemma-4-31b-it: NIM 池可用模型
|
||||
TEST_MODELS = [
|
||||
"deepseek-v4-flash", # sensenova 通道
|
||||
"minimaxai/minimax-m3", # NIM-k1 (已知可用)
|
||||
"google/gemma-4-31b-it", # NIM 池备选
|
||||
]
|
||||
|
||||
|
||||
def api_call(model, max_tokens=20):
|
||||
"""发一次真实请求，返回 (ok, latency_ms, model, content_len)"""
|
||||
t0 = time.time()
|
||||
try:
|
||||
r = requests.post(f"{API}/chat/completions",
|
||||
json={"model": model,
|
||||
"messages": [{"role": "user", "content": "ping"}],
|
||||
"max_tokens": max_tokens},
|
||||
headers={"Authorization": f"Bearer {KEY}"},
|
||||
timeout=30, stream=False)
|
||||
elapsed = round((time.time() - t0) * 1000)
|
||||
if r.status_code == 200:
|
||||
try:
|
||||
data = r.json()
|
||||
m = data.get("model", model)
|
||||
choices = data.get("choices", [])
|
||||
content_len = len(choices[0].get("message", {}).get("content", "")) if choices else 0
|
||||
return True, elapsed, m, content_len
|
||||
except Exception:
|
||||
return True, elapsed, model, 0
|
||||
else:
|
||||
return False, elapsed, f"HTTP {r.status_code}", 0
|
||||
except Exception as e:
|
||||
return False, round((time.time() - t0) * 1000), f"EXC: {str(e)[:50]}", 0
|
||||
|
||||
|
||||
def check_service():
|
||||
"""检查 NewAPI 服务 + models API 是否健康"""
|
||||
try:
|
||||
r = requests.get(f"{API}/models",
|
||||
headers={"Authorization": f"Bearer {KEY}"},
|
||||
timeout=10)
|
||||
if r.status_code == 200:
|
||||
models = r.json().get("data", [])
|
||||
return True, len(models)
|
||||
return False, 0
|
||||
except Exception:
|
||||
return False, 0
|
||||
|
||||
|
||||
def main():
|
||||
# 确保 state 目录存在
|
||||
os.makedirs(STATE_DIR, exist_ok=True)
|
||||
|
||||
# 1. 服务健康
|
||||
svc_ok, model_count = check_service()
|
||||
if not svc_ok:
|
||||
print(f"🚨 NewAPI 服务不可达或 models API 失败")
|
||||
return
|
||||
|
||||
# 2. 测试每个模型
|
||||
results = []
|
||||
for m in TEST_MODELS:
|
||||
ok, lat, routed, content_len = api_call(m)
|
||||
results.append({
|
||||
"model": m,
|
||||
"ok": ok,
|
||||
"latency_ms": lat,
|
||||
"routed_to": routed,
|
||||
"content_len": content_len
|
||||
})
|
||||
|
||||
# 3. 判断是否有异常
|
||||
fails = [r for r in results if not r["ok"]]
|
||||
slow = [r for r in results if r["ok"] and r["latency_ms"] > 8000]
|
||||
|
||||
# 4. 追加 JSONL 观测记录
|
||||
obs = {
|
||||
"ts": datetime.now(timezone.utc).isoformat(),
|
||||
"service": "ok" if svc_ok else "fail",
|
||||
"model_count": model_count,
|
||||
"tests": results
|
||||
}
|
||||
with open(LOG, "a") as f:
|
||||
f.write(json.dumps(obs) + "\n")
|
||||
|
||||
# 5. 写 state.json（供其他脚本查询）
|
||||
with open(STATE_FILE, "w") as f:
|
||||
json.dump(obs, f, indent=2)
|
||||
|
||||
# 6. 报警逻辑
|
||||
if fails:
|
||||
msgs = [f"❌ {r['model']}: {r['routed_to']} ({r['latency_ms']}ms)" for r in fails]
|
||||
print(f"🚨 NewAPI 异常 ({len(fails)}/{len(results)} 失败):")
|
||||
for m in msgs:
|
||||
print(m)
|
||||
elif slow:
|
||||
msgs = [f"⚠️ {r['model']}: {r['latency_ms']}ms" for r in slow]
|
||||
print(f"⚠️ NewAPI 慢响应 ({len(slow)}/{len(results)} > 8s):")
|
||||
for m in msgs:
|
||||
print(m)
|
||||
# else: 静默（健康）
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
NewAPI 周度评估报告 — 汇总观测数据，输出评估结论
|
||||
每周日 18:00 由 cron 触发，输出可读报告（非静默，始终输出）
|
||||
评估维度：可用性 / 延迟 / 路由一致性
|
||||
替代 omniroute-weekly-report.py（OmniRoute 已于 2026-09-02 关停）
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import statistics
|
||||
from datetime import datetime, timezone, timedelta
|
||||
|
||||
STATE_DIR = os.path.expanduser("~/.hermes/newapi-observe")
|
||||
LOG = os.path.join(STATE_DIR, "observations.jsonl")
|
||||
|
||||
# 评估阈值
|
||||
LATENCY_WARN_MS = 8000 # 平均延迟超 8s 警告
|
||||
SUCCESS_RATE_MIN = 0.90 # 成功率低于 90% 警告
|
||||
|
||||
|
||||
def load_observations(days=7):
|
||||
if not os.path.exists(LOG):
|
||||
return []
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=days)
|
||||
obs = []
|
||||
with open(LOG) as f:
|
||||
for line in f:
|
||||
try:
|
||||
d = json.loads(line)
|
||||
ts = datetime.fromisoformat(d["ts"])
|
||||
if ts >= cutoff:
|
||||
obs.append(d)
|
||||
except Exception:
|
||||
continue
|
||||
return obs
|
||||
|
||||
|
||||
def main():
|
||||
obs = load_observations(7)
|
||||
n = len(obs)
|
||||
|
||||
print(f"=== NewAPI 周度评估（最近 7 天）===")
|
||||
print(f"观测次数: {n}")
|
||||
print(f"生成时间: {datetime.now(timezone.utc).isoformat()}")
|
||||
print()
|
||||
|
||||
if n == 0:
|
||||
print("⚠️ 无观测数据（cron 可能在观测脚本切换期间未运行）")
|
||||
return
|
||||
|
||||
# 1. 服务可用性
|
||||
svc_ok = sum(1 for o in obs if o.get("service") == "ok")
|
||||
svc_rate = svc_ok / n
|
||||
print(f"【服务可用性】 {svc_ok}/{n} = {svc_rate*100:.1f}%")
|
||||
|
||||
# 2. 按模型统计
|
||||
by_model = {}
|
||||
for o in obs:
|
||||
for t in o.get("tests", []):
|
||||
m = t["model"]
|
||||
if m not in by_model:
|
||||
by_model[m] = {"ok": 0, "total": 0, "latencies": []}
|
||||
by_model[m]["total"] += 1
|
||||
if t["ok"]:
|
||||
by_model[m]["ok"] += 1
|
||||
by_model[m]["latencies"].append(t["latency_ms"])
|
||||
|
||||
print(f"\n【按模型表现】")
|
||||
for m, d in sorted(by_model.items()):
|
||||
rate = d["ok"] / d["total"] * 100 if d["total"] else 0
|
||||
if d["latencies"]:
|
||||
avg = statistics.mean(d["latencies"])
|
||||
p95 = sorted(d["latencies"])[int(len(d["latencies"])*0.95)] if len(d["latencies"]) > 1 else d["latencies"][0]
|
||||
else:
|
||||
avg = p95 = 0
|
||||
status = "✅" if rate >= SUCCESS_RATE_MIN*100 and avg < LATENCY_WARN_MS else "⚠️"
|
||||
print(f" {status} {m}: {d['ok']}/{d['total']} = {rate:.0f}%, 平均 {avg:.0f}ms, P95 {p95:.0f}ms")
|
||||
|
||||
# 3. 总体评估
|
||||
total_tests = sum(d["total"] for d in by_model.values())
|
||||
total_ok = sum(d["ok"] for d in by_model.values())
|
||||
overall_rate = total_ok / total_tests * 100 if total_tests else 0
|
||||
print(f"\n【总体】 {total_ok}/{total_tests} = {overall_rate:.1f}%")
|
||||
|
||||
if overall_rate < SUCCESS_RATE_MIN*100:
|
||||
print("⚠️ 建议: 检查 NewAPI 上游 channel 健康度")
|
||||
else:
|
||||
print("✅ 评估: NewAPI 表现健康，可作为本地主网关")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because one or more lines are too long
|
|
@ -346,14 +346,14 @@
|
|||
"created_by": "agent",
|
||||
"last_patched_at": "2026-09-01T13:24:30.353399+00:00",
|
||||
"last_reused_patch_generation": 2,
|
||||
"last_used_at": "2026-09-01T14:46:17.329481+00:00",
|
||||
"last_viewed_at": "2026-09-01T14:46:17.312828+00:00",
|
||||
"last_used_at": "2026-09-01T17:40:43.660373+00:00",
|
||||
"last_viewed_at": "2026-09-01T17:40:43.644850+00:00",
|
||||
"patch_count": 4,
|
||||
"patch_generation": 2,
|
||||
"pinned": false,
|
||||
"state": "active",
|
||||
"use_count": 16,
|
||||
"view_count": 16
|
||||
"use_count": 17,
|
||||
"view_count": 17
|
||||
},
|
||||
"blocked-page-recovery": {
|
||||
"archived_at": null,
|
||||
|
|
@ -839,14 +839,14 @@
|
|||
"created_by": "agent",
|
||||
"last_patched_at": "2026-08-26T05:42:43.675272+00:00",
|
||||
"last_reused_patch_generation": 2,
|
||||
"last_used_at": "2026-09-01T12:49:40.492187+00:00",
|
||||
"last_viewed_at": "2026-09-01T12:49:40.486311+00:00",
|
||||
"last_used_at": "2026-09-01T15:52:18.292077+00:00",
|
||||
"last_viewed_at": "2026-09-01T15:52:18.282846+00:00",
|
||||
"patch_count": 98,
|
||||
"patch_generation": 2,
|
||||
"pinned": false,
|
||||
"state": "active",
|
||||
"use_count": 88,
|
||||
"view_count": 88
|
||||
"use_count": 89,
|
||||
"view_count": 89
|
||||
},
|
||||
"devops/bge-embed-crash-loop-fix": {
|
||||
"archived_at": null,
|
||||
|
|
@ -1655,14 +1655,14 @@
|
|||
"created_by": null,
|
||||
"last_patched_at": "2026-08-12T05:13:32.138839+00:00",
|
||||
"last_reused_patch_generation": 1,
|
||||
"last_used_at": "2026-09-01T13:17:29.163415+00:00",
|
||||
"last_viewed_at": "2026-09-01T13:17:29.139591+00:00",
|
||||
"last_used_at": "2026-09-01T17:41:09.065587+00:00",
|
||||
"last_viewed_at": "2026-09-01T17:41:09.051579+00:00",
|
||||
"patch_count": 81,
|
||||
"patch_generation": 1,
|
||||
"pinned": false,
|
||||
"state": "active",
|
||||
"use_count": 139,
|
||||
"view_count": 128
|
||||
"use_count": 140,
|
||||
"view_count": 129
|
||||
},
|
||||
"hermes-venv-dependency-safety": {
|
||||
"archived_at": null,
|
||||
|
|
@ -1813,6 +1813,21 @@
|
|||
"use_count": 5,
|
||||
"view_count": 5
|
||||
},
|
||||
"kanban-router": {
|
||||
"archived_at": null,
|
||||
"created_at": "2026-09-01T15:56:37.956717+00:00",
|
||||
"created_by": null,
|
||||
"last_patched_at": null,
|
||||
"last_reused_patch_generation": 0,
|
||||
"last_used_at": "2026-09-01T15:56:37.961319+00:00",
|
||||
"last_viewed_at": "2026-09-01T15:56:37.956727+00:00",
|
||||
"patch_count": 0,
|
||||
"patch_generation": 0,
|
||||
"pinned": false,
|
||||
"state": "active",
|
||||
"use_count": 1,
|
||||
"view_count": 1
|
||||
},
|
||||
"kanban-routing": {
|
||||
"archived_at": null,
|
||||
"created_at": "2026-09-01T13:21:22.328206+00:00",
|
||||
|
|
@ -1948,16 +1963,16 @@
|
|||
"archived_at": null,
|
||||
"created_at": "2026-08-01T13:47:26.532313+00:00",
|
||||
"created_by": "agent",
|
||||
"last_patched_at": "2026-08-20T18:26:57.337252+00:00",
|
||||
"last_reused_patch_generation": 29,
|
||||
"last_used_at": "2026-08-29T10:58:53.508614+00:00",
|
||||
"last_viewed_at": "2026-08-29T10:58:53.501617+00:00",
|
||||
"patch_count": 35,
|
||||
"patch_generation": 29,
|
||||
"last_patched_at": "2026-09-01T18:34:24.564573+00:00",
|
||||
"last_reused_patch_generation": 30,
|
||||
"last_used_at": "2026-09-01T18:33:03.244527+00:00",
|
||||
"last_viewed_at": "2026-09-01T18:33:03.239992+00:00",
|
||||
"patch_count": 40,
|
||||
"patch_generation": 34,
|
||||
"pinned": false,
|
||||
"state": "active",
|
||||
"use_count": 39,
|
||||
"view_count": 39
|
||||
"use_count": 45,
|
||||
"view_count": 45
|
||||
},
|
||||
"llm-wiki": {
|
||||
"archived_at": null,
|
||||
|
|
@ -2861,14 +2876,14 @@
|
|||
"created_by": "agent",
|
||||
"last_patched_at": "2026-09-01T13:48:27.218650+00:00",
|
||||
"last_reused_patch_generation": 27,
|
||||
"last_used_at": "2026-09-01T14:46:17.334034+00:00",
|
||||
"last_viewed_at": "2026-09-01T14:46:17.325238+00:00",
|
||||
"last_used_at": "2026-09-01T18:32:51.753345+00:00",
|
||||
"last_viewed_at": "2026-09-01T18:32:51.742812+00:00",
|
||||
"patch_count": 230,
|
||||
"patch_generation": 27,
|
||||
"pinned": false,
|
||||
"state": "active",
|
||||
"use_count": 191,
|
||||
"view_count": 191
|
||||
"use_count": 200,
|
||||
"view_count": 200
|
||||
},
|
||||
"self-hosted-tunneling": {
|
||||
"archived_at": null,
|
||||
|
|
|
|||
|
|
@ -7,10 +7,12 @@ description: LLM 网关运维 — OmniRoute/NewAPI 类 AI 网关的评估、部
|
|||
|
||||
## 触发场景
|
||||
- 用户分享新 AI 网关/路由项目（如 OmniRoute、OneAPI 类）→ 评估是否替代现有 NewAPI
|
||||
- **用户问"X 网关是否有必要保留/关停"→ 拉真实调用量做决策**（不只是看进程在不在）→ 决策后看 `references/gateway-shutdown-procedure-20260902.md` 走 6 步标准流程
|
||||
- 部署/试跑网关、做 systemd 常驻、接入看门狗
|
||||
- 写脚本调网关 API（chat/completions）遇超时/挂起
|
||||
- 把 cron 任务从 NewAPI 切换到其他网关 provider
|
||||
- 需要持续观测网关稳定性、为"是否全面切换"提供数据
|
||||
- 周期性自检网关使用率（cron `0 9 1 * *`）
|
||||
|
||||
## 核心方法论：评估是否替代现有网关
|
||||
|
||||
|
|
@ -19,6 +21,10 @@ description: LLM 网关运维 — OmniRoute/NewAPI 类 AI 网关的评估、部
|
|||
3. **生产链路绝不动**：付费主模型（如 deepseek-v4-flash 走官方 API）不切
|
||||
4. **观测 2-4 周**再决定是否迁移——必须有量化数据支撑，不拍脑袋
|
||||
5. 评估维度：稳定性/免费模型供给/路由能力/成本控制/国内网络兼容/生态成熟度
|
||||
6. **logs 表 0 调用 ≠ 服务僵尸**（2026-09-02 教训）：`logs` 是写日志表，不代表服务可用。判断"僵尸"必跑 3 步——L1 进程活 + L2 直连 200 + L3 `provider_models_cache.json` 列了模型。三步全过才认活（详细见 `references/gateway-real-usage-evaluation-20260901.md` 的"修正版"章节）
|
||||
7. **进程内存要算子进程总和**（2026-09-02 教训）：OmniRoute 主进程 46MB + 子进程 394MB = **441MB 总占用**，只看主进程会少算 90%
|
||||
8. **HTTP 410/404 ≠ key 失效**（2026-09-02 教训）：NVIDIA 端下架特定模型（410 "has retired"）≠ API key 废了——**同一个 key 通常能跑多个模型**，重测 `minimaxai/minimax-m3` 或 `google/gemma-4-31b-it` 等常用模型验证。`9 个 NIM key 8 个模型下架` 的判断是错的——9/9 全部活跃，只是测错了模型。
|
||||
9. **评估完成后如何安全关停**（2026-09-02 OmniRoute 实战）：6 步标准流程——①备份 4 件套（脚本/服务/配置/状态）②先写替代观测脚本（不能只关停就跑，否则 cron 一直报错）③改 cron 任务（no_agent script 字段必须相对路径）④systemctl stop + disable ⑤验证替代品 + 内存释放 ⑥归档不删原可执行文件。详见 `references/gateway-shutdown-procedure-20260902.md`。**patch 工具拒绝改 config.yaml**——provider 段留着不动通常没事（端口已死不会触发）。
|
||||
|
||||
## OmniRoute 部署（npm 全局）
|
||||
|
||||
|
|
@ -410,4 +416,6 @@ NewAPI 添加渠道有两种方式：API（会 panic，版本 bug）和 Web UI
|
|||
- `references/newapi-channel-management-20260820.md` — NewAPI 渠道管理坑：SQLite 直改完整性、API panic bug、Web UI 替代方案
|
||||
- `references/newapi-sensenova-channel-20260821.md` — Sensenova Token Plan 免费接入：5 模型 + abilities 联动 + reserved word 坑
|
||||
- `references/openclaw-config-validation-pitfalls-20260821.md` — OpenClaw openclaw.json 配置验证坑：defaultModel/defaultProvider 非法、output 字段不合法、models 数组必填
|
||||
- 微信(iLink)/QQ 平台接入 Hermes gateway 的 .env 环境变量铁律见 **hermes-debug 第 6 节**（WEIXIN_TOKEN/QQ_APP_ID 只认 .env 不读 config.yaml 段；v0.20 需 WEIXIN_ALLOW_ALL_USERS=true；QQ 用 `hermes pairing approve qqbot <CODE>` 授权）
|
||||
- `references/gateway-real-usage-evaluation-20260901.md` — 评估方法论：3 步框架（L1 进程/L2 配置/L3 调用）+ 决策树 + 410 ≠ key 失效陷阱
|
||||
- `references/gateway-shutdown-procedure-20260902.md` — **关停标准流程**：6 步（备份→写替代脚本→改 cron→systemctl stop+disable→验证→归档）+ 4 个 cron/patch 陷阱
|
||||
- `references/2026-08-04-omniroute-cron-failure.md` — OmniRoute cron 504 失败案例（auto/chat 路由慢模型）
|
||||
|
|
|
|||
Loading…
Reference in New Issue