288 lines
11 KiB
Python
288 lines
11 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
distill-model-watchdog.py — 蒸馏模型看门狗（30min 轻量探针）
|
||
============================================================
|
||
守护对象: 织忆 distill (zhiyid.service LLM_MODEL) + TencentDB L1 (tdai-gateway.yaml model)
|
||
|
||
为什么需要:
|
||
- model-health.py 每 6h 才跑，免费模型挂了要等半天
|
||
- model-health.py 测"对话能力"，distill 需要"JSON 输出能力"，探针类型不对
|
||
- 免费模型经常挂（2026-08-02 实测 m3 连续空响应、gpt-oss content=null）
|
||
|
||
逻辑:
|
||
1. 读当前 LLM_MODEL（zhiyid.service）
|
||
2. 测 JSON 输出能力（真实调用，内容可解析为 JSON 才通过）
|
||
3. 通过 → 静默（空输出 = no-agent cron 不发送）
|
||
4. 失败 → 按优先级从候选池逐个测 → 找到第一个可用 → 更新两处配置 → 重启 → 飞书报警
|
||
5. 全部候选失败 → 飞书报警"所有蒸馏模型都挂了"
|
||
|
||
候选池顺序 = 2026-08-02 实测 JSON 输出可用 + 按质量排序
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import re
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
import urllib.request
|
||
import urllib.error
|
||
from datetime import datetime, timezone
|
||
|
||
API = "http://127.0.0.1:3000/v1"
|
||
AGNES_API = "https://apihub.agnes-ai.com/v1"
|
||
SENSENOVA_API = "https://token.sensenova.cn/v1"
|
||
ZHIPU_API = "https://open.bigmodel.cn/api/paas/v4"
|
||
KEY_ENV = None # 从 zhiyid.service 读取
|
||
AGNES_KEY = ""
|
||
SENSENOVA_KEY = ""
|
||
ZHIPU_KEY = ""
|
||
for _l in open(os.path.expanduser("~/.hermes/.env"), encoding="utf-8"):
|
||
if _l.startswith("AGNES_API_KEY=") and not _l.startswith("#"):
|
||
AGNES_KEY = _l.strip().split("=", 1)[1]
|
||
if _l.startswith("SENSENOVA_API_KEY=") and not _l.startswith("#"):
|
||
SENSENOVA_KEY = _l.strip().split("=", 1)[1]
|
||
ZHIYID_SERVICE = os.path.expanduser("~/.config/systemd/user/zhiyid.service")
|
||
TDDB_CONFIG = os.path.expanduser("~/.memory-tencentdb/memory-tdai/tdai-gateway.yaml")
|
||
FEISHU_WEBHOOK = "https://open.feishu.cn/open-apis/bot/v2/hook/446db983-e392-4d2c-bfb8-f9060e5df3ad"
|
||
|
||
# 候选池（优先级降序，2026-08-23 实测延迟排序）
|
||
CANDIDATE_POOL = [
|
||
"glm-4-flash", # ⭐ 智谱直连，721ms 最快
|
||
"nvidia/nemotron-3-super-120b-a12b", # NVIDIA NewAPI，814ms
|
||
"agnes-2.0-flash", # Agnes 直连，983ms
|
||
"deepseek-v4-flash", # 商汤直连，1613ms
|
||
]
|
||
|
||
def _get_endpoint(model: str) -> tuple:
|
||
"""agnes 走 Agnes API，商汤走直连 SenseNova API，其余走 NewAPI"""
|
||
if model.startswith("agnes-"):
|
||
return AGNES_API, AGNES_KEY
|
||
if model.startswith(("deepseek-", "glm-", "sensenova-")):
|
||
return SENSENOVA_API, SENSENOVA_KEY
|
||
return API, _get_key()
|
||
|
||
# 已知绝对不可用的（不重复测，直接跳过）
|
||
KNOWN_BAD = [
|
||
"openai/gpt-oss-120b", "openai/gpt-oss-20b", # reasoning, content=null
|
||
"minimaxai/minimax-m3", "minimaxai/minimax-m2.7", # 空响应/EOL
|
||
"stepfun-ai/step-3.5-flash", "qwen/qwen3.5-122b-a10b", # EOL
|
||
"mistralai/mistral-large-3-675b", "mistralai/mistral-large-3-675b-instruct-2512", # EOL/无渠道
|
||
"nvidia/nemotron-3-super-120b-a12b", # reasoning 回显
|
||
"mistralai/mistral-medium-3.5-128b", # 非JSON
|
||
"deepseek-ai/deepseek-v3.2", # openai_error
|
||
]
|
||
|
||
# ============ 工具 ============
|
||
|
||
def _get_key():
|
||
"""从 zhiyid.service 读 LLM_API_KEY（唯一真源）"""
|
||
try:
|
||
with open(ZHIYID_SERVICE) as f:
|
||
for line in f:
|
||
m = re.search(r"LLM_API_KEY=(\S+)", line)
|
||
if m:
|
||
return m.group(1)
|
||
except Exception:
|
||
pass
|
||
return None
|
||
|
||
def _get_current_model():
|
||
"""读 zhiyid.service 当前 LLM_MODEL"""
|
||
try:
|
||
with open(ZHIYID_SERVICE) as f:
|
||
for line in f:
|
||
m = re.search(r"LLM_MODEL=(\S+)", line)
|
||
if m:
|
||
return m.group(1)
|
||
except Exception:
|
||
pass
|
||
return None
|
||
|
||
def _test_json(model: str, timeout: int = 25) -> bool:
|
||
"""真实调用测试：返回内容必须是可解析的 JSON（剥离 code fence 后）"""
|
||
# ⚠️ 2026-08-17：Agnes 2.0-flash 是推理模型（reasoning_tokens 占大头），
|
||
# max_tokens 必须 ≥400 否则 JSON 被截断（finish_reason=length）→ 误判模型挂了
|
||
payload = json.dumps({
|
||
"model": model,
|
||
"messages": [
|
||
{"role": "system", "content": "输出严格JSON，不要markdown代码块"},
|
||
{"role": "user", "content": '提取实体：牧尘喜欢简洁。输出 {"entities":[],"decisions":[],"conclusions":[]} 格式'},
|
||
],
|
||
"max_tokens": 500,
|
||
}).encode()
|
||
base_url, k = _get_endpoint(model)
|
||
req = urllib.request.Request(
|
||
f"{base_url}/chat/completions", data=payload,
|
||
headers={"Authorization": f"Bearer {k}", "Content-Type": "application/json"},
|
||
method="POST",
|
||
)
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||
body = json.loads(resp.read())
|
||
msg = body.get("choices", [{}])[0].get("message", {}) or {}
|
||
content = msg.get("content") or ""
|
||
if not content.strip():
|
||
return False # reasoning 模型 content=null
|
||
cleaned = re.sub(r"```[a-zA-Z]*\s*", "", content)
|
||
cleaned = re.sub(r"\s*```", "", cleaned).strip()
|
||
# 兜底：如果还有残留，直接找第一个 { 到最后一个 }
|
||
if not cleaned.startswith("{"):
|
||
start = cleaned.find("{")
|
||
end = cleaned.rfind("}")
|
||
if start >= 0 and end > start:
|
||
cleaned = cleaned[start:end+1]
|
||
json.loads(cleaned)
|
||
return True
|
||
except Exception:
|
||
return False
|
||
|
||
def _update_zhiyid(model: str) -> bool:
|
||
"""更新 zhiyid.service 的 LLM_MODEL + LLM_ENDPOINT/LLM_API_BASE/LLM_API_KEY 联动
|
||
agnes → Agnes 直连；商汤 → SenseNova 直连；其他 → NewAPI"""
|
||
try:
|
||
agnes_key = AGNES_KEY or ""
|
||
sensenova_key = SENSENOVA_KEY or ""
|
||
newapi_key = _get_key() or ""
|
||
# agnes 模型 → Agnes 直连
|
||
if model.startswith("agnes-"):
|
||
endpoint = "https://apihub.agnes-ai.com/v1/chat/completions"
|
||
api_base = "https://apihub.agnes-ai.com/v1"
|
||
key = agnes_key
|
||
# 商汤模型 → SenseNova 直连（绕过 NewAPI）
|
||
elif model.startswith(("deepseek-", "glm-", "sensenova-")):
|
||
endpoint = "https://token.sensenova.cn/v1/chat/completions"
|
||
api_base = "https://token.sensenova.cn/v1"
|
||
key = sensenova_key
|
||
else:
|
||
endpoint = "http://127.0.0.1:3000/v1/chat/completions"
|
||
api_base = "http://127.0.0.1:3000/v1"
|
||
key = newapi_key
|
||
with open(ZHIYID_SERVICE) as f:
|
||
content = f.read()
|
||
# 逐个替换（找不到就保持原样）
|
||
new_content = content
|
||
new_content = re.sub(r"LLM_ENDPOINT=\S+", f"LLM_ENDPOINT={endpoint}", new_content)
|
||
new_content = re.sub(r"LLM_API_BASE=\S+", f"LLM_API_BASE={api_base}", new_content)
|
||
new_content = re.sub(r"LLM_MODEL=\S+", f"LLM_MODEL={model}", new_content)
|
||
if key:
|
||
new_content = re.sub(r"LLM_API_KEY=\S+", f"LLM_API_KEY={key}", new_content)
|
||
if new_content == content:
|
||
return False
|
||
with open(ZHIYID_SERVICE, "w") as f:
|
||
f.write(new_content)
|
||
subprocess.run(["systemctl", "--user", "daemon-reload"], check=True)
|
||
subprocess.run(["systemctl", "--user", "restart", "zhiyid"], check=True)
|
||
return True
|
||
except Exception as e:
|
||
print(f" ❌ 更新 zhiyid.service 失败: {e}")
|
||
return False
|
||
|
||
def _update_tddb(model: str) -> bool:
|
||
"""更新 tdai-gateway.yaml 的 llm.model + 重启（若文件存在）
|
||
只改 llm 段的 model，绝不动 embedding.model（bge-m3）"""
|
||
if not os.path.exists(TDDB_CONFIG):
|
||
return False
|
||
try:
|
||
with open(TDDB_CONFIG) as f:
|
||
lines = f.readlines()
|
||
in_llm = False
|
||
changed = False
|
||
for i, line in enumerate(lines):
|
||
if line.strip() == "llm:":
|
||
in_llm = True
|
||
continue
|
||
if in_llm and line.strip().startswith("model:"):
|
||
indent = line[:len(line) - len(line.lstrip())]
|
||
lines[i] = f"{indent}model: {model}\n"
|
||
changed = True
|
||
in_llm = False
|
||
break
|
||
if not changed:
|
||
return False
|
||
new_content = "".join(lines)
|
||
# 备份
|
||
bak = TDDB_CONFIG + ".bak-watchdog"
|
||
with open(bak, "w") as f:
|
||
f.write(new_content)
|
||
with open(TDDB_CONFIG, "w") as f:
|
||
f.write(new_content)
|
||
subprocess.run(["systemctl", "--user", "restart", "tdai-gateway"], check=True)
|
||
return True
|
||
except Exception as e:
|
||
print(f" ❌ 更新 tdai-gateway.yaml 失败: {e}")
|
||
return False
|
||
|
||
def _feishu_alert(title: str, content: str):
|
||
"""飞书告警卡片"""
|
||
try:
|
||
payload = json.dumps({
|
||
"msg_type": "interactive",
|
||
"card": {
|
||
"header": {"title": {"tag": "plain_text", "content": title}, "template": "red"},
|
||
"elements": [{"tag": "markdown", "content": content}],
|
||
},
|
||
}).encode()
|
||
req = urllib.request.Request(FEISHU_WEBHOOK, data=payload,
|
||
headers={"Content-Type": "application/json"}, method="POST")
|
||
with urllib.request.urlopen(req, timeout=10):
|
||
pass
|
||
except Exception as e:
|
||
print(f" 飞书通知失败: {e}")
|
||
|
||
# ============ 主流程 ============
|
||
|
||
def main():
|
||
global KEY_ENV
|
||
KEY_ENV = _get_key()
|
||
if not KEY_ENV:
|
||
print("🔴 无法读取 LLM_API_KEY，跳过本轮")
|
||
return
|
||
|
||
current = _get_current_model()
|
||
if not current:
|
||
print("🔴 无法读取当前 LLM_MODEL，跳过本轮")
|
||
return
|
||
|
||
ts = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
|
||
|
||
# 1. 测当前模型
|
||
if _test_json(current):
|
||
# 健康，静默退出（no-agent cron 空输出不发送）
|
||
return
|
||
|
||
# 2. 当前模型挂了 → 找替补
|
||
print(f"🔴 [{ts}] 蒸馏模型 {current} JSON 输出失败，开始切换...")
|
||
replacement = None
|
||
for cand in CANDIDATE_POOL:
|
||
if cand == current or cand in KNOWN_BAD:
|
||
continue
|
||
print(f" 🔄 测试替补 {cand}...")
|
||
if _test_json(cand):
|
||
replacement = cand
|
||
print(f" ✅ {cand} 可用")
|
||
break
|
||
|
||
if not replacement:
|
||
msg = f"**⚠️ 所有蒸馏模型都挂了**\n\n⏰ {ts}\n当前: `{current}`\n候选全部失败: {', '.join(CANDIDATE_POOL)}\n\n请人工检查 Agnes/NewAPI 渠道"
|
||
_feishu_alert("🔴 蒸馏模型全部不可用", msg)
|
||
print(msg)
|
||
return
|
||
|
||
# 3. 更新两处配置
|
||
z_ok = _update_zhiyid(replacement)
|
||
t_ok = _update_tddb(replacement)
|
||
|
||
changed_parts = []
|
||
if z_ok:
|
||
changed_parts.append("zhiyid.service")
|
||
if t_ok:
|
||
changed_parts.append("tdai-gateway.yaml")
|
||
|
||
msg = f"**🔄 蒸馏模型已自动切换**\n\n⏰ {ts}\n`{current}` → `{replacement}`\n更新: {', '.join(changed_parts) if changed_parts else '无（配置已是最新）'}\n\n原因: 原模型 JSON 输出失败（免费模型挂了）"
|
||
_feishu_alert("🔄 蒸馏模型自动切换", msg)
|
||
print(msg)
|
||
|
||
if __name__ == "__main__":
|
||
main()
|