104 lines
3.6 KiB
Python
104 lines
3.6 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
OmniRoute 周度评估报告 — 汇总观测数据，输出评估结论
|
||
每周日由 cron 触发，输出可读报告（非静默，始终输出）
|
||
评估维度：可用性 / 延迟 / 成本 / 路由一致性 / cron 运行状态
|
||
"""
|
||
import json
|
||
import os
|
||
import statistics
|
||
from datetime import datetime, timezone, timedelta
|
||
|
||
STATE_DIR = os.path.expanduser("~/.hermes/omniroute-observe")
|
||
LOG = os.path.join(STATE_DIR, "observations.jsonl")
|
||
|
||
# 评估阈值
|
||
LATENCY_WARN_MS = 15000 # 平均延迟超 15s 警告
|
||
SUCCESS_RATE_MIN = 0.90 # 成功率低于 90% 警告
|
||
|
||
|
||
def load_observations(days=7):
|
||
if not os.path.exists(LOG):
|
||
return []
|
||
cutoff = datetime.now(timezone.utc) - timedelta(days=days)
|
||
obs = []
|
||
with open(LOG) as f:
|
||
for line in f:
|
||
try:
|
||
d = json.loads(line)
|
||
ts = datetime.fromisoformat(d["ts"])
|
||
if ts >= cutoff:
|
||
obs.append(d)
|
||
except Exception:
|
||
continue
|
||
return obs
|
||
|
||
|
||
def main():
|
||
obs = load_observations(7)
|
||
n = len(obs)
|
||
|
||
# 汇总
|
||
total_tests = sum(o["total_tests"] for o in obs)
|
||
total_ok = sum(o["success_count"] for o in obs)
|
||
success_rate = total_ok / total_tests if total_tests else 1.0
|
||
|
||
latencies = []
|
||
routed_models = {}
|
||
total_cost = 0.0
|
||
service_ok = 0
|
||
for o in obs:
|
||
if o["service"] == "active":
|
||
service_ok += 1
|
||
for t in o["tests"]:
|
||
latencies.append(t["latency_ms"])
|
||
routed_models[t["routed_to"]] = routed_models.get(t["routed_to"], 0) + 1
|
||
try:
|
||
total_cost += float(t["cost"])
|
||
except Exception:
|
||
pass
|
||
|
||
avg_lat = statistics.mean(latencies) if latencies else 0
|
||
p95_lat = sorted(latencies)[int(len(latencies) * 0.95) - 1] if latencies else 0
|
||
|
||
# cron 状态
|
||
cron_ok = 0
|
||
cron_total = 0
|
||
for o in obs:
|
||
for name, st in (o.get("cron_status") or {}).items():
|
||
cron_total += 1
|
||
if st.get("has_output"):
|
||
cron_ok += 1
|
||
|
||
print(f"# 📊 OmniRoute 周度评估报告（{datetime.now().strftime('%Y-%m-%d')}）")
|
||
print(f"\n## 观测样本：{n} 次采集（7 天内）")
|
||
print(f"- 服务可用性: {service_ok}/{n} 次 active（{service_ok/n*100:.0f}%）" if n else "- 无数据")
|
||
print(f"- API 测试: {total_ok}/{total_tests} 成功（成功率 {success_rate*100:.1f}%）")
|
||
print(f"- 平均延迟: {avg_lat:.0f}ms | P95: {p95_lat:.0f}ms")
|
||
print(f"- 累计成本: ${total_cost:.6f}")
|
||
print(f"- 路由模型分布: {json.dumps(routed_models, ensure_ascii=False)}")
|
||
print(f"- 已切换 cron 运行: {cron_ok}/{cron_total} 次有输出")
|
||
|
||
# 结论
|
||
print("\n## 评估结论")
|
||
issues = []
|
||
if service_ok < n:
|
||
issues.append(f"⚠️ 服务可用性 {service_ok}/{n}，有宕机记录")
|
||
if success_rate < SUCCESS_RATE_MIN:
|
||
issues.append(f"⚠️ 成功率 {success_rate*100:.1f}% < 阈值 {SUCCESS_RATE_MIN*100:.0f}%")
|
||
if avg_lat > LATENCY_WARN_MS:
|
||
issues.append(f"⚠️ 平均延迟 {avg_lat:.0f}ms > 阈值 {LATENCY_WARN_MS}ms")
|
||
|
||
if not issues:
|
||
print("✅ **OmniRoute 运行稳定，可以继续扩大切换范围**")
|
||
print("- 建议：可再将 skill-curator-weekly / 牵挂提醒 / 股票矛盾分析周报切到 omniroute-local")
|
||
else:
|
||
print("❌ **存在问题，建议暂停扩大切换**")
|
||
for i in issues:
|
||
print(f"- {i}")
|
||
print("- 建议：回滚异常任务到 newapi-local，检查 omniroute.service 日志")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|