213 lines
5.5 KiB
Bash
Executable File
213 lines
5.5 KiB
Bash
Executable File
#!/bin/bash
|
||
# ============================================================
|
||
# model-health.sh - NewAPI 模型健康巡检脚本
|
||
# 每 6 小时由 cron 触发（no_agent 模式）
|
||
#
|
||
# 测试模型列表中的每个模型的：
|
||
# - 响应延迟（TTFT + 总时间）
|
||
# - HTTP 状态码
|
||
# - 响应内容是否完整
|
||
# - 3 次连续测试的稳定性
|
||
#
|
||
# 输出：~/.hermes/model-health.json
|
||
# ============================================================
|
||
|
||
API="http://127.0.0.1:3000/v1"
|
||
KEY="0ExNiLblJvIWBDpkS50fwOBw4MmqLyKdHJK5iQtlw9dOMWBP"
|
||
OUTPUT="$HOME/.hermes/model-health.json"
|
||
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||
|
||
# 测试列表 —— 按场景分组
|
||
declare -a MODELS=(
|
||
# 快速响应（日常）
|
||
"minimaxai/minimax-m3"
|
||
"minimaxai/minimax-m2.7"
|
||
"stepfun-ai/step-3.5-flash"
|
||
"deepseek-ai/deepseek-v3.2"
|
||
|
||
# 标准推理（主力）
|
||
"qwen/qwen3.5-122b-a10b"
|
||
"mistralai/mistral-large-3-675b-instruct-2512"
|
||
"mistralai/mistral-medium-3.5-128b"
|
||
"meta/llama-3.3-70b-instruct"
|
||
|
||
# 编程专用
|
||
"qwen/qwen3-coder-480b-a35b-instruct"
|
||
"qwen/qwen2.5-coder-32b-instruct"
|
||
|
||
# 中文优化
|
||
"z-ai/glm4.7"
|
||
"z-ai/glm5"
|
||
"moonshotai/kimi-k2-instruct"
|
||
"bytedance/seed-oss-36b-instruct"
|
||
|
||
# 轻量/特殊
|
||
"microsoft/phi-4-mini-instruct"
|
||
"meta/llama-4-maverick-17b-128e-instruct"
|
||
)
|
||
|
||
test_model() {
|
||
local model=$1
|
||
local trial=$2
|
||
local prompt="回复一句话：今天天气不错。"
|
||
|
||
local start=$(date +%s%N)
|
||
|
||
local http_body=$(mktemp)
|
||
local http_code
|
||
|
||
http_code=$(curl -s -w "%{http_code}" -o "$http_body" \
|
||
--max-time 30 \
|
||
-H "Authorization: Bearer $KEY" \
|
||
-H "Content-Type: application/json" \
|
||
-d "{\"model\":\"$model\",\"messages\":[{\"role\":\"user\",\"content\":\"$prompt\"}],\"max_tokens\":50}" \
|
||
"$API/chat/completions" 2>&1)
|
||
|
||
local end=$(date +%s%N)
|
||
local total_ms=$(( ($end - $start) / 1000000 ))
|
||
|
||
if [ "$http_code" = "200" ]; then
|
||
# 提取关键指标
|
||
local content=$(python3 -c "
|
||
import json, sys
|
||
try:
|
||
d = json.load(open('$http_body'))
|
||
choice = d['choices'][0]
|
||
msg = choice.get('message', {})
|
||
finish = choice.get('finish_reason', '')
|
||
content = msg.get('content', '')
|
||
reasoning = msg.get('reasoning_content', '')
|
||
usage = d.get('usage', {})
|
||
has_content = 1 if content.strip() else 0
|
||
has_reasoning = 1 if reasoning and reasoning.strip() else 0
|
||
pt = usage.get('prompt_tokens', 0)
|
||
ct = usage.get('completion_tokens', 0)
|
||
ttft = d.get('nvext', {}).get('timing', {}).get('ttft_ms', -1)
|
||
print(f'{has_content}|{has_reasoning}|{pt}|{ct}|{ttft}|{finish}')
|
||
except Exception as e:
|
||
print(f'PARSE_ERROR|0|0|0|-1|{str(e)}')
|
||
" 2>&1)
|
||
|
||
rm -f "$http_body"
|
||
|
||
IFS='|' read -r has_content has_reasoning pt ct ttft finish <<< "$content"
|
||
|
||
echo "OK|${total_ms}|${ttft}|${has_content}|${has_reasoning}|${pt}|${ct}|${finish}"
|
||
else
|
||
rm -f "$http_body"
|
||
echo "FAIL|${total_ms}|-1|0|0|0|0|HTTP_${http_code}"
|
||
fi
|
||
}
|
||
|
||
# 开始测试
|
||
echo "[" > "$OUTPUT.tmp"
|
||
first=true
|
||
|
||
for model in "${MODELS[@]}"; do
|
||
# 每模型测 3 次，取均值
|
||
ok_count=0
|
||
fail_count=0
|
||
total_latency=0
|
||
total_ttft=0
|
||
last_status=""
|
||
last_finish=""
|
||
|
||
for trial in 1 2 3; do
|
||
result=$(test_model "$model" "$trial")
|
||
|
||
IFS='|' read -r status latency ttft has_content has_reasoning pt ct finish <<< "$result"
|
||
|
||
if [ "$status" = "OK" ]; then
|
||
ok_count=$((ok_count + 1))
|
||
total_latency=$((total_latency + latency))
|
||
total_ttft=$((total_ttft + ttft))
|
||
last_status="ok"
|
||
last_finish="$finish"
|
||
else
|
||
fail_count=$((fail_count + 1))
|
||
last_status="fail"
|
||
last_finish="$finish"
|
||
fi
|
||
done
|
||
|
||
# 计算统计
|
||
stability=""
|
||
if [ "$ok_count" -eq 3 ]; then
|
||
stability="stable"
|
||
elif [ "$ok_count" -ge 1 ]; then
|
||
stability="unstable"
|
||
else
|
||
stability="dead"
|
||
fi
|
||
|
||
avg_latency=0
|
||
avg_ttft=0
|
||
if [ "$ok_count" -gt 0 ]; then
|
||
avg_latency=$((total_latency / ok_count))
|
||
avg_ttft=$((total_ttft / ok_count))
|
||
fi
|
||
|
||
# 输出 JSON 行
|
||
if [ "$first" = true ]; then
|
||
first=false
|
||
else
|
||
echo "," >> "$OUTPUT.tmp"
|
||
fi
|
||
|
||
cat >> "$OUTPUT.tmp" << JSONBLOCK
|
||
{
|
||
"model": "$model",
|
||
"timestamp": "$TIMESTAMP",
|
||
"tests": 3,
|
||
"success": $ok_count,
|
||
"failure": $fail_count,
|
||
"avg_latency_ms": $avg_latency,
|
||
"avg_ttft_ms": $avg_ttft,
|
||
"stability": "$stability",
|
||
"last_status": "$last_status",
|
||
"last_finish": "$last_finish"
|
||
}
|
||
JSONBLOCK
|
||
done
|
||
|
||
echo "]" >> "$OUTPUT.tmp"
|
||
|
||
# 添加汇总统计
|
||
python3 -c "
|
||
import json
|
||
|
||
with open('$OUTPUT.tmp') as f:
|
||
data = json.load(f)
|
||
|
||
total = len(data)
|
||
healthy = sum(1 for m in data if m['stability'] == 'stable')
|
||
flaky = sum(1 for m in data if m['stability'] == 'unstable')
|
||
dead = sum(1 for m in data if m['stability'] == 'dead')
|
||
|
||
# 按稳定性分组
|
||
stable_models = [m for m in data if m['stability'] == 'stable']
|
||
fastest = sorted(stable_models, key=lambda x: x['avg_latency_ms'])[:3] if stable_models else []
|
||
|
||
summary = {
|
||
'timestamp': '$TIMESTAMP',
|
||
'total_models': total,
|
||
'stable': healthy,
|
||
'unstable': flaky,
|
||
'dead': dead,
|
||
'fastest_stable': [m['model'] for m in fastest],
|
||
'recommendations': {
|
||
'fast': [m['model'] for m in fastest],
|
||
'default': [m['model'] for m in sorted(stable_models, key=lambda x: x.get('avg_latency_ms', 9999))[:5]],
|
||
},
|
||
'models': data
|
||
}
|
||
|
||
with open('$OUTPUT', 'w') as f:
|
||
json.dump(summary, f, indent=2, ensure_ascii=False)
|
||
|
||
print(f'巡检完成: {healthy}个稳定 / {flaky}个不稳定 / {dead}个死')
|
||
print(f'最快稳定: {\", \".join(summary[\"recommendations\"][\"fast\"])}')
|
||
"
|
||
|
||
rm -f "$OUTPUT.tmp"
|