199 lines
6.9 KiB
Bash
199 lines
6.9 KiB
Bash
#!/bin/bash
|
|
# 织忆三方交叉健康检查 (v11.23)
|
|
#
|
|
# 用途: 回答 "全面检查织忆" / "织忆没事吧" / "织忆挂了么" 这类问题时,
|
|
# 一次跑完三类信号 (进程 + 端口 + 端点)。
|
|
#
|
|
# 设计原因 (2026-06-29 真实踩坑):
|
|
# 单一 curl 不行 —— 进程可能正处于 systemd Restart 间隙,curl 看到一个
|
|
# "瞬时 connection refused", 但同一时刻 `ss -tlnp` 显示端口正在 listen,
|
|
# 真实状态是 "正常, 启动瞬态"。单信号不可信。
|
|
# 铁律: ps + ss + curl 三方必须同时拉 + 交叉判, 禁止单信号下结论。
|
|
#
|
|
# 用法:
|
|
# ~/.hermes/skills/zhiyi/zhiyi/scripts/three-way-check.sh # 默认查询
|
|
# QUIET=1 .../three-way-check.sh # 只打印判定行
|
|
#
|
|
# 返回:
|
|
# 0 = 全部 OK (正常)
|
|
# 1 = 出现任意 FAIL (需要修复)
|
|
#
|
|
# 依赖: bash, curl, ss, ps, awk, python3
|
|
# 不依赖: jq (避免没装就挂)
|
|
|
|
# ---------- 配置 ----------
|
|
# 默认走 SKILL.md 同款硬编码 key (与 scripts/daily-check.sh 一致);
|
|
# 留 override 入口方便 cron / watchdog 注入。
|
|
API_KEY="${ZHIYI_KEY_OVERRIDE:-zhiyi-dev-key-2026}"
|
|
QUIET="${QUIET:-0}"
|
|
|
|
# ---------- helpers ----------
|
|
ok() { [ "$QUIET" = "1" ] && return; echo "[$(date '+%Y-%m-%d %H:%M:%S')] OK $*"; }
|
|
warn(){ [ "$QUIET" = "1" ] && return; echo "[$(date '+%Y-%m-%d %H:%M:%S')] WARN $*"; }
|
|
err() { [ "$QUIET" = "1" ] && return; echo "[$(date '+%Y-%m-%d %H:%M:%S')] FAIL $*"; }
|
|
sep() { [ "$QUIET" = "1" ] && return; echo "------------------------------------------------------------"; }
|
|
|
|
# 检测某个进程是否真在 (不限 PID 个数; 只要存在一个就算在)
|
|
proc_exists() {
|
|
local pat="$1"
|
|
ps -eo pid,etime,cmd 2>/dev/null | awk -v pat="$pat" '$0 ~ pat {found=1} END{exit !found}'
|
|
}
|
|
|
|
# 检测某个端口是否真在 listen
|
|
port_listening() {
|
|
local port="$1"
|
|
ss -tlnH "sport = :$port" 2>/dev/null | awk 'NF{found=1} END{exit !found}'
|
|
}
|
|
|
|
# 仅打印 HTTP code (curl 返回 000 表示连接失败)
|
|
endpoint_code() {
|
|
local url="$1"
|
|
curl -s -m 3 -o /dev/null -w '%{http_code}' "$url" 2>/dev/null || echo "000"
|
|
}
|
|
|
|
# 同时回 code + body
|
|
endpoint_full() {
|
|
local url="$1"
|
|
local code body
|
|
code=$(endpoint_code "$url")
|
|
body=$(curl -s -m 3 "$url" 2>/dev/null)
|
|
echo "$code|$body"
|
|
}
|
|
|
|
# ---------- 1. 进程层 ----------
|
|
P_ZHIYID=0
|
|
P_CONSOLIDATE=0
|
|
P_BGE=0
|
|
proc_exists 'zhiyid-new' && P_ZHIYID=1
|
|
proc_exists 'zhiyi-consolidate' && P_CONSOLIDATE=1
|
|
proc_exists 'bge_embed_server\.py|python3.*8000' && P_BGE=1
|
|
|
|
# ---------- 2. 端口层 ----------
|
|
P_PORT_7821=0
|
|
P_PORT_8000=0
|
|
port_listening 7821 && P_PORT_7821=1
|
|
port_listening 8000 && P_PORT_8000=1
|
|
|
|
# ---------- 3. 端点层 ----------
|
|
ZHIYID_RAW=$(curl -s -m 3 -H "X-API-Key: ${API_KEY}" -o /dev/null -w '%{http_code}' "http://localhost:7821/api/v1/health" 2>/dev/null || echo "000")
|
|
ZHIYID_CODE="$ZHIYID_RAW"
|
|
ZHIYID_BODY=""
|
|
|
|
BGE_RAW=$(endpoint_full "http://localhost:8000/health")
|
|
BGE_CODE="${BGE_RAW%%|*}"
|
|
BGE_BODY="${BGE_RAW#*|}"
|
|
|
|
ZHIYID_OK=0
|
|
BGE_OK=0
|
|
[ "$ZHIYID_CODE" = "200" ] && ZHIYID_OK=1
|
|
[ "$BGE_CODE" = "200" ] && BGE_OK=1
|
|
|
|
# ---------- 抽样功能层 ----------
|
|
RECALL_OUT=$(curl -s -m 5 -X POST -H "X-API-Key: ${API_KEY}" \
|
|
-H "Content-Type: application/json" \
|
|
-d '{"query":"xiaowei","top_k":2}' \
|
|
"http://localhost:7821/api/v1/recall" 2>/dev/null)
|
|
RECALL_COUNT=$(echo "$RECALL_OUT" | python3 -c "
|
|
import sys, json
|
|
try:
|
|
d = json.load(sys.stdin)
|
|
print(d.get('count', 0))
|
|
except Exception:
|
|
print(0)" 2>/dev/null)
|
|
|
|
GRAPH_STATS=$(curl -s -m 3 -H "X-API-Key: ${API_KEY}" \
|
|
"http://localhost:7821/api/v1/graph/stats" 2>/dev/null)
|
|
GRAPH_NODES=$(echo "$GRAPH_STATS" | python3 -c "
|
|
import sys, json
|
|
try: print(json.load(sys.stdin).get('node_count', '?'))
|
|
except Exception: print('?')" 2>/dev/null)
|
|
GRAPH_EDGES=$(echo "$GRAPH_STATS" | python3 -c "
|
|
import sys, json
|
|
try: print(json.load(sys.stdin).get('edge_count', '?'))
|
|
except Exception: print('?')" 2>/dev/null)
|
|
|
|
# ---------- 打印 ----------
|
|
sep
|
|
echo "织忆三方交叉健康检查 $(date '+%Y-%m-%d %H:%M:%S')"
|
|
sep
|
|
echo "组件 进程 端口 端点 结论"
|
|
echo "------------------ ---- ---- ---- ----"
|
|
|
|
# 行: zhiyid (端点 + 端口 + 进程三方都有)
|
|
verdict=""
|
|
ps_="-"; pt_="-"; ep_="-"
|
|
if [ "$P_ZHIYID" = "1" ] && [ "$P_PORT_7821" = "1" ] && [ "$ZHIYID_OK" = "1" ]; then
|
|
verdict="OK "; ps_="OK"; pt_="OK"; ep_="OK"
|
|
elif [ "$P_ZHIYID" = "1" ] && [ "$P_PORT_7821" = "1" ] && [ "$ZHIYID_OK" != "1" ]; then
|
|
verdict="REBOOT"; ps_="OK"; pt_="OK"; ep_="FAIL"
|
|
elif [ "$P_ZHIYID" = "1" ] && [ "$P_PORT_7821" != "1" ]; then
|
|
verdict="START_FAIL"; ps_="OK"; pt_="FAIL"; ep_="FAIL"
|
|
elif [ "$P_ZHIYID" != "1" ]; then
|
|
verdict="DOWN"; ps_="FAIL"; pt_="FAIL"; ep_="FAIL"
|
|
fi
|
|
printf "%-18s %-4s %-4s %-4s %s\n" "zhiyid (7821)" "$ps_" "$pt_" "$ep_" "$verdict"
|
|
|
|
# 行: bge-embed
|
|
verdict=""
|
|
ps_="-"; pt_="-"; ep_="-"
|
|
if [ "$P_BGE" = "1" ] && [ "$P_PORT_8000" = "1" ] && [ "$BGE_OK" = "1" ]; then
|
|
verdict="OK "; ps_="OK"; pt_="OK"; ep_="OK"
|
|
elif [ "$P_BGE" = "1" ] && [ "$P_PORT_8000" = "1" ] && [ "$BGE_OK" != "1" ]; then
|
|
verdict="REBOOT"; ps_="OK"; pt_="OK"; ep_="FAIL"
|
|
elif [ "$P_BGE" = "1" ] && [ "$P_PORT_8000" != "1" ]; then
|
|
verdict="START_FAIL"; ps_="OK"; pt_="FAIL"; ep_="FAIL"
|
|
elif [ "$P_BGE" != "1" ]; then
|
|
verdict="DOWN"; ps_="FAIL"; pt_="FAIL"; ep_="FAIL"
|
|
fi
|
|
printf "%-18s %-4s %-4s %-4s %s\n" "bge-embed (8000)" "$ps_" "$pt_" "$ep_" "$verdict"
|
|
|
|
# 行: consolidate (走 IPC socket, 不走端口)
|
|
SOCK="/tmp/zhiyi-ipc.sock"
|
|
verdict=""
|
|
ps_="-"; pt_="-"; ep_="-"
|
|
if [ "$P_CONSOLIDATE" = "1" ] && [ -S "$SOCK" ]; then
|
|
verdict="OK "; ps_="OK"; pt_="OK"
|
|
elif [ "$P_CONSOLIDATE" = "1" ]; then
|
|
verdict="PROC+SOCK_MISSING"; ps_="OK"; pt_="FAIL"
|
|
else
|
|
verdict="DOWN"; ps_="FAIL"; pt_="FAIL"
|
|
fi
|
|
printf "%-18s %-4s %-4s %-4s %s\n" "consolidate (sock)" "$ps_" "$pt_" "$ep_" "$verdict"
|
|
|
|
# 功能层
|
|
sep
|
|
echo "功能抽样 (仅在三方都通过时有意义):"
|
|
ok " recall xiaowei -> ${RECALL_COUNT} 条"
|
|
ok " graph: 节点=${GRAPH_NODES} 边=${GRAPH_EDGES}"
|
|
|
|
# 异常 body 仅在非 QUIET 时打印
|
|
if [ "$QUIET" != "1" ]; then
|
|
[ "$ZHIYID_OK" != "1" ] && warn "zhiyid body: $ZHIYID_BODY"
|
|
[ "$BGE_OK" != "1" ] && warn "bge-embed body: $BGE_BODY"
|
|
fi
|
|
|
|
# ---------- 判定 + 退出码 ----------
|
|
EXIT_CODE=0
|
|
if [ "$P_ZHIYID$ZHIYID_OK$P_PORT_7821" != "111" ]; then
|
|
err "zhiyid 不健康 进程=$P_ZHIYID 端口=$P_PORT_7821 端点=$ZHIYID_OK"
|
|
EXIT_CODE=1
|
|
fi
|
|
if [ "$P_BGE$BGE_OK$P_PORT_8000" != "111" ]; then
|
|
err "bge-embed 不健康 进程=$P_BGE 端口=$P_PORT_8000 端点=$BGE_OK"
|
|
EXIT_CODE=1
|
|
fi
|
|
if [ "$P_CONSOLIDATE" != "1" ] || [ ! -S "$SOCK" ]; then
|
|
sock_status=$([ -S "$SOCK" ] && echo "在" || echo "缺席")
|
|
err "consolidate 不健康 进程=$P_CONSOLIDATE socket=$sock_status"
|
|
EXIT_CODE=1
|
|
fi
|
|
|
|
sep
|
|
if [ "$EXIT_CODE" = "0" ]; then
|
|
ok "织忆三方交叉验证 -> 全绿"
|
|
else
|
|
err "织忆三方交叉验证 -> 至少一项异常 (exit=$EXIT_CODE)"
|
|
fi
|
|
|
|
exit "$EXIT_CODE"
|