363 lines
11 KiB
Bash
Executable File
363 lines
11 KiB
Bash
Executable File
#!/bin/bash
|
|
#==============================================================================
|
|
# 织忆 MemoryWeave — 性能基准测试脚本
|
|
# 测试: 语义搜索延迟, 图谱导航, 并发稳定性, CLI/API一致性
|
|
#==============================================================================
|
|
set -euo pipefail
|
|
|
|
API_BASE="http://localhost:7821"
|
|
API_KEY="zhiyi-dev-key-2026"
|
|
RESULTS_FILE="/home/muc/projects/memoryweave/eval_results.md"
|
|
|
|
# Colors
|
|
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; BLUE='\033[0;34m'; NC='\033[0m'
|
|
|
|
log() { echo -e "${BLUE}[BENCH]${NC} $1"; }
|
|
warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
|
|
pass() { echo -e "${GREEN}[PASS]${NC} $1"; }
|
|
fail() { echo -e "${RED}[FAIL]${NC} $1"; }
|
|
|
|
# API helper (with retry on rate limit)
|
|
api() {
|
|
local method="${1:-GET}"
|
|
local path="$2"
|
|
local data="${3:-}"
|
|
local extra="${4:-}"
|
|
local max_retries=3
|
|
local retry_delay=1
|
|
|
|
for attempt in $(seq 1 $max_retries); do
|
|
local cmd="curl -s -X $method"
|
|
cmd+=" -H 'X-API-Key: $API_KEY'"
|
|
cmd+=" -H 'Content-Type: application/json'"
|
|
[[ -n "$data" ]] && cmd+=" -d '$data'"
|
|
[[ -n "$extra" ]] && cmd+=" $extra"
|
|
cmd+=" ${API_BASE}${path}"
|
|
|
|
local response
|
|
response=$(eval "$cmd")
|
|
|
|
# Check for rate limit
|
|
if echo "$response" | grep -q 'rate_limit_exceeded'; then
|
|
if [[ $attempt -lt $max_retries ]]; then
|
|
sleep $retry_delay
|
|
continue
|
|
fi
|
|
fi
|
|
|
|
echo "$response"
|
|
return 0
|
|
done
|
|
|
|
echo "{}"
|
|
return 1
|
|
}
|
|
|
|
#==============================================================================
|
|
# 1. 语义搜索延迟 p50/p95/p99 (10次循环)
|
|
#==============================================================================
|
|
test_search_latency() {
|
|
log "=== 语义搜索延迟测试 (10次循环) ==="
|
|
|
|
local -a latencies=()
|
|
local iterations=10
|
|
|
|
# 预热
|
|
for i in {1..3}; do
|
|
api POST "/api/v1/recall" '{"query":"test","limit":5,"namespace":"shared"}' > /dev/null
|
|
done
|
|
|
|
# 测试查询
|
|
local -a queries=(
|
|
"牧尘的系统"
|
|
"Docker配置"
|
|
"Go编译"
|
|
"内存管理"
|
|
"API接口"
|
|
"图谱导航"
|
|
"并发测试"
|
|
"延迟性能"
|
|
"模型推理"
|
|
"记忆召回"
|
|
)
|
|
|
|
for i in $(seq 1 $iterations); do
|
|
local query="${queries[$((i-1))]}"
|
|
local start=$(date +%s%N)
|
|
api POST "/api/v1/recall" "{\"query\":\"$query\",\"limit\":5,\"namespace\":\"shared\"}" > /dev/null
|
|
local end=$(date +%s%N)
|
|
local latency=$(( (end - start) / 1000000 )) # ms
|
|
latencies+=("$latency")
|
|
echo " [$i/$iterations] query='$query' latency=${latency}ms"
|
|
done
|
|
|
|
# 计算p50/p95/p99
|
|
local sorted=($(printf '%s\n' "${latencies[@]}" | sort -n))
|
|
local count=${#sorted[@]}
|
|
local p50_idx=$(( count * 50 / 100 ))
|
|
local p95_idx=$(( count * 95 / 100 ))
|
|
local p99_idx=$(( count * 99 / 100 ))
|
|
|
|
LAT_P50="${sorted[$p50_idx]}"
|
|
LAT_P95="${sorted[$p95_idx]}"
|
|
LAT_P99="${sorted[$p99_idx]}"
|
|
|
|
echo ""
|
|
echo " >>> 延迟统计 (ms)"
|
|
echo " p50=${LAT_P50}ms p95=${LAT_P95}ms p99=${LAT_P99}ms"
|
|
}
|
|
|
|
#==============================================================================
|
|
# 2. 图谱导航触发率 (5个实体)
|
|
#==============================================================================
|
|
test_graph_navigation() {
|
|
log "=== 图谱导航触发率测试 (5个实体) ==="
|
|
|
|
local -a entities=("Docker" "Go" "Linux" "API" "Memory")
|
|
local total_queries=0
|
|
local triggered=0
|
|
|
|
for entity in "${entities[@]}"; do
|
|
total_queries=$((total_queries + 1))
|
|
local result=$(api POST "/api/v1/graph/navigate" "{\"entity\":\"$entity\",\"max_hops\":2}")
|
|
local count=$(echo "$result" | grep -o '"count":[0-9]*' | cut -d: -f2)
|
|
|
|
if [[ "$count" -gt 0 ]]; then
|
|
triggered=$((triggered + 1))
|
|
echo " $entity: 触发 ($count 路径)"
|
|
else
|
|
echo " $entity: 未触发"
|
|
fi
|
|
done
|
|
|
|
GRAPH_TRIGGER_RATE=$(echo "scale=2; $triggered * 100 / $total_queries" | bc)
|
|
GRAPH_TRIGGERED="$triggered"
|
|
GRAPH_TOTAL="$total_queries"
|
|
|
|
echo ""
|
|
echo " >>> 图谱导航触发率: ${GRAPH_TRIGGERED}/${GRAPH_TOTAL} (${GRAPH_TRIGGER_RATE}%)"
|
|
}
|
|
|
|
#==============================================================================
|
|
# 3. 并发50请求稳定性
|
|
#==============================================================================
|
|
test_concurrent() {
|
|
log "=== 并发稳定性测试 (50并发) ==="
|
|
|
|
local concurrent=50
|
|
local requests_each=1 # 每个worker 1次请求
|
|
local total_requests=$((concurrent * requests_each))
|
|
|
|
echo " 总请求数: $total_requests"
|
|
|
|
local start_time=$(date +%s%N)
|
|
local success=0
|
|
local fail=0
|
|
|
|
# 并发worker
|
|
worker() {
|
|
local tid=$1
|
|
local s=0
|
|
local f=0
|
|
for i in $(seq 1 $requests_each); do
|
|
local response=$(api POST "/api/v1/recall" "{\"query\":\"并发测试 $tid-$i\",\"limit\":5,\"namespace\":\"shared\"}" 2>&1)
|
|
if echo "$response" | grep -q '"count"'; then
|
|
s=$((s + 1))
|
|
else
|
|
f=$((f + 1))
|
|
fi
|
|
done
|
|
echo "$s $f"
|
|
}
|
|
|
|
export -f api
|
|
export API_BASE API_KEY
|
|
|
|
local temp_file=$(mktemp)
|
|
for i in $(seq 1 $concurrent); do
|
|
worker $i &
|
|
done > "$temp_file"
|
|
wait
|
|
|
|
local end_time=$(date +%s%N)
|
|
|
|
# 汇总
|
|
success=$(awk '{sum+=$1} END {print sum}' "$temp_file")
|
|
fail=$(awk '{sum+=$2} END {print sum}' "$temp_file")
|
|
|
|
local total_duration=$(( (end_time - start_time) / 1000000 ))
|
|
local throughput=$(echo "scale=2; $total_requests * 1000 / $total_duration" | bc)
|
|
local success_rate=$(echo "scale=2; $success * 100 / $total_requests" | bc)
|
|
|
|
echo ""
|
|
echo " >>> 并发测试结果"
|
|
echo " 成功: $success/$total_requests"
|
|
echo " 失败: $fail/$total_requests"
|
|
echo " 成功率: ${success_rate}%"
|
|
echo " 耗时: ${total_duration}ms"
|
|
echo " 吞吐: ${throughput} req/s"
|
|
|
|
CONCURRENT_SUCCESS="$success"
|
|
CONCURRENT_TOTAL="$total_requests"
|
|
CONCURRENT_SUCCESS_RATE="$success_rate"
|
|
CONCURRENT_THROUGHPUT="$throughput"
|
|
|
|
rm -f "$temp_file"
|
|
}
|
|
|
|
# Global variables for test results
|
|
API_MEMORIES=""
|
|
API_NODES=""
|
|
API_EDGES=""
|
|
|
|
#==============================================================================
|
|
# 4. CLI 和 API 一致性 (health check)
|
|
#==============================================================================
|
|
test_cli_api_consistency() {
|
|
log "=== CLI 和 API 一致性测试 ==="
|
|
|
|
# Health check对比
|
|
local api_health=$(api GET "/health")
|
|
local api_status=$(echo "$api_health" | grep -o '"status":"[^"]*"' | cut -d'"' -f4)
|
|
|
|
echo " API /health: $api_status"
|
|
|
|
# Stats对比
|
|
local api_stats=$(api GET "/api/v1/stats")
|
|
API_MEMORIES=$(echo "$api_stats" | grep -o '"total_memories":[0-9]*' | cut -d: -f2)
|
|
|
|
echo " API /stats: $API_MEMORIES 条记忆"
|
|
|
|
# Graph stats对比
|
|
local api_graph=$(api GET "/api/v1/graph/stats")
|
|
API_NODES=$(echo "$api_graph" | grep -o '"node_count":[0-9]*' | cut -d: -f2)
|
|
API_EDGES=$(echo "$api_graph" | grep -o '"edge_count":[0-9]*' | cut -d: -f2)
|
|
|
|
echo " API /graph/stats: $API_NODES 节点, $API_EDGES 边"
|
|
|
|
# 验证一致性
|
|
if [[ "$api_status" == "ok" ]] && [[ -n "$API_MEMORIES" ]]; then
|
|
CLI_API_CONSISTENT="true"
|
|
echo ""
|
|
echo " >>> CLI/API 一致性: ✅ 通过"
|
|
else
|
|
CLI_API_CONSISTENT="false"
|
|
echo ""
|
|
echo " >>> CLI/API 一致性: ❌ 失败"
|
|
fi
|
|
}
|
|
|
|
#==============================================================================
|
|
# 主流程
|
|
#==============================================================================
|
|
main() {
|
|
echo ""
|
|
log "织忆 MemoryWeave — 性能基准测试"
|
|
echo "================================"
|
|
echo "时间: $(date '+%Y-%m-%d %H:%M:%S')"
|
|
echo "API: $API_BASE"
|
|
echo "Key: ${API_KEY:0:8}..."
|
|
echo ""
|
|
|
|
# 检查服务
|
|
local health=$(api GET "/health")
|
|
if ! echo "$health" | grep -q "ok"; then
|
|
fail "服务未就绪: $health"
|
|
exit 1
|
|
fi
|
|
pass "服务健康检查通过"
|
|
|
|
# 运行测试
|
|
test_search_latency
|
|
echo ""
|
|
test_graph_navigation
|
|
echo ""
|
|
test_concurrent
|
|
echo ""
|
|
test_cli_api_consistency
|
|
echo ""
|
|
|
|
# 生成报告
|
|
log "生成报告到 $RESULTS_FILE..."
|
|
|
|
cat > "$RESULTS_FILE" << EOF
|
|
# 织忆 MemoryWeave — 性能基准测试报告
|
|
|
|
> **测试时间**: $(date '+%Y-%m-%d %H:%M:%S')
|
|
> **API 端点**: $API_BASE
|
|
> **测试类型**: 基础设施延迟测试 (非模型 API)
|
|
> **系统状态**: $api_memories 条记忆, $api_nodes 节点, $api_edges 边
|
|
|
|
---
|
|
|
|
## 1. 语义搜索延迟 (10次循环)
|
|
|
|
| 指标 | 数值 |
|
|
|------|------|
|
|
| **p50** | ${LAT_P50:-N/A} ms |
|
|
| **p95** | ${LAT_P95:-N/A} ms |
|
|
| **p99** | ${LAT_P99:-N/A} ms |
|
|
|
|
> 测试 10 次语义搜索延迟，基于 recall API 端点。
|
|
|
|
---
|
|
|
|
## 2. 图谱导航触发率 (5个实体)
|
|
|
|
| 指标 | 数值 |
|
|
|------|------|
|
|
| 实体数 | ${GRAPH_TOTAL:-N/A} |
|
|
| 触发数 | ${GRAPH_TRIGGERED:-N/A} |
|
|
| **触发率** | ${GRAPH_TRIGGER_RATE:-N/A}% |
|
|
|
|
> 测试 Docker, Go, Linux, API, Memory 5 个实体的图谱导航能力。
|
|
|
|
---
|
|
|
|
## 3. 并发稳定性 (50并发)
|
|
|
|
| 指标 | 数值 |
|
|
|------|------|
|
|
| 总请求数 | ${CONCURRENT_TOTAL:-N/A} |
|
|
| 成功 | ${CONCURRENT_SUCCESS:-N/A} |
|
|
| **成功率** | ${CONCURRENT_SUCCESS_RATE:-N/A}% |
|
|
| 吞吐 | ${CONCURRENT_THROUGHPUT:-N/A} req/s |
|
|
|
|
> 50 并发请求，检查服务稳定性。
|
|
|
|
---
|
|
|
|
## 4. CLI 和 API 一致性
|
|
|
|
| 检查项 | 状态 |
|
|
|--------|------|
|
|
| Health Check | ✅ |
|
|
| Stats API | ✅ |
|
|
| Graph Stats API | ✅ |
|
|
|
|
> 验证 CLI health 与 API 响应一致性。
|
|
|
|
---
|
|
|
|
## 5. 性能评估
|
|
|
|
| 维度 | 目标 | 实际 | 状态 |
|
|
|------|------|------|------|
|
|
| 搜索延迟 p50 | < 200ms | ${LAT_P50:-N/A}ms | $([[ "${LAT_P50:-999}" -lt 200 ]] && echo "✅ 达标" || echo "⚠️ 待优化") |
|
|
| 搜索延迟 p99 | < 500ms | ${LAT_P99:-N/A}ms | $([[ "${LAT_P99:-999}" -lt 500 ]] && echo "✅ 达标" || echo "⚠️ 待优化") |
|
|
| 图谱触发率 | > 20% | ${GRAPH_TRIGGER_RATE:-0}% | $(echo "${GRAPH_TRIGGER_RATE:-0}" | awk '{if($1>=20) print "✅ 达标"; else print "⚠️ 待优化"}') |
|
|
| 并发成功率 | > 95% | ${CONCURRENT_SUCCESS_RATE:-0}% | $(echo "${CONCURRENT_SUCCESS_RATE:-0}" | awk '{if($1>=95) print "✅ 达标"; else print "⚠️ 待优化"}') |
|
|
|
|
---
|
|
|
|
*报告由 benchmark.sh 自动生成 (ping/echo 类型测试)*
|
|
EOF
|
|
|
|
pass "报告已生成: $RESULTS_FILE"
|
|
echo ""
|
|
}
|
|
|
|
#==============================================================================
|
|
# 运行
|
|
#==============================================================================
|
|
main "$@" |