diff --git a/scripts/model-health.py b/scripts/model-health.py index 276f4416..6df91f4a 100755 --- a/scripts/model-health.py +++ b/scripts/model-health.py @@ -37,6 +37,28 @@ CANDIDATE_POOL = [ "nvidia/nemotron-mini-4b-instruct", # 4B 兜底 ] +# 已知上下文长度(K=tokens) +CONTEXT_LENGTHS = { + "openai/gpt-oss-120b": 128, + "nvidia/nemotron-3-super-120b-a12b": 128, + "mistralai/mistral-nemotron": 128, + "nvidia/nvidia-nemotron-nano-9b-v2": 128, + "meta/llama-3.1-8b-instruct": 128, + "nvidia/nemotron-mini-4b-instruct": 128, + "nvidia/llama-3.3-nemotron-super-49b-v1": 128, + "nvidia/llama-3.3-nemotron-super-49b-v1.5": 128, + "mistralai/mistral-medium-3.5-128b": 128, + "minimaxai/minimax-m2.7": 256, + "minimaxai/minimax-m3": 256, + "stepfun-ai/step-3.5-flash": 8, + "qwen/qwen3.5-122b-a10b": 128, + "qwen/qwen3-next-80b-a3b-thinking": 128, + "moonshotai/kimi-k2-instruct": 128, + "mistralai/devstral-2-123b-instruct-2512": 128, + "deepseek-v4-flash": 256, + "deepseek-ai/deepseek-v4-pro": 256, +} + # 已知忽略的模型(系统/不支持/垃圾,永远不测也不自动加入) KNOWN_IGNORE = { "gpt-4o", "gpt-4o-mini", "gpt-4o-audio-preview", "gpt-4o-mini-audio-preview", @@ -165,6 +187,20 @@ def _speed_score(latency_ms: int, fastest_latency: int) -> float: return min(round(ratio * 100), 100) +def _context_score(model: str) -> float: + """上下文长度分:越长越高 256K→100, 128K→80, 64K→60, 32K→40, 8K→10""" + ctx = CONTEXT_LENGTHS.get(model, 128) # 未知默认128 + if ctx >= 256: + return 100 + if ctx >= 128: + return 80 + if ctx >= 64: + return 60 + if ctx >= 32: + return 40 + return max(round(ctx / 8 * 10), 5) + + def _run_quality_probe(model: str) -> dict: """运行质量探针,返回探针分和详细结果""" probe_results = [] @@ -332,9 +368,10 @@ def test_model(model: str, fastest_latency: int = None) -> dict: ss = _speed_score(avg_latency, fastest_latency) if fastest_latency and avg_latency > 0 else 50 stab_s = 100 if stability == "stable" else (50 if stability == "unstable" else 0) probe_s = probe["probe_score"] + cs = _context_score(model) rank_score = round( - probe_s * 0.35 + ps * 0.25 + fs * 0.20 + stab_s * 0.10 + ss * 0.10 + probe_s * 0.30 + cs * 0.15 + ps * 0.20 + fs * 0.15 + stab_s * 0.10 + ss * 0.10 ) return { @@ -351,6 +388,8 @@ def test_model(model: str, fastest_latency: int = None) -> dict: "probe_detail": probe["probe_detail"], "rank_score": rank_score, "param_b": param_b, + "context_k": CONTEXT_LENGTHS.get(model, 128), + "context_score": cs, "family_score": fs, "param_score": ps, }