Skeleton for benchmarking ≤4B-parameter Ollama models on MemPalace
classification and extraction tasks. Outputs per-(model, task, mode)
metrics: accuracy, latency (TTFT, TPS, e2e p50/p95), VRAM (resident
and peak).
Layout under benchmarks/model_eval/:
- candidates.yaml: 16 models across 3 tiers, with family/size/variant
metadata pulled from ollama.com/library/<family>/tags
- metrics.py: timing extraction from Ollama response, VRAMPoller for
peak-memory tracking via nvidia-smi, embedding-similarity scoring,
host-info introspection, percentile aggregation
- runner.py: runs one (model, task, mode) triple, dispatches per-task
prompt building and scoring
- orchestrator.py: iterates candidates × tasks, writes CSV
- tasks/{calibration,room_classification,entity_extraction,
memory_extraction}/{prompts.py,score.py}: per-task prompt builders
and scorers
Uses mempalace.llm_client.get_provider directly so the benchmark runs
the same code path as production. strip_thinking_tokens lives locally
in metrics.py for now; will switch to mempalace.local_model once that
module lands on develop.
Datasets land in a follow-up commit.
329 lines
12 KiB
Python
329 lines
12 KiB
Python
"""Run one (model, task, mode) triple. Output a single result row.
|
|
|
|
Usage:
|
|
python -m benchmarks.model_eval.runner \\
|
|
--model qwen3:4b-instruct-2507-q4_K_M \\
|
|
--task room_classification \\
|
|
--mode closed \\
|
|
--dataset-dir tests/fixtures/mempalace_bench
|
|
|
|
Designed to be called by orchestrator.py for matrix runs, but standalone
|
|
runnable for one-off debugging.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import dataclasses
|
|
import json
|
|
import sys
|
|
import time
|
|
from dataclasses import asdict, dataclass, field
|
|
from pathlib import Path
|
|
from typing import Any, Callable, Optional
|
|
|
|
from mempalace.llm_client import LLMError, LLMResponse, get_provider
|
|
|
|
from .metrics import (
|
|
HostInfo,
|
|
TimingAggregate,
|
|
TimingSample,
|
|
VRAMPoller,
|
|
aggregate_timings,
|
|
extract_timing,
|
|
gather_host_info,
|
|
strip_thinking_tokens,
|
|
vram_resident_mb,
|
|
)
|
|
from .tasks.calibration import prompts as cal_prompts, score as cal_score
|
|
from .tasks.entity_extraction import prompts as ent_prompts, score as ent_score
|
|
from .tasks.memory_extraction import prompts as mem_prompts, score as mem_score
|
|
from .tasks.room_classification import prompts as rc_prompts, score as rc_score
|
|
|
|
|
|
@dataclass
|
|
class Result:
|
|
model_tag: str
|
|
task: str
|
|
mode: str
|
|
accuracy: float
|
|
extras: dict
|
|
timing: TimingAggregate
|
|
vram_resident_mb: Optional[int]
|
|
vram_peak_mb: Optional[int]
|
|
host: HostInfo
|
|
run_date: str
|
|
n_samples: int
|
|
error: Optional[str] = None
|
|
|
|
|
|
def load_jsonl(path: Path) -> list[dict]:
|
|
with open(path, "r", encoding="utf-8") as f:
|
|
return [json.loads(line) for line in f if line.strip()]
|
|
|
|
|
|
def _classify_with_timing(
|
|
provider, system: str, user: str, json_mode: bool
|
|
) -> tuple[Optional[LLMResponse], TimingSample, Optional[str]]:
|
|
t0 = time.perf_counter()
|
|
try:
|
|
response = provider.classify(system=system, user=user, json_mode=json_mode)
|
|
except LLMError as e:
|
|
return None, TimingSample(0, 0, 0, 0, 0), str(e)
|
|
elapsed = time.perf_counter() - t0
|
|
timing = extract_timing(response.raw, elapsed)
|
|
return response, timing, None
|
|
|
|
|
|
def _build_prompt(task: str, mode: str, sample: dict, label: dict) -> tuple[str, str, bool]:
|
|
"""Return (system, user, json_mode)."""
|
|
if task == "calibration":
|
|
return cal_prompts.SYSTEM, cal_prompts.build_user_prompt(sample["text"], sample["classes"]), False
|
|
if task == "room_classification":
|
|
if mode == "closed":
|
|
return (
|
|
rc_prompts.CLOSED_SYSTEM,
|
|
rc_prompts.build_closed_user(sample["agent"], sample["session_summary"], sample["__rooms__"]),
|
|
False,
|
|
)
|
|
return (
|
|
rc_prompts.OPEN_SYSTEM,
|
|
rc_prompts.build_open_user(sample["agent"], sample["session_summary"]),
|
|
False,
|
|
)
|
|
if task == "entity_extraction":
|
|
return ent_prompts.SYSTEM, ent_prompts.build_user(sample["text"]), True
|
|
if task == "memory_extraction":
|
|
return mem_prompts.SYSTEM, mem_prompts.build_user(sample["text"]), True
|
|
raise ValueError(f"Unknown task: {task}")
|
|
|
|
|
|
def _score_one(task: str, mode: str, predicted: str, sample: dict, label: dict, endpoint: str) -> dict:
|
|
if task == "calibration":
|
|
return cal_score.score(predicted, label["label"], sample["classes"])
|
|
if task == "room_classification":
|
|
if mode == "closed":
|
|
return rc_score.score_closed(predicted, label["closed_set_label"], sample["__rooms__"])
|
|
return rc_score.score_open(predicted, label["preferred_open_label"], endpoint=endpoint)
|
|
if task == "entity_extraction":
|
|
return ent_score.score(predicted, label["entities"])
|
|
if task == "memory_extraction":
|
|
return mem_score.score(predicted, label["memories"], endpoint=endpoint)
|
|
raise ValueError(f"Unknown task: {task}")
|
|
|
|
|
|
def _aggregate_accuracy(task: str, mode: str, scores: list[dict]) -> tuple[float, dict]:
|
|
"""Return (primary_accuracy, extras_dict)."""
|
|
if not scores:
|
|
return 0.0, {}
|
|
|
|
if task == "calibration" or (task == "room_classification" and mode == "closed"):
|
|
correct = sum(1 for s in scores if s.get("correct"))
|
|
return correct / len(scores), {
|
|
"correct": correct,
|
|
"total": len(scores),
|
|
}
|
|
|
|
if task == "room_classification" and mode == "open":
|
|
sims = [s.get("similarity", 0.0) for s in scores]
|
|
exacts = sum(1 for s in scores if s.get("exact_match"))
|
|
mean_sim = sum(sims) / len(sims)
|
|
return mean_sim, {
|
|
"mean_similarity": mean_sim,
|
|
"exact_match_count": exacts,
|
|
"high_similarity_count": sum(1 for s in sims if s >= 0.8),
|
|
"low_similarity_count": sum(1 for s in sims if s < 0.5),
|
|
}
|
|
|
|
if task == "entity_extraction":
|
|
f1 = sum(s.get("f1", 0.0) for s in scores) / len(scores)
|
|
precision = sum(s.get("precision", 0.0) for s in scores) / len(scores)
|
|
recall = sum(s.get("recall", 0.0) for s in scores) / len(scores)
|
|
valid_json_rate = sum(1 for s in scores if s.get("valid_json")) / len(scores)
|
|
return f1, {
|
|
"mean_f1": f1,
|
|
"mean_precision": precision,
|
|
"mean_recall": recall,
|
|
"valid_json_rate": valid_json_rate,
|
|
}
|
|
|
|
if task == "memory_extraction":
|
|
coverage = sum(s.get("coverage", 0.0) for s in scores) / len(scores)
|
|
hallucination = sum(s.get("hallucination_rate", 0.0) for s in scores) / len(scores)
|
|
type_accuracy = sum(s.get("type_accuracy", 0.0) for s in scores) / len(scores)
|
|
valid_json_rate = sum(1 for s in scores if s.get("valid_json")) / len(scores)
|
|
return coverage, {
|
|
"mean_coverage": coverage,
|
|
"mean_hallucination_rate": hallucination,
|
|
"mean_type_accuracy": type_accuracy,
|
|
"valid_json_rate": valid_json_rate,
|
|
}
|
|
|
|
return 0.0, {}
|
|
|
|
|
|
def run(
|
|
model_tag: str,
|
|
task: str,
|
|
mode: str,
|
|
dataset_dir: Path,
|
|
endpoint: str = "http://localhost:11434",
|
|
warmup: int = 1,
|
|
n_samples: Optional[int] = None,
|
|
strip_thinking: bool = True,
|
|
) -> Result:
|
|
"""Run one (model, task, mode) triple. Returns a Result."""
|
|
host = gather_host_info()
|
|
run_date = time.strftime("%Y-%m-%d %H:%M:%S UTC", time.gmtime())
|
|
|
|
task_dir = dataset_dir / task
|
|
samples = load_jsonl(task_dir / "dataset.jsonl")
|
|
labels = load_jsonl(task_dir / "labels.jsonl")
|
|
if len(samples) != len(labels):
|
|
return Result(
|
|
model_tag=model_tag, task=task, mode=mode,
|
|
accuracy=0.0, extras={}, timing=aggregate_timings([]),
|
|
vram_resident_mb=None, vram_peak_mb=None, host=host,
|
|
run_date=run_date, n_samples=0,
|
|
error=f"Sample/label count mismatch: {len(samples)} vs {len(labels)}",
|
|
)
|
|
|
|
if task == "room_classification":
|
|
room_lists = load_jsonl(task_dir / "room_lists.jsonl")
|
|
if len(room_lists) != len(samples):
|
|
return Result(
|
|
model_tag=model_tag, task=task, mode=mode,
|
|
accuracy=0.0, extras={}, timing=aggregate_timings([]),
|
|
vram_resident_mb=None, vram_peak_mb=None, host=host,
|
|
run_date=run_date, n_samples=0,
|
|
error=f"Room-list count mismatch: {len(room_lists)} vs {len(samples)}",
|
|
)
|
|
for s, rl in zip(samples, room_lists):
|
|
s["__rooms__"] = rl["rooms"]
|
|
|
|
if n_samples is not None:
|
|
samples = samples[:n_samples]
|
|
labels = labels[:n_samples]
|
|
|
|
try:
|
|
provider = get_provider("ollama", model=model_tag, endpoint=endpoint, timeout=180)
|
|
except LLMError as e:
|
|
return Result(
|
|
model_tag=model_tag, task=task, mode=mode,
|
|
accuracy=0.0, extras={}, timing=aggregate_timings([]),
|
|
vram_resident_mb=None, vram_peak_mb=None, host=host,
|
|
run_date=run_date, n_samples=0,
|
|
error=f"Provider init failed: {e}",
|
|
)
|
|
|
|
if warmup > 0 and samples:
|
|
s0, l0 = samples[0], labels[0]
|
|
try:
|
|
sys_p, user_p, json_mode = _build_prompt(task, mode, s0, l0)
|
|
for _ in range(warmup):
|
|
provider.classify(system=sys_p, user=user_p, json_mode=json_mode)
|
|
except LLMError as e:
|
|
return Result(
|
|
model_tag=model_tag, task=task, mode=mode,
|
|
accuracy=0.0, extras={}, timing=aggregate_timings([]),
|
|
vram_resident_mb=None, vram_peak_mb=None, host=host,
|
|
run_date=run_date, n_samples=0,
|
|
error=f"Warmup failed: {e}",
|
|
)
|
|
|
|
poller = VRAMPoller()
|
|
poller.start()
|
|
|
|
timings: list[TimingSample] = []
|
|
scores: list[dict] = []
|
|
errors: list[str] = []
|
|
|
|
for sample, label in zip(samples, labels):
|
|
sys_p, user_p, json_mode = _build_prompt(task, mode, sample, label)
|
|
response, timing, err = _classify_with_timing(provider, sys_p, user_p, json_mode)
|
|
if err is not None:
|
|
errors.append(err)
|
|
continue
|
|
text = response.text
|
|
if strip_thinking:
|
|
text = strip_thinking_tokens(text, response.raw)
|
|
timings.append(timing)
|
|
try:
|
|
score_result = _score_one(task, mode, text, sample, label, endpoint)
|
|
scores.append(score_result)
|
|
except Exception as e:
|
|
errors.append(f"score error: {e}")
|
|
|
|
peak_vram = poller.stop()
|
|
resident_vram = vram_resident_mb(model_tag)
|
|
|
|
accuracy, extras = _aggregate_accuracy(task, mode, scores)
|
|
if errors:
|
|
extras["error_count"] = len(errors)
|
|
extras["error_sample"] = errors[0]
|
|
|
|
return Result(
|
|
model_tag=model_tag,
|
|
task=task,
|
|
mode=mode,
|
|
accuracy=accuracy,
|
|
extras=extras,
|
|
timing=aggregate_timings(timings),
|
|
vram_resident_mb=resident_vram,
|
|
vram_peak_mb=peak_vram,
|
|
host=host,
|
|
run_date=run_date,
|
|
n_samples=len(scores),
|
|
)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Run one (model, task, mode) benchmark triple")
|
|
parser.add_argument("--model", required=True, help="Ollama model tag")
|
|
parser.add_argument("--task", required=True, choices=["room_classification", "entity_extraction", "memory_extraction", "calibration"])
|
|
parser.add_argument("--mode", default="closed", choices=["closed", "open", "default"], help="Mode: closed/open for room_classification, 'default' otherwise")
|
|
parser.add_argument("--dataset-dir", required=True, type=Path, help="Path to the bench dataset root")
|
|
parser.add_argument("--endpoint", default="http://localhost:11434")
|
|
parser.add_argument("--warmup", type=int, default=1)
|
|
parser.add_argument("--n", type=int, default=None, help="Limit to first N samples (for debugging)")
|
|
parser.add_argument("--no-strip-thinking", action="store_true")
|
|
args = parser.parse_args()
|
|
|
|
if args.task != "room_classification" and args.mode in ("closed", "open"):
|
|
args.mode = "default"
|
|
|
|
result = run(
|
|
model_tag=args.model,
|
|
task=args.task,
|
|
mode=args.mode,
|
|
dataset_dir=args.dataset_dir,
|
|
endpoint=args.endpoint,
|
|
warmup=args.warmup,
|
|
n_samples=args.n,
|
|
strip_thinking=not args.no_strip_thinking,
|
|
)
|
|
|
|
print(json.dumps(_result_to_dict(result), indent=2))
|
|
if result.error:
|
|
sys.exit(1)
|
|
|
|
|
|
def _result_to_dict(r: Result) -> dict:
|
|
return {
|
|
"model_tag": r.model_tag,
|
|
"task": r.task,
|
|
"mode": r.mode,
|
|
"n_samples": r.n_samples,
|
|
"accuracy": round(r.accuracy, 4),
|
|
"extras": r.extras,
|
|
"timing": asdict(r.timing),
|
|
"vram_resident_mb": r.vram_resident_mb,
|
|
"vram_peak_mb": r.vram_peak_mb,
|
|
"host": asdict(r.host),
|
|
"run_date": r.run_date,
|
|
"error": r.error,
|
|
}
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|