From 8591d133da281ca48d8a600e7c4fab5360a06d8f Mon Sep 17 00:00:00 2001
From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com>
Date: Thu, 14 May 2026 01:27:53 -0300
Subject: [PATCH] fix(benchmarks): default --num-ctx to 4096 for
 apples-to-apples comparison
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Without an explicit num_ctx, each candidate ran at its Modelfile default
(32k for the Gemma4 variants, larger for qwen3), so VRAM and latency
weren't comparable across families — a 32k-default model pre-allocates
KV cache a 4k-default model doesn't. The flag's own docstring promised
"apples-to-apples" but defaulted to None, defeating the intent.

All current benchmark prompts fit comfortably under 4k tokens
(memory_extraction is the longest at ~500). Users with longer prompts
can still pass --num-ctx <larger>.

Adds a methodology note to the 2026-05-13 multilingual report so its
VRAM/latency numbers aren't conflated with future runs at the new default.
---
 benchmarks/model_eval/orchestrator.py                  | 10 ++++++----
 .../model_eval/reports/2026-05-13-multilingual.md      |  2 ++
 benchmarks/model_eval/runner.py                        | 10 ++++++----
 3 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/benchmarks/model_eval/orchestrator.py b/benchmarks/model_eval/orchestrator.py
index 2f28387..30666bb 100644
--- a/benchmarks/model_eval/orchestrator.py
+++ b/benchmarks/model_eval/orchestrator.py
@@ -160,10 +160,12 @@ def main():
                              "else http://localhost:11434.")
     parser.add_argument("--embed-model", default=_EMBED_MODEL,
                         help=f"Embedding model for semantic-similarity scoring. Default: {_EMBED_MODEL}.")
-    parser.add_argument("--num-ctx", type=int, default=None,
-                        help="Override Ollama context window per request (sent as options.num_ctx). "
-                             "Forces apples-to-apples comparison across candidates regardless of "
-                             "their Modelfile defaults.")
+    parser.add_argument("--num-ctx", type=int, default=4096,
+                        help="Ollama context window per request (sent as options.num_ctx). "
+                             "Defaults to 4096 so every candidate runs at the same window regardless "
+                             "of its Modelfile default — without this, a 32k-default model pre-allocates "
+                             "KV cache that a 4k-default model doesn't, and accuracy/latency/VRAM stop "
+                             "being comparable. Pass a larger value if a task prompt exceeds 4k tokens.")
     parser.add_argument("--warmup", type=int, default=1)
     parser.add_argument("--n", type=int, default=None, help="Limit each task to first N samples (debug mode)")
     parser.add_argument(
diff --git a/benchmarks/model_eval/reports/2026-05-13-multilingual.md b/benchmarks/model_eval/reports/2026-05-13-multilingual.md
index a760ec5..5bffed8 100644
--- a/benchmarks/model_eval/reports/2026-05-13-multilingual.md
+++ b/benchmarks/model_eval/reports/2026-05-13-multilingual.md
@@ -4,6 +4,8 @@
 Hardware: RTX 3080 Laptop 8 GB · Ollama 0.23.3
 Embed model: `nomic-embed-text` · Dataset: n=20 (calibration), n=40 (memory/entity), n=101 (room)
 
+> **Methodology note:** this run predates the `--num-ctx 4096` default. Each model used its own Modelfile context window (32k for the Gemma4 variants, larger for qwen3), which means VRAM and latency numbers across families aren't strictly apples-to-apples. Accuracy is unaffected — the prompts are well under any model's window — but expect e2e and VRAM to drop in subsequent runs with the new default.
+
 ---
 
 ## Models
diff --git a/benchmarks/model_eval/runner.py b/benchmarks/model_eval/runner.py
index a4bad49..6147171 100644
--- a/benchmarks/model_eval/runner.py
+++ b/benchmarks/model_eval/runner.py
@@ -449,10 +449,12 @@ def main():
                         help=f"Embedding model for semantic-similarity scoring "
                              f"(memory_extraction, room_classification:open). "
                              f"Default: {_EMBED_MODEL}.")
-    parser.add_argument("--num-ctx", type=int, default=None,
-                        help="Override Ollama context window per request (sent as options.num_ctx). "
-                             "When set, overrides the model's Modelfile default — useful for "
-                             "apples-to-apples comparison across candidates with different defaults.")
+    parser.add_argument("--num-ctx", type=int, default=4096,
+                        help="Ollama context window per request (sent as options.num_ctx). "
+                             "Defaults to 4096 so every candidate runs at the same window regardless "
+                             "of its Modelfile default — without this, a 32k-default model pre-allocates "
+                             "KV cache that a 4k-default model doesn't, and accuracy/latency/VRAM stop "
+                             "being comparable. Pass a larger value if a task prompt exceeds 4k tokens.")
     args = parser.parse_args()
 
     if args.task != "room_classification" and args.mode in ("closed", "open"):