From 9bd07d96fe85a07b79937fbc304b244ad7ade50f Mon Sep 17 00:00:00 2001 From: jaylfc Date: Fri, 24 Apr 2026 10:50:54 +0100 Subject: [PATCH] feat(bench): --thinking-mode flag to opt back into reasoning on generator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds --thinking-mode to locomo_runner.py. Default off (think=false in the Ollama request, matches current behaviour). When set, don't pass think=false — lets Qwen3/3.5/3.6 emit reasoning tokens before the answer. Use case: measure whether chain-of-thought improves LoCoMo Judge score. Our qwen3.5:9b + adj=1 landed at 0.481 with thinking off vs gemma4:e2b + adj=1 at 0.465 — only +0.016 for 2x the parameters. Thinking-mode on might recover more of the 9B's capability at the cost of ~30x slower generation. Threaded through _process_qa. Recorded in meta.thinking_mode. --- benchmarks/locomo_runner.py | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/benchmarks/locomo_runner.py b/benchmarks/locomo_runner.py index 1497921d..fae7af8f 100644 --- a/benchmarks/locomo_runner.py +++ b/benchmarks/locomo_runner.py @@ -129,17 +129,18 @@ def _bleu1(pred: str, ref: str) -> float: async def _ollama_generate(client: httpx.AsyncClient, url: str, model: str, - prompt: str, temperature: float = 0.2) -> str: - # think=false disables reasoning mode for Qwen3/3.5/3.6 and other - # thinking-capable models. Without it, the model emits 200+ hidden - # reasoning tokens per call (Ollama strips them from the response but - # bills generation time), making benchmarks 10-20x slower than needed. - resp = await client.post( - f"{url}/api/generate", - json={"model": model, "prompt": prompt, "stream": False, - "think": False, - "options": {"temperature": temperature}}, - ) + prompt: str, temperature: float = 0.2, + thinking_mode: bool = False) -> str: + # Default: think=false disables reasoning mode on Qwen3/3.5/3.6 generators + # (10-20x faster; hidden reasoning adds no visible content but bills full + # generation time). Set thinking_mode=True (via --thinking-mode on the CLI) + # to measure whether reasoning-on improves answer quality. Slow — expect + # ~150s per call vs ~5s with thinking off. + payload = {"model": model, "prompt": prompt, "stream": False, + "options": {"temperature": temperature}} + if not thinking_mode: + payload["think"] = False + resp = await client.post(f"{url}/api/generate", json=payload) resp.raise_for_status() return (resp.json().get("response") or "").strip() @@ -391,6 +392,7 @@ async def _process_qa( reranker: object | None, multihop_decompose: bool, full_context: bool, + thinking_mode: bool, turn_index: dict[str, dict], ) -> dict | None: if "answer" not in qa: @@ -461,6 +463,7 @@ async def _process_qa( predicted = await _ollama_generate( client, ollama_url, model, ANSWER_PROMPT.format(context=context, question=question), + thinking_mode=thinking_mode, ) except Exception as exc: predicted = f"[generation_error: {exc}]" @@ -580,6 +583,7 @@ async def _guarded( reranker=reranker, multihop_decompose=args.multihop_decompose, full_context=args.full_context, + thinking_mode=args.thinking_mode, turn_index=turn_index, ) except Exception as e: @@ -646,6 +650,7 @@ async def _guarded( "reranker": args.reranker, "multihop_decompose": args.multihop_decompose, "full_context": args.full_context, + "thinking_mode": args.thinking_mode, "strategy": args.strategy, "total_qa": len(results), "categories_included": sorted(include_cats), @@ -740,6 +745,13 @@ def _parse_args() -> argparse.Namespace: "performance. --retrieval-top-k, --adjacent-turns, " "--llm-query-expansion, and --multihop-decompose are " "ignored when this flag is set.") + p.add_argument("--thinking-mode", action="store_true", + help="Enable reasoning-mode on the generator (do not pass " + "think=false). For Qwen3/3.5/3.6 this lets the model " + "emit hidden reasoning tokens before the answer. " + "Slow — expect ~150s per call vs ~5s with thinking " + "off — but useful to test whether chain-of-thought " + "improves answer quality. Default off.") p.add_argument("--strategy", choices=["vector-only", "full"], default="vector-only") p.add_argument("--out", default=None) p.add_argument("--run-id", default=None)