noonghunna · Aug 18, 2026 · Aug 18, 2026 · Aug 18, 2026
diff --git a/docs/FAQ.md b/docs/FAQ.md
@@ -82,6 +82,31 @@ Depends on the arch. The short version:
 
 Full hardware-acceleration matrix (which dtypes/quants run on Tensor Cores natively vs in software, per GPU class) at [DTYPE_MATRIX.md](DTYPE_MATRIX.md), including the weight-only vs weight+activation axis and the NVFP4 / MXFP4 / FP6 Blackwell additions.
 
+### How do I turn the speculative-decoding drafter off?
+
+`SPEC_N=0`, on every compose that ships a drafter:
+
+```bash
+SPEC_N=0 bash scripts/switch.sh vllm/dual    # drafter off
+SPEC_N=6 bash scripts/switch.sh vllm/dual    # depth 6 instead of this compose's default
+```
+
+`SPEC=off` does the same thing and is kept as an alias, so older notes and
+existing `.env` files keep working. A non-numeric `SPEC_N` is a hard boot error
+rather than a silent fall back to "off" — a typo can never quietly leave the
+drafter running, nor quietly disable it.
+
+This is worth stating plainly because turning the drafter off is the documented
+mitigation for [vllm#50021](https://github.com/vllm-project/vllm/pull/50021), and
+until 2026-08-18 the knob was **not** uniform: some composes read `SPEC`, some
+`SPEC_N`, some `SPEC_N_MAX` or `NUM_SPEC_TOKENS`, and ten hardcoded the drafter
+with no runtime escape at all — their header told you to delete a line from the
+YAML. Applying the wrong variable produced a healthy-looking server with the
+drafter still running. `scripts/tests/test-spec-toggle-contract.sh` enforces the
+single contract now, by running each entrypoint and inspecting the argv the
+engine would actually receive.
+
+
 ### What is the W4A8 knob and should I turn it on?
 
 `VLLM_MARLIN_INPUT_DTYPE=int8` on the Qwen vLLM composes (`vllm/dual`, `vllm/minimal`) runs

diff --git a/models/gemma-4-12b/vllm/compose/single/autoround-int8/mtp.yml b/models/gemma-4-12b/vllm/compose/single/autoround-int8/mtp.yml
@@ -4,6 +4,7 @@
 #   Engine:    vLLM (vllm/vllm-openai:gemma4-unified — Gemma-4 arch-preview image)
 #   Topology:  Single 3090 (TP=1)
 #   Drafter:   assistant external drafter, n=4 default (google/gemma-4-12B-it-assistant ~0.5B)
+#              Toggle: SPEC_N=<n> sets depth; SPEC_N=0 (or SPEC=off) disables
 #   KV:        bf16 / auto — single kv-dtype (gemma4-unified has no fp8/INT8-PTH path)
 #   Vision:    yes (gemma4_unified is multimodal; served text-only here)
 #   Max ctx:   256K (262144) — MTP fits the FULL ctx on one card (KV pool ~310K tok,
@@ -48,6 +49,10 @@ services:
       - ../../../cache/torch_compile:/root/.cache/vllm/torch_compile_cache
       - ../../../cache/triton:/root/.triton/cache
     environment:
+      # Drafter toggle — docker only forwards what is declared here. The DEFAULT
+      # lives in the entrypoint, not in these lines, so it cannot drift.
+      - SPEC_N=${SPEC_N:-}
+      - SPEC=${SPEC:-}
       - NVIDIA_VISIBLE_DEVICES=${ESTATE_GPUS:-${NVIDIA_VISIBLE_DEVICES:-all}}
       # CUDA-level GPU mask — the runtime-agnostic half of GPU selection
       # (#610): launch.sh --gpus exports GPU UUIDs here. CDI runtimes (NixOS,
@@ -78,7 +83,27 @@ services:
       - /bin/bash
       - -c
       - |
-        exec vllm serve ${VLLM_ENFORCE_EAGER:+--enforce-eager} "$@"
+        # -- Drafter toggle: the same contract on every club-3090 compose --------
+        #   SPEC_N=<n>  drafter depth (this compose defaults to 4); SPEC_N=0 disables
+        #   SPEC=off    back-compat alias for SPEC_N=0
+        # Turning the drafter off is the documented mitigation for vllm#50021 and must
+        # never require editing this file. A non-numeric SPEC_N is a hard error, so a
+        # typo can never masquerade as a silent "drafter off".
+        _spec_n="$${SPEC_N:-4}"
+        case "$${SPEC:-}" in off|OFF|Off|false|no|0) _spec_n=0 ;; esac
+        case "$$_spec_n" in ""|*[!0-9]*)
+          echo "[spec] SPEC_N='$$_spec_n' is not a non-negative integer." >&2
+          echo "[spec] Fix: SPEC_N=<n> sets depth; SPEC_N=0 (or SPEC=off) disables." >&2
+          exit 1 ;;
+        esac
+        SPEC_ARGS=()
+        if [ "$$_spec_n" -gt 0 ]; then
+          SPEC_ARGS=(--speculative-config "{\"model\":\"/root/.cache/huggingface/${SPEC_MODEL_SUBDIR:-gemma-4-12b-it-assistant}\",\"num_speculative_tokens\":$$_spec_n}")
+          echo "[spec] drafter ON - draft-model ${SPEC_MODEL_SUBDIR:-gemma-4-12b-it-assistant} n=$$_spec_n" >&2
+        else
+          echo "[spec] drafter OFF (SPEC_N=0 / SPEC=off) - no speculative decoding" >&2
+        fi
+        exec vllm serve ${VLLM_ENFORCE_EAGER:+--enforce-eager} "$@" "$${SPEC_ARGS[@]}"
       - --
     command:
       - --model
@@ -99,8 +124,6 @@ services:
       - "${GPU_MEMORY_UTILIZATION:-0.92}"
       - --max-num-seqs
       - "${MAX_NUM_SEQS:-4}"
-      - --speculative-config
-      - '{"model":"/root/.cache/huggingface/${SPEC_MODEL_SUBDIR:-gemma-4-12b-it-assistant}","num_speculative_tokens":${SPEC_N:-4}}'
       - --trust-remote-code
       - --enable-auto-tool-choice
       - --tool-call-parser

diff --git a/models/gemma-4-12b/vllm/compose/single/qat-w4a16/mtp.yml b/models/gemma-4-12b/vllm/compose/single/qat-w4a16/mtp.yml
@@ -4,6 +4,7 @@
 #   Engine:    vLLM (vllm/vllm-openai:gemma4-unified — Gemma-4 arch-preview image)
 #   Topology:  Single 3090 (TP=1)
 #   Drafter:   assistant external drafter, n=4 default (google/gemma-4-12B-it-assistant ~0.5B)
+#              Toggle: SPEC_N=<n> sets depth; SPEC_N=0 (or SPEC=off) disables
 #   KV:        bf16 / auto — single kv-dtype (gemma4-unified has no fp8/INT8-PTH path)
 #   Vision:    yes (gemma4_unified is multimodal; served text-only here)
 #   Max ctx:   256K (262144) — MTP fits the FULL ctx on one card (KV pool ~310K tok,
@@ -56,6 +57,10 @@ services:
       # vLLM force-quantizes it because patch_dense is built with quant_config but no prefix=).
       - ../../../patches/gemma4-unified-vision-unquant:/etc/club3090/g4patch:ro
     environment:
+      # Drafter toggle — docker only forwards what is declared here. The DEFAULT
+      # lives in the entrypoint, not in these lines, so it cannot drift.
+      - SPEC_N=${SPEC_N:-}
+      - SPEC=${SPEC:-}
       - NVIDIA_VISIBLE_DEVICES=${ESTATE_GPUS:-${NVIDIA_VISIBLE_DEVICES:-all}}
       # CUDA-level GPU mask — the runtime-agnostic half of GPU selection
       # (#610): launch.sh --gpus exports GPU UUIDs here. CDI runtimes (NixOS,
@@ -87,7 +92,27 @@ services:
       - /bin/bash
       - -c
       - |
-        exec vllm serve ${VLLM_ENFORCE_EAGER:+--enforce-eager} "$@"
+        # -- Drafter toggle: the same contract on every club-3090 compose --------
+        #   SPEC_N=<n>  drafter depth (this compose defaults to 4); SPEC_N=0 disables
+        #   SPEC=off    back-compat alias for SPEC_N=0
+        # Turning the drafter off is the documented mitigation for vllm#50021 and must
+        # never require editing this file. A non-numeric SPEC_N is a hard error, so a
+        # typo can never masquerade as a silent "drafter off".
+        _spec_n="$${SPEC_N:-4}"
+        case "$${SPEC:-}" in off|OFF|Off|false|no|0) _spec_n=0 ;; esac
+        case "$$_spec_n" in ""|*[!0-9]*)
+          echo "[spec] SPEC_N='$$_spec_n' is not a non-negative integer." >&2
+          echo "[spec] Fix: SPEC_N=<n> sets depth; SPEC_N=0 (or SPEC=off) disables." >&2
+          exit 1 ;;
+        esac
+        SPEC_ARGS=()
+        if [ "$$_spec_n" -gt 0 ]; then
+          SPEC_ARGS=(--speculative-config "{\"model\":\"/root/.cache/huggingface/${SPEC_MODEL_SUBDIR:-gemma-4-12b-it-assistant}\",\"num_speculative_tokens\":$$_spec_n}")
+          echo "[spec] drafter ON - draft-model ${SPEC_MODEL_SUBDIR:-gemma-4-12b-it-assistant} n=$$_spec_n" >&2
+        else
+          echo "[spec] drafter OFF (SPEC_N=0 / SPEC=off) - no speculative decoding" >&2
+        fi
+        exec vllm serve ${VLLM_ENFORCE_EAGER:+--enforce-eager} "$@" "$${SPEC_ARGS[@]}"
       - --
     command:
       - --model
@@ -108,8 +133,6 @@ services:
       - "${GPU_MEMORY_UTILIZATION:-0.92}"
       - --max-num-seqs
       - "${MAX_NUM_SEQS:-4}"
-      - --speculative-config
-      - '{"model":"/root/.cache/huggingface/${SPEC_MODEL_SUBDIR:-gemma-4-12b-it-assistant}","num_speculative_tokens":${SPEC_N:-4}}'
       - --trust-remote-code
       - --enable-auto-tool-choice
       - --tool-call-parser

diff --git a/models/gemma-4-26b-a4b/vllm/compose/single/awq/int8.yml b/models/gemma-4-26b-a4b/vllm/compose/single/awq/int8.yml
@@ -3,6 +3,7 @@
 #   Model:     Gemma 4 26B-A4B MoE (cyankiwi AWQ-4bit, compressed-tensors)
 #   Topology:  Single 3090 (TP=1)
 #   Drafter:   MTP n=4 — google/gemma-4-26B-A4B-it-assistant (external, ~0.97 GB)
+#              Toggle: SPEC_N=<n> sets depth; SPEC_N=0 (or SPEC=off) disables
 #   KV:        int8_per_token_head (1 byte/token, via vendored PR #40391 overlay)
 #   Vision:    off (limit-mm-per-prompt image=0 audio=0)
 #   Max ctx:   176K default (single-card boot ceiling ~185K WITH the external MTP
@@ -83,6 +84,10 @@ services:
       # whole dir when #40391 merges. See ../../../patches/vllm-pr40391-v0.22.0/README.md.
       - ../../../patches/vllm-pr40391-v0.22.0:/etc/club3090/pr40391:ro
     environment:
+      # Drafter toggle — docker only forwards what is declared here. The DEFAULT
+      # lives in the entrypoint, not in these lines, so it cannot drift.
+      - SPEC_N=${SPEC_N:-}
+      - SPEC=${SPEC:-}
       - NVIDIA_VISIBLE_DEVICES=${ESTATE_GPUS:-${NVIDIA_VISIBLE_DEVICES:-all}}
       # CUDA-level GPU mask — the runtime-agnostic half of GPU selection
       # (#610): launch.sh --gpus exports GPU UUIDs here. CDI runtimes (NixOS,
@@ -118,7 +123,27 @@ services:
         # stock v0.22.0 BEFORE vllm imports. Fail-loud: a non-zero exit aborts
         # boot rather than serving a half-patched engine.
         bash /etc/club3090/pr40391/install.sh
-        exec vllm serve "$@"
+        # -- Drafter toggle: the same contract on every club-3090 compose --------
+        #   SPEC_N=<n>  drafter depth (this compose defaults to 4); SPEC_N=0 disables
+        #   SPEC=off    back-compat alias for SPEC_N=0
+        # Turning the drafter off is the documented mitigation for vllm#50021 and must
+        # never require editing this file. A non-numeric SPEC_N is a hard error, so a
+        # typo can never masquerade as a silent "drafter off".
+        _spec_n="$${SPEC_N:-4}"
+        case "$${SPEC:-}" in off|OFF|Off|false|no|0) _spec_n=0 ;; esac
+        case "$$_spec_n" in ""|*[!0-9]*)
+          echo "[spec] SPEC_N='$$_spec_n' is not a non-negative integer." >&2
+          echo "[spec] Fix: SPEC_N=<n> sets depth; SPEC_N=0 (or SPEC=off) disables." >&2
+          exit 1 ;;
+        esac
+        SPEC_ARGS=()
+        if [ "$$_spec_n" -gt 0 ]; then
+          SPEC_ARGS=(--speculative-config "{\"model\":\"/root/.cache/huggingface/gemma-4-26b-a4b-it-assistant\",\"num_speculative_tokens\":$$_spec_n}")
+          echo "[spec] drafter ON - draft-model gemma-4-26b-a4b-it-assistant n=$$_spec_n" >&2
+        else
+          echo "[spec] drafter OFF (SPEC_N=0 / SPEC=off) - no speculative decoding" >&2
+        fi
+        exec vllm serve "$@" "$${SPEC_ARGS[@]}"
       - --
     command:
       - --override-generation-config
@@ -182,6 +207,4 @@ services:
       - "${LONG_PREFILL_TOKEN_THRESHOLD:-2048}"
       # External MTP assistant drafter — n=4 per gemma-26b-it-assistant DrafterProfile.
       # Same external-drafter path as the dual; bypasses MoE expert routing.
-      - --speculative-config
-      - '{"model":"/root/.cache/huggingface/gemma-4-26b-a4b-it-assistant","num_speculative_tokens":4}'
 # Engine-profile: vllm-gemma-stable
diff --git a/models/gemma-4-31b/vllm/compose/dual/autoround-int4/bf16-mtp.yml b/models/gemma-4-31b/vllm/compose/dual/autoround-int4/bf16-mtp.yml
@@ -3,6 +3,7 @@
 #   Model:     Gemma 4 31B (Intel AutoRound INT4)
 #   Topology:  Dual 3090 (TP=2, NVLink auto-detected via NVLINK_MODE) — single-card boot-OOMs on 24 GB
 #   Drafter:   MTP n=4 (Google's official `gemma-4-31B-it-assistant`, BF16 0.5B)
+#              Toggle: SPEC_N=<n> sets depth; SPEC_N=0 (or SPEC=off) disables
 #   KV:        bfloat16 (2 bytes/token)
 #   Vision:    yes
 #   Max ctx:   131K default @ 4 streams (NIAH-validated to 120K) — BF16 KV pool
@@ -89,6 +90,10 @@ services:
       # NVLink auto-detection — runs inside container at boot.
       - ../../../../../../scripts/detect_nvlink.sh:/etc/club3090/detect_nvlink.sh:ro
     environment:
+      # Drafter toggle — docker only forwards what is declared here. The DEFAULT
+      # lives in the entrypoint, not in these lines, so it cannot drift.
+      - SPEC_N=${SPEC_N:-}
+      - SPEC=${SPEC:-}
       - NVIDIA_VISIBLE_DEVICES=${ESTATE_GPUS:-${NVIDIA_VISIBLE_DEVICES:-all}}
       # CUDA-level GPU mask — the runtime-agnostic half of GPU selection
       # (#610): launch.sh --gpus exports GPU UUIDs here. CDI runtimes (NixOS,
@@ -132,10 +137,30 @@ services:
         bash /etc/club3090/pr42006/install.sh
         # NVLink auto-detection (sets NCCL env vars, _NVLINK_ENABLED).
         source /etc/club3090/detect_nvlink.sh
+        # -- Drafter toggle: the same contract on every club-3090 compose --------
+        #   SPEC_N=<n>  drafter depth (this compose defaults to 4); SPEC_N=0 disables
+        #   SPEC=off    back-compat alias for SPEC_N=0
+        # Turning the drafter off is the documented mitigation for vllm#50021 and must
+        # never require editing this file. A non-numeric SPEC_N is a hard error, so a
+        # typo can never masquerade as a silent "drafter off".
+        _spec_n="$${SPEC_N:-4}"
+        case "$${SPEC:-}" in off|OFF|Off|false|no|0) _spec_n=0 ;; esac
+        case "$$_spec_n" in ""|*[!0-9]*)
+          echo "[spec] SPEC_N='$$_spec_n' is not a non-negative integer." >&2
+          echo "[spec] Fix: SPEC_N=<n> sets depth; SPEC_N=0 (or SPEC=off) disables." >&2
+          exit 1 ;;
+        esac
+        SPEC_ARGS=()
+        if [ "$$_spec_n" -gt 0 ]; then
+          SPEC_ARGS=(--speculative-config "{\"model\":\"/root/.cache/huggingface/gemma-4-31b-it-assistant\",\"num_speculative_tokens\":$$_spec_n}")
+          echo "[spec] drafter ON - draft-model gemma-4-31b-it-assistant n=$$_spec_n" >&2
+        else
+          echo "[spec] drafter OFF (SPEC_N=0 / SPEC=off) - no speculative decoding" >&2
+        fi
         if [ "$${_NVLINK_ENABLED:-0}" = "1" ]; then
-          exec vllm serve "$$@"
+          exec vllm serve "$$@" "$${SPEC_ARGS[@]}"
         else
-          exec vllm serve --disable-custom-all-reduce "$$@"
+          exec vllm serve --disable-custom-all-reduce "$$@" "$${SPEC_ARGS[@]}"
         fi
       - --
     command:
@@ -199,5 +224,3 @@ services:
       - gemma4
       - --chat-template
       - /etc/club3090/gemma-canonical-template.jinja
-      - --speculative-config
-      - '{"model":"/root/.cache/huggingface/gemma-4-31b-it-assistant","num_speculative_tokens":4}'
diff --git a/models/gemma-4-31b/vllm/compose/dual/autoround-int4/int8.yml b/models/gemma-4-31b/vllm/compose/dual/autoround-int4/int8.yml
@@ -3,6 +3,7 @@
 #   Model:     Gemma 4 31B (Intel AutoRound INT4)
 #   Topology:  Dual 3090 (TP=2, NVLink auto-detected via NVLINK_MODE)
 #   Drafter:   MTP n=4 (Google's official `gemma-4-31B-it-assistant`, BF16 0.5B)
+#              Toggle: SPEC_N=<n> sets depth; SPEC_N=0 (or SPEC=off) disables
 #   KV:        int8_per_token_head (1 byte/token, via vendored PR #40391 overlay)
 #   Vision:    yes
 #   Max ctx:   262K native default (dual-card max-ctx rule) @ 4-stream cap; INT8 PTH pool ~2× BF16
@@ -151,6 +152,11 @@ services:
       #     minimal overlay surface. Tracked in docs/UPSTREAM.md (re-test trigger:
       #     streaming multi-tool-call regression).
     environment:
+      # Drafter toggle — docker only forwards what is declared here. The DEFAULT
+      # lives in the entrypoint, not in these lines, so it cannot drift.
+      - SPEC_N=${SPEC_N:-}
+      - SPEC=${SPEC:-}
+      - SPEC_N_MAX=${SPEC_N_MAX:-}
       - NVIDIA_VISIBLE_DEVICES=${ESTATE_GPUS:-${NVIDIA_VISIBLE_DEVICES:-all}}
       # CUDA-level GPU mask — the runtime-agnostic half of GPU selection
       # (#610): launch.sh --gpus exports GPU UUIDs here. CDI runtimes (NixOS,
@@ -195,10 +201,30 @@ services:
         bash /etc/club3090/pr42006/install.sh
         # NVLink auto-detection (sets NCCL env vars, _NVLINK_ENABLED).
         source /etc/club3090/detect_nvlink.sh
+        # -- Drafter toggle: the same contract on every club-3090 compose --------
+        #   SPEC_N=<n>  drafter depth (this compose defaults to 4); SPEC_N=0 disables
+        #   SPEC=off    back-compat alias for SPEC_N=0
+        # Turning the drafter off is the documented mitigation for vllm#50021 and must
+        # never require editing this file. A non-numeric SPEC_N is a hard error, so a
+        # typo can never masquerade as a silent "drafter off".
+        _spec_n="$${SPEC_N:-$${SPEC_N_MAX:-4}}"
+        case "$${SPEC:-}" in off|OFF|Off|false|no|0) _spec_n=0 ;; esac
+        case "$$_spec_n" in ""|*[!0-9]*)
+          echo "[spec] SPEC_N='$$_spec_n' is not a non-negative integer." >&2
+          echo "[spec] Fix: SPEC_N=<n> sets depth; SPEC_N=0 (or SPEC=off) disables." >&2
+          exit 1 ;;
+        esac
+        SPEC_ARGS=()
+        if [ "$$_spec_n" -gt 0 ]; then
+          SPEC_ARGS=(--speculative-config "{\"model\":\"/root/.cache/huggingface/gemma-4-31b-it-assistant\",\"num_speculative_tokens\":$$_spec_n}")
+          echo "[spec] drafter ON - draft-model gemma-4-31b-it-assistant n=$$_spec_n" >&2
+        else
+          echo "[spec] drafter OFF (SPEC_N=0 / SPEC=off) - no speculative decoding" >&2
+        fi
         if [ "$${_NVLINK_ENABLED:-0}" = "1" ]; then
-          exec vllm serve "$$@"
+          exec vllm serve "$$@" "$${SPEC_ARGS[@]}"
         else
-          exec vllm serve --disable-custom-all-reduce "$$@"
+          exec vllm serve --disable-custom-all-reduce "$$@" "$${SPEC_ARGS[@]}"
         fi
       - --
     command:
@@ -270,5 +296,3 @@ services:
       # qwen3.6-27b/vllm/compose/dual/autoround-int4/int8.yml (Qwen's built-in MTP caps at
       # n=3 architecturally, so the canonical apples-to-apples bench uses
       # SPEC_N_MAX=3 here).
-      - --speculative-config
-      - '{"model":"/root/.cache/huggingface/gemma-4-31b-it-assistant","num_speculative_tokens":${SPEC_N_MAX:-4}}'