Fix NVIDIA workload startup settings and Docker readiness

.buildkite/pipeline.yaml

@@ -7,4 +7,5 @@ steps:
       - python3 -m pip install --user pyyaml
       - python3 .buildkite/test_generate_pipeline.py
       - python3 .buildkite/test_benchmark_repetitions.py
+      - python3 .buildkite/test_server_lifecycle.py
       - export PATH="$HOME/.local/bin:$PATH" && python3 .buildkite/generate_pipeline.py | buildkite-agent pipeline upload

.buildkite/test_server_lifecycle.py

@@ -93,6 +93,54 @@ def test_stop_process_is_bounded_when_term_is_ignored():
     assert time.monotonic() - started < 3
 
 
+def test_readiness_fails_promptly_for_stopped_or_removed_docker_container():
+    for inspect_result in ("echo false", "return 1"):
+        result = run_bash(
+            "set -euo pipefail; "
+            "VLLM_SERVER_CONTAINER=failed-server; "
+            "server_is_healthy() { return 1; }; "
+            "docker() { case $1 in "
+            f"inspect) {inspect_result} ;; "
+            "logs) echo 'engine initialization failed'; return 1 ;; "
+            "esac; }; "
+            "wait_healthy 8000 3600 expected-model",
+            timeout=3,
+        )
+        assert result.returncode == 1, result.stderr
+        assert "stopped or became unavailable" in result.stderr
+        assert "engine initialization failed" in result.stderr
+        assert "server never came up" not in result.stderr
+
+
+def test_readiness_waits_for_running_docker_container_to_become_healthy():
+    result = run_bash(
+        "set -euo pipefail; "
+        "VLLM_SERVER_CONTAINER=starting-server; attempts=0; "
+        "server_is_healthy() { attempts=$((attempts + 1)); (( attempts >= 2 )); }; "
+        "docker() { [[ $1 == inspect ]] && echo true; }; "
+        "sleep() { :; }; "
+        "wait_healthy 8000 3600 expected-model; "
+        '[[ $attempts == 2 ]]',
+        timeout=3,
+    )
+    assert result.returncode == 0, result.stderr
+    assert "server healthy" in result.stdout
+
+
+def test_native_readiness_does_not_require_docker():
+    result = run_bash(
+        "set -euo pipefail; "
+        "VLLM_SERVER_PID=$$; VLLM_SERVER_CONTAINER=; attempts=0; "
+        "server_is_healthy() { attempts=$((attempts + 1)); (( attempts >= 2 )); }; "
+        "docker() { echo 'unexpected Docker call' >&2; return 1; }; "
+        "sleep() { :; }; "
+        "wait_healthy 8000 3600 expected-model",
+        timeout=3,
+    )
+    assert result.returncode == 0, result.stderr
+    assert "unexpected Docker call" not in result.stderr
+
+
 def main():
     tests = [value for name, value in sorted(globals().items()) if name.startswith("test_")]
     failed = 0

README.md

@@ -112,6 +112,7 @@ A few things worth knowing:
 - **`vllm.image` is normally just a fallback.** The `VLLM_IMAGE` / `VLLM_COMMIT` build-time env vars override it, which is what you want for nightly perf tracking across a specific vLLM commit. Set **`pin_image: true`** only as a rare escape hatch for a model that genuinely cannot be served by the nightly under test (e.g. support landed in a dedicated image but not yet in nightly) — it makes the workload keep its own `image` regardless of the override. Do not pin models that current nightlies already serve.
 - **`nightly`** controls only the nightly schedule. Recipes with `nightly: false` (or omitted) are still triggerable explicitly via the `WORKLOADS` env var.
 - **`timeout_in_minutes`** overrides the Buildkite step timeout (default: `120`). This is separate from `lm_eval.model_args.timeout`, which controls individual API requests.
+- **`vllm.startup_timeout_s`** limits the readiness wait (default: `3600`). The wait ends early if the native server process exits or the Docker container stops or disappears. Server lifecycle regression tests run in the CPU pipeline-generation step.
 - **`lm_eval.tasks` is a list** because each entry runs as a separate `lm_eval` invocation — `--num_fewshot` is a single global flag, so different shot counts need separate runs. Each task's results land in `results/<name>/<task-name>/`.
 - **`vllm_bench` runs first** if both blocks are present — that way perf-pipeline bugs surface quickly instead of waiting on a full lm-eval pass.
 - **`vllm_bench` uses the `random` dataset with `--ignore-eos`** so every request prefills exactly `input_len` and decodes exactly `output_len` tokens — that's what makes the per-GPU decode throughput meaningful. Pair it with `backend: openai` (the `/v1/completions` endpoint) for exact token control. Avoid `dataset: speed_bench` for throughput numbers: it requires `--skip-tokenizer-init`, which makes `vllm bench serve` cap every request at a single output token, so output throughput reads as ~0.

lib/server.sh

@@ -45,6 +45,9 @@ PY
 
 start_server() {
   local container=$1 port=$2 image=$3 model=$4 serve_args=$5 env=$6 runtime=${7:-docker}
+  VLLM_SERVER_PID=""
+  VLLM_SERVER_CONTAINER=""
+  VLLM_LOG_FILE=""
   echo "--- :rocket: starting vllm: $model"
 
   if [[ "$runtime" == "native" ]]; then
@@ -90,6 +93,8 @@ start_server() {
       "$model" --port "$port" $serve_args
   fi
 
+  VLLM_SERVER_CONTAINER="$container"
+
   # Install pytest to avoid cupy.testing import failure during torch.compile
   docker exec "$container" pip install -q pytest 2>/dev/null || true
 
@@ -116,7 +121,7 @@ raise SystemExit(0 if any(model.get("id") == expected for model in models) else
 wait_healthy() {
   local port=$1 timeout=${2:-3600} expected_model=${3:-}
   echo "+++ :hourglass: waiting for /health (timeout ${timeout}s)"
-  local now start deadline next_status elapsed
+  local now start deadline next_status elapsed container_running
   start=$(date +%s)
   deadline=$(( start + timeout ))
   next_status=$(( start + 60 ))
@@ -130,6 +135,16 @@ wait_healthy() {
       [[ -n "${VLLM_LOG_FILE:-}" ]] && tail -n 80 "$VLLM_LOG_FILE" >&2 || true
       return 1
     fi
+    if [[ -n "${VLLM_SERVER_CONTAINER:-}" ]]; then
+      # Docker servers have no local PID. With --rm, a crashed container can
+      # disappear entirely; neither case should consume the readiness timeout.
+      if ! container_running=$(docker inspect --format '{{.State.Running}}' "$VLLM_SERVER_CONTAINER" 2>/dev/null) ||
+          [[ "$container_running" != "true" ]]; then
+        echo "vLLM container $VLLM_SERVER_CONTAINER stopped or became unavailable before becoming healthy" >&2
+        docker logs --tail 80 "$VLLM_SERVER_CONTAINER" >&2 || true
+        return 1
+      fi
+    fi
     now=$(date +%s)
     if (( now >= next_status )); then
       elapsed=$(( now - start ))

workloads/deepseek_v4_flash_b200.yaml

@@ -19,7 +19,7 @@ vllm:
     --kv-cache-dtype fp8
     --block-size 256
     --moe-backend deep_gemm_mega_moe
-    --attention_config.use_fp4_indexer_cache=True
+    --attention-config.indexer_kv_dtype=mxfp4
     --tokenizer-mode deepseek_v4
     --tool-call-parser deepseek_v4
     --reasoning-parser deepseek_v4

workloads/glm_5_3_h200.yaml

@@ -10,8 +10,11 @@ nightly: true
 
 vllm:
   model: zai-org/GLM-5.3
+  # The default 1M context exceeds the KV cache available on 8x H200.
+  # 64K covers the 9K perf requests and the configured GSM8K sequence length.
   serve_args: >-
     --tensor-parallel-size 8
+    --max-model-len 65536
     --kv-cache-dtype fp8
     --speculative-config.method mtp
     --speculative-config.num_speculative_tokens 5