diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index 10e9327..3ad9d27 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -7,4 +7,5 @@ steps: - python3 -m pip install --user pyyaml - python3 .buildkite/test_generate_pipeline.py - python3 .buildkite/test_benchmark_repetitions.py + - python3 .buildkite/test_server_lifecycle.py - export PATH="$HOME/.local/bin:$PATH" && python3 .buildkite/generate_pipeline.py | buildkite-agent pipeline upload diff --git a/.buildkite/test_server_lifecycle.py b/.buildkite/test_server_lifecycle.py index cbf076f..bf45ddf 100644 --- a/.buildkite/test_server_lifecycle.py +++ b/.buildkite/test_server_lifecycle.py @@ -93,6 +93,54 @@ def test_stop_process_is_bounded_when_term_is_ignored(): assert time.monotonic() - started < 3 +def test_readiness_fails_promptly_for_stopped_or_removed_docker_container(): + for inspect_result in ("echo false", "return 1"): + result = run_bash( + "set -euo pipefail; " + "VLLM_SERVER_CONTAINER=failed-server; " + "server_is_healthy() { return 1; }; " + "docker() { case $1 in " + f"inspect) {inspect_result} ;; " + "logs) echo 'engine initialization failed'; return 1 ;; " + "esac; }; " + "wait_healthy 8000 3600 expected-model", + timeout=3, + ) + assert result.returncode == 1, result.stderr + assert "stopped or became unavailable" in result.stderr + assert "engine initialization failed" in result.stderr + assert "server never came up" not in result.stderr + + +def test_readiness_waits_for_running_docker_container_to_become_healthy(): + result = run_bash( + "set -euo pipefail; " + "VLLM_SERVER_CONTAINER=starting-server; attempts=0; " + "server_is_healthy() { attempts=$((attempts + 1)); (( attempts >= 2 )); }; " + "docker() { [[ $1 == inspect ]] && echo true; }; " + "sleep() { :; }; " + "wait_healthy 8000 3600 expected-model; " + '[[ $attempts == 2 ]]', + timeout=3, + ) + assert result.returncode == 0, result.stderr + assert "server healthy" in result.stdout + + +def test_native_readiness_does_not_require_docker(): + result = run_bash( + "set -euo pipefail; " + "VLLM_SERVER_PID=$$; VLLM_SERVER_CONTAINER=; attempts=0; " + "server_is_healthy() { attempts=$((attempts + 1)); (( attempts >= 2 )); }; " + "docker() { echo 'unexpected Docker call' >&2; return 1; }; " + "sleep() { :; }; " + "wait_healthy 8000 3600 expected-model", + timeout=3, + ) + assert result.returncode == 0, result.stderr + assert "unexpected Docker call" not in result.stderr + + def main(): tests = [value for name, value in sorted(globals().items()) if name.startswith("test_")] failed = 0 diff --git a/README.md b/README.md index 0514e5f..50ed9c0 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,7 @@ A few things worth knowing: - **`vllm.image` is normally just a fallback.** The `VLLM_IMAGE` / `VLLM_COMMIT` build-time env vars override it, which is what you want for nightly perf tracking across a specific vLLM commit. Set **`pin_image: true`** only as a rare escape hatch for a model that genuinely cannot be served by the nightly under test (e.g. support landed in a dedicated image but not yet in nightly) — it makes the workload keep its own `image` regardless of the override. Do not pin models that current nightlies already serve. - **`nightly`** controls only the nightly schedule. Recipes with `nightly: false` (or omitted) are still triggerable explicitly via the `WORKLOADS` env var. - **`timeout_in_minutes`** overrides the Buildkite step timeout (default: `120`). This is separate from `lm_eval.model_args.timeout`, which controls individual API requests. +- **`vllm.startup_timeout_s`** limits the readiness wait (default: `3600`). The wait ends early if the native server process exits or the Docker container stops or disappears. Server lifecycle regression tests run in the CPU pipeline-generation step. - **`lm_eval.tasks` is a list** because each entry runs as a separate `lm_eval` invocation — `--num_fewshot` is a single global flag, so different shot counts need separate runs. Each task's results land in `results///`. - **`vllm_bench` runs first** if both blocks are present — that way perf-pipeline bugs surface quickly instead of waiting on a full lm-eval pass. - **`vllm_bench` uses the `random` dataset with `--ignore-eos`** so every request prefills exactly `input_len` and decodes exactly `output_len` tokens — that's what makes the per-GPU decode throughput meaningful. Pair it with `backend: openai` (the `/v1/completions` endpoint) for exact token control. Avoid `dataset: speed_bench` for throughput numbers: it requires `--skip-tokenizer-init`, which makes `vllm bench serve` cap every request at a single output token, so output throughput reads as ~0. diff --git a/lib/server.sh b/lib/server.sh index 1c2ebb7..3ded372 100644 --- a/lib/server.sh +++ b/lib/server.sh @@ -45,6 +45,9 @@ PY start_server() { local container=$1 port=$2 image=$3 model=$4 serve_args=$5 env=$6 runtime=${7:-docker} + VLLM_SERVER_PID="" + VLLM_SERVER_CONTAINER="" + VLLM_LOG_FILE="" echo "--- :rocket: starting vllm: $model" if [[ "$runtime" == "native" ]]; then @@ -90,6 +93,8 @@ start_server() { "$model" --port "$port" $serve_args fi + VLLM_SERVER_CONTAINER="$container" + # Install pytest to avoid cupy.testing import failure during torch.compile docker exec "$container" pip install -q pytest 2>/dev/null || true @@ -116,7 +121,7 @@ raise SystemExit(0 if any(model.get("id") == expected for model in models) else wait_healthy() { local port=$1 timeout=${2:-3600} expected_model=${3:-} echo "+++ :hourglass: waiting for /health (timeout ${timeout}s)" - local now start deadline next_status elapsed + local now start deadline next_status elapsed container_running start=$(date +%s) deadline=$(( start + timeout )) next_status=$(( start + 60 )) @@ -130,6 +135,16 @@ wait_healthy() { [[ -n "${VLLM_LOG_FILE:-}" ]] && tail -n 80 "$VLLM_LOG_FILE" >&2 || true return 1 fi + if [[ -n "${VLLM_SERVER_CONTAINER:-}" ]]; then + # Docker servers have no local PID. With --rm, a crashed container can + # disappear entirely; neither case should consume the readiness timeout. + if ! container_running=$(docker inspect --format '{{.State.Running}}' "$VLLM_SERVER_CONTAINER" 2>/dev/null) || + [[ "$container_running" != "true" ]]; then + echo "vLLM container $VLLM_SERVER_CONTAINER stopped or became unavailable before becoming healthy" >&2 + docker logs --tail 80 "$VLLM_SERVER_CONTAINER" >&2 || true + return 1 + fi + fi now=$(date +%s) if (( now >= next_status )); then elapsed=$(( now - start )) diff --git a/workloads/deepseek_v4_flash_b200.yaml b/workloads/deepseek_v4_flash_b200.yaml index 99ce0b5..e1aac20 100644 --- a/workloads/deepseek_v4_flash_b200.yaml +++ b/workloads/deepseek_v4_flash_b200.yaml @@ -19,7 +19,7 @@ vllm: --kv-cache-dtype fp8 --block-size 256 --moe-backend deep_gemm_mega_moe - --attention_config.use_fp4_indexer_cache=True + --attention-config.indexer_kv_dtype=mxfp4 --tokenizer-mode deepseek_v4 --tool-call-parser deepseek_v4 --reasoning-parser deepseek_v4 diff --git a/workloads/glm_5_3_h200.yaml b/workloads/glm_5_3_h200.yaml index f5ee68f..e85253a 100644 --- a/workloads/glm_5_3_h200.yaml +++ b/workloads/glm_5_3_h200.yaml @@ -10,8 +10,11 @@ nightly: true vllm: model: zai-org/GLM-5.3 + # The default 1M context exceeds the KV cache available on 8x H200. + # 64K covers the 9K perf requests and the configured GSM8K sequence length. serve_args: >- --tensor-parallel-size 8 + --max-model-len 65536 --kv-cache-dtype fp8 --speculative-config.method mtp --speculative-config.num_speculative_tokens 5