Skip to content

Commit 967aec1

Browse files
JOSH1024JOSH1024
authored
Merge branch 'main' into fix-mtp-kv-offloading-segfault
2 parents ee5a29c + 469f3dc commit 967aec1

52 files changed

Lines changed: 3586 additions & 203 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.buildkite/intel_jobs/expert_parallelism_intel.yaml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
group: Expert Parallelism
2+
depends_on:
3+
- image-build-xpu
4+
steps:
5+
- label: EPLB Algorithm
6+
key: eplb-algorithm
7+
timeout_in_minutes: 45
8+
device: intel_gpu
9+
no_plugin: true
10+
working_dir: "."
11+
env:
12+
REGISTRY: "public.ecr.aws/q9t5s3a7"
13+
REPO: "vllm-ci-test-repo"
14+
VLLM_TEST_DEVICE: "xpu"
15+
source_file_dependencies:
16+
- vllm/distributed/eplb
17+
- tests/distributed/test_eplb_algo.py
18+
- tests/distributed/test_eplb_utils.py
19+
commands:
20+
- >-
21+
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
22+
'cd tests &&
23+
pytest -v -s distributed/test_eplb_algo.py'

‎.buildkite/intel_jobs/misc_intel.yaml‎

Lines changed: 109 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,17 @@ steps:
3838
REPO: "vllm-ci-test-repo"
3939
VLLM_TEST_DEVICE: "xpu"
4040
source_file_dependencies:
41-
- vllm/
41+
- vllm/config/
42+
- vllm/distributed/
43+
- vllm/engine/
44+
- vllm/inputs/
45+
- vllm/logger.py
46+
- vllm/model_executor/
47+
- vllm/platforms/
48+
- vllm/sampling_params.py
49+
- vllm/transformers_utils/
50+
- vllm/utils/
51+
- vllm/v1/
4252
- tests/v1/sample
4353
- tests/v1/logits_processors
4454
- tests/v1/test_oracle.py
@@ -76,3 +86,101 @@ steps:
7686
cd tests &&
7787
pytest -v -s v1/kv_offload &&
7888
pytest -v -s v1/kv_connector/unit/test_offloading_connector.py'
89+
90+
- label: Regression
91+
key: regression
92+
timeout_in_minutes: 30
93+
device: intel_gpu
94+
no_plugin: true
95+
working_dir: "."
96+
env:
97+
REGISTRY: "public.ecr.aws/q9t5s3a7"
98+
REPO: "vllm-ci-test-repo"
99+
VLLM_TEST_DEVICE: "xpu"
100+
source_file_dependencies:
101+
- vllm/config/
102+
- vllm/distributed/
103+
- vllm/engine/
104+
- vllm/inputs/
105+
- vllm/model_executor/
106+
- vllm/multimodal/
107+
- vllm/platforms/
108+
- vllm/sampling_params.py
109+
- vllm/transformers_utils/
110+
- vllm/utils/
111+
- vllm/v1/
112+
- tests/test_regression
113+
commands:
114+
- >-
115+
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
116+
'pip install modelscope &&
117+
cd tests &&
118+
pytest -v -s test_regression.py'
119+
120+
- label: Metrics, Tracing (2 GPUs)
121+
key: metrics-tracing-2-gpus
122+
timeout_in_minutes: 30
123+
num_devices: 2
124+
device: intel_gpu
125+
no_plugin: true
126+
working_dir: "."
127+
env:
128+
REGISTRY: "public.ecr.aws/q9t5s3a7"
129+
REPO: "vllm-ci-test-repo"
130+
VLLM_TEST_DEVICE: "xpu"
131+
source_file_dependencies:
132+
- vllm/config/
133+
- vllm/distributed/
134+
- vllm/engine/
135+
- vllm/inputs/
136+
- vllm/model_executor/
137+
- vllm/multimodal/
138+
- vllm/platforms/
139+
- vllm/sampling_params.py
140+
- vllm/tracing/
141+
- vllm/transformers_utils/
142+
- vllm/utils/
143+
- vllm/v1/
144+
- tests/v1/tracing
145+
commands:
146+
- >-
147+
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
148+
'pip install opentelemetry-sdk\>=1.26.0 opentelemetry-api\>=1.26.0 opentelemetry-exporter-otlp\>=1.26.0 opentelemetry-semantic-conventions-ai\>=0.4.1 &&
149+
cd tests &&
150+
pytest -v -s v1/tracing'
151+
152+
- label: Async Engine, Inputs, Utils, Worker
153+
key: async-engine-inputs-utils-worker
154+
timeout_in_minutes: 30
155+
device: intel_gpu
156+
no_plugin: true
157+
working_dir: "."
158+
env:
159+
REGISTRY: "public.ecr.aws/q9t5s3a7"
160+
REPO: "vllm-ci-test-repo"
161+
VLLM_TEST_DEVICE: "xpu"
162+
source_file_dependencies:
163+
- vllm/assets/
164+
- vllm/config/
165+
- vllm/distributed/
166+
- vllm/engine/
167+
- vllm/inputs/
168+
- vllm/model_executor/
169+
- vllm/multimodal/
170+
- vllm/platforms/
171+
- vllm/sampling_params.py
172+
- vllm/tokenizers/
173+
- vllm/transformers_utils/
174+
- vllm/utils/
175+
- vllm/v1/
176+
- tests/detokenizer
177+
- tests/multimodal
178+
- tests/utils_
179+
commands:
180+
- >-
181+
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
182+
'cd tests &&
183+
pip install av &&
184+
pytest -v -s detokenizer &&
185+
pytest -v -s -m "not cpu_test" ./multimodal &&
186+
pytest -v -s utils_ --ignore=utils_/test_mem_utils.py'

‎.buildkite/intel_jobs/models_multimodal_intel.yaml‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
group: Models - Multimodal
2+
depends_on:
3+
- image-build-xpu
4+
steps:
5+
- label: "Multi-Modal Models (Standard) 1: qwen2"
6+
key: multi-modal-models-standard-1-qwen2
7+
timeout_in_minutes: 45
8+
device: intel_gpu
9+
no_plugin: true
10+
working_dir: "."
11+
env:
12+
REGISTRY: "public.ecr.aws/q9t5s3a7"
13+
REPO: "vllm-ci-test-repo"
14+
VLLM_TEST_DEVICE: "xpu"
15+
source_file_dependencies:
16+
- vllm/
17+
- tests/models/multimodal
18+
commands:
19+
- >-
20+
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
21+
'pip install av git+https://github.com/TIGER-AI-Lab/Mantis.git &&
22+
cd tests &&
23+
pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2" &&
24+
pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model'
25+
26+
- label: "Multi-Modal Models (Standard) 2: qwen3 + gemma"
27+
key: multi-modal-models-standard-2-qwen3-gemma
28+
timeout_in_minutes: 45
29+
device: intel_gpu
30+
no_plugin: true
31+
working_dir: "."
32+
env:
33+
REGISTRY: "public.ecr.aws/q9t5s3a7"
34+
REPO: "vllm-ci-test-repo"
35+
VLLM_TEST_DEVICE: "xpu"
36+
source_file_dependencies:
37+
- vllm/
38+
- tests/models/multimodal
39+
commands:
40+
- >-
41+
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
42+
'pip install git+https://github.com/TIGER-AI-Lab/Mantis.git &&
43+
cd tests &&
44+
pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model'
45+
46+
- label: "Multi-Modal Models (Standard) 3: llava + qwen2_vl"
47+
key: multi-modal-models-standard-3-llava-qwen2-vl
48+
timeout_in_minutes: 45
49+
device: intel_gpu
50+
no_plugin: true
51+
working_dir: "."
52+
env:
53+
REGISTRY: "public.ecr.aws/q9t5s3a7"
54+
REPO: "vllm-ci-test-repo"
55+
VLLM_TEST_DEVICE: "xpu"
56+
source_file_dependencies:
57+
- vllm/
58+
- tests/models/multimodal
59+
commands:
60+
- >-
61+
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
62+
'pip install git+https://github.com/TIGER-AI-Lab/Mantis.git &&
63+
cd tests &&
64+
pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma" &&
65+
pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model'
66+
67+
- label: "Multi-Modal Models (Standard) 4: other + whisper"
68+
key: multi-modal-models-standard-4-other-whisper
69+
timeout_in_minutes: 45
70+
device: intel_gpu
71+
no_plugin: true
72+
working_dir: "."
73+
env:
74+
REGISTRY: "public.ecr.aws/q9t5s3a7"
75+
REPO: "vllm-ci-test-repo"
76+
VLLM_TEST_DEVICE: "xpu"
77+
source_file_dependencies:
78+
- vllm/
79+
- tests/models/multimodal
80+
commands:
81+
- >-
82+
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
83+
'pip install av git+https://github.com/TIGER-AI-Lab/Mantis.git &&
84+
cd tests &&
85+
pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing'
86+
87+
- label: Multi-Modal Processor # 44min
88+
key: multi-modal-processor
89+
timeout_in_minutes: 45
90+
device: intel_gpu
91+
no_plugin: true
92+
working_dir: "."
93+
env:
94+
REGISTRY: "public.ecr.aws/q9t5s3a7"
95+
REPO: "vllm-ci-test-repo"
96+
VLLM_TEST_DEVICE: "xpu"
97+
source_file_dependencies:
98+
- vllm/
99+
- tests/models/multimodal
100+
- tests/models/registry.py
101+
commands:
102+
- >-
103+
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
104+
'pip install av matplotlib ftfy git+https://github.com/TIGER-AI-Lab/Mantis.git &&
105+
pip install open-clip-torch --no-deps &&
106+
cd tests &&
107+
pytest -v -s models/multimodal/processing/test_tensor_schema.py
108+
--deselect "tests/models/multimodal/processing/test_tensor_schema.py::test_model_tensor_schema[mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4]"
109+
--deselect "tests/models/multimodal/processing/test_tensor_schema.py::test_model_tensor_schema[Qwen/Qwen2.5-Omni-7B-AWQ]"
110+
--num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB'
111+
parallelism: 4

‎.buildkite/scripts/hardware_ci/run-amd-test.sh‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,10 @@
2828
###############################################################################
2929
set -o pipefail
3030

31-
# Export Python path
32-
export PYTHONPATH=".."
31+
# Export Python path for commands that run directly on the host. Containerized
32+
# tests set this to /vllm-workspace below so spawned Python processes do not
33+
# depend on their current working directory.
34+
export PYTHONPATH="${PYTHONPATH:-..}"
3335

3436
###############################################################################
3537
# Helper Functions
@@ -377,6 +379,14 @@ HF_CACHE="$(realpath ~)/huggingface"
377379
mkdir -p "${HF_CACHE}"
378380
HF_MOUNT="/root/.cache/huggingface"
379381

382+
# Hugging Face Hub defaults to 10s request/download timeouts, while the ROCm
383+
# CI image currently raises downloads to 60s. AMD model-test jobs routinely
384+
# start from a cold or partially-populated shared cache, and the 60s read cap
385+
# has still timed out before pytest reached the vLLM behavior under test.
386+
# Keep the CI default explicit and overridable from the Buildkite environment.
387+
: "${HF_HUB_DOWNLOAD_TIMEOUT:=300}"
388+
: "${HF_HUB_ETAG_TIMEOUT:=60}"
389+
380390
# ---- Command source selection ----
381391
# Prefer VLLM_TEST_COMMANDS (preserves all inner quoting intact).
382392
# Fall back to $* for backward compatibility, but warn that inner
@@ -416,7 +426,14 @@ fi
416426

417427
echo "Final commands: $commands"
418428

419-
MYPYTHONPATH=".."
429+
MYPYTHONPATH="/vllm-workspace"
430+
431+
container_job_id="${BUILDKITE_JOB_ID:-${BUILDKITE_PARALLEL_JOB:-0}}"
432+
container_job_id="${container_job_id//[^A-Za-z0-9_.-]/_}"
433+
container_job_id_short="${container_job_id:0:8}"
434+
CONTAINER_TMPDIR="/tmp/vllm-${container_job_id_short}"
435+
CONTAINER_CACHE_ROOT="/tmp/vllm-buildkite-${container_job_id}/cache"
436+
CONTAINER_PREFLIGHT="mkdir -p \"\$TMPDIR\" \"\$TORCHINDUCTOR_CACHE_DIR\" \"\$TRITON_CACHE_DIR\" \"\$VLLM_CACHE_ROOT\" \"\$XDG_CACHE_HOME\" && python -c \"import encodings, importlib.metadata as im, importlib.util as iu; [im.version(d) for d in ('transformers', 'torch', 'ray', 'sympy', 'markupsafe', 'vllm')]; missing=[m for m in ('torch.utils.model_zoo', 'transformers.models.nomic_bert', 'ray.dag', 'sympy.physics', 'markupsafe._speedups') if iu.find_spec(m) is None]; assert not missing, missing\""
420437

421438
# Verify GPU access
422439
render_gid=$(getent group render | cut -d: -f3)
@@ -493,17 +510,24 @@ else
493510
--group-add "$render_gid" \
494511
--rm \
495512
-e HF_TOKEN \
513+
-e "HF_HUB_DOWNLOAD_TIMEOUT=${HF_HUB_DOWNLOAD_TIMEOUT}" \
514+
-e "HF_HUB_ETAG_TIMEOUT=${HF_HUB_ETAG_TIMEOUT}" \
496515
-e AWS_ACCESS_KEY_ID \
497516
-e AWS_SECRET_ACCESS_KEY \
498517
-e BUILDKITE_PARALLEL_JOB \
499518
-e BUILDKITE_PARALLEL_JOB_COUNT \
500519
-v "${HF_CACHE}:${HF_MOUNT}" \
501520
-e "HF_HOME=${HF_MOUNT}" \
502521
-e "PYTHONPATH=${MYPYTHONPATH}" \
522+
-e "TMPDIR=${CONTAINER_TMPDIR}/tmp" \
523+
-e "TORCHINDUCTOR_CACHE_DIR=${CONTAINER_CACHE_ROOT}/torchinductor" \
524+
-e "TRITON_CACHE_DIR=${CONTAINER_CACHE_ROOT}/triton" \
525+
-e "VLLM_CACHE_ROOT=${CONTAINER_CACHE_ROOT}/vllm" \
526+
-e "XDG_CACHE_HOME=${CONTAINER_CACHE_ROOT}/xdg" \
503527
-e "PYTORCH_ROCM_ARCH=" \
504528
--name "${container_name}" \
505529
"${image_name}" \
506-
/bin/bash -c "${commands}"
530+
/bin/bash -c "${CONTAINER_PREFLIGHT} && ${commands}"
507531

508532
exit_code=$?
509533
handle_pytest_exit "$exit_code"

‎.buildkite/scripts/hardware_ci/run-intel-test.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -372,6 +372,8 @@ export HF_TOKEN ZE_AFFINITY_MASK
372372
--entrypoint='' \
373373
-e HF_TOKEN \
374374
-e ZE_AFFINITY_MASK \
375+
-e BUILDKITE_PARALLEL_JOB \
376+
-e BUILDKITE_PARALLEL_JOB_COUNT \
375377
-e CMDS \
376378
--name "${container_name}" \
377379
"${IMAGE}" \

0 commit comments

Comments
 (0)