diff --git a/buildkite/pipeline_generator/buildkite_step.py b/buildkite/pipeline_generator/buildkite_step.py index 42c1dd6b..82881965 100644 --- a/buildkite/pipeline_generator/buildkite_step.py +++ b/buildkite/pipeline_generator/buildkite_step.py @@ -299,8 +299,12 @@ def _create_amd_mirror_step(step: Step, original_commands: List[str], amd: Dict[ if working_dir: amd_commands_str = f"cd {working_dir} && {amd_commands_str}" - # Add AMD test script wrapper - amd_command_wrapped = f'bash .buildkite/scripts/hardware_ci/run-amd-test.sh "{amd_commands_str}"' + # Pass commands via VLLM_TEST_COMMANDS env var instead of positional + # argument. This preserves all inner quoting (double quotes from + # _prepare_commands' single-to-double conversion, pytest -m/-k + # expressions, etc.) because Buildkite sets env vars directly in the + # process environment without shell interpretation. + amd_command_wrapped = "bash .buildkite/scripts/hardware_ci/run-amd-test.sh" # Extract device name from queue name device_type = amd_device.replace("amd_", "") if amd_device.startswith("amd_") else amd_device @@ -335,7 +339,7 @@ def _create_amd_mirror_step(step: Step, original_commands: List[str], amd: Dict[ commands=[amd_command_wrapped], depends_on=["image-build-amd"], agents={"queue": amd_queue}, - env={"DOCKER_BUILDKIT": "1"}, + env={"DOCKER_BUILDKIT": "1", "VLLM_TEST_COMMANDS": amd_commands_str}, priority=200, soft_fail=False, retry=amd_retry, diff --git a/buildkite/test-template-amd.j2 b/buildkite/test-template-amd.j2 index de5bb7db..1131d532 100644 --- a/buildkite/test-template-amd.j2 +++ b/buildkite/test-template-amd.j2 @@ -356,9 +356,10 @@ plugins: {% else %} queue: amd_mi325_1 {% endif %} - command: bash .buildkite/scripts/hardware_ci/run-amd-test.sh "(command rocm-smi || true) && export VLLM_TEST_GROUP_NAME={{ step.agent_pool }}-{{ step.label | replace(" ", "-") | lower | replace("(", "") | replace(")", "") | replace("%", "") | replace(",", "-") | replace("+", "-") }} && export VLLM_ALLOW_DEPRECATED_BEAM_SEARCH=1 && cd {{ (step.working_dir or default_working_dir) | safe }} ; {{ step.command or (step.commands | join(" && ")) | safe }}" + command: bash .buildkite/scripts/hardware_ci/run-amd-test.sh env: DOCKER_BUILDKIT: "1" + VLLM_TEST_COMMANDS: "(command rocm-smi || true) && export VLLM_TEST_GROUP_NAME={{ step.agent_pool }}-{{ step.label | replace(' ', '-') | lower | replace('(', '') | replace(')', '') | replace('%', '') | replace(',', '-') | replace('+', '-') }} && export VLLM_ALLOW_DEPRECATED_BEAM_SEARCH=1 && cd {{ (step.working_dir or default_working_dir) | safe }} && {{ step.command or (step.commands | join(' && ')) | safe }}" priority: 100 {% if step.grade and step.grade == "Blocking" %} soft_fail: false diff --git a/buildkite/test-template-ci.j2 b/buildkite/test-template-ci.j2 index 96638c01..5b8e6754 100644 --- a/buildkite/test-template-ci.j2 +++ b/buildkite/test-template-ci.j2 @@ -612,9 +612,10 @@ steps: {% else %} queue: amd_mi325_1 {% endif%} - command: bash .buildkite/scripts/hardware_ci/run-amd-test.sh "(command rocm-smi || true) && export VLLM_ALLOW_DEPRECATED_BEAM_SEARCH=1 && cd {{ (step.working_dir or default_working_dir) | safe }} ; {{ step.command or (step.commands | join(" && ")) | safe }}" + command: bash .buildkite/scripts/hardware_ci/run-amd-test.sh env: DOCKER_BUILDKIT: "1" + VLLM_TEST_COMMANDS: "(command rocm-smi || true) && export VLLM_ALLOW_DEPRECATED_BEAM_SEARCH=1 && cd {{ (step.working_dir or default_working_dir) | safe }} && {{ step.command or (step.commands | join(' && ')) | safe }}" priority: 100 {% endif %} {% endfor %}