diff --git a/.github/workflows/pr-test-amd-extra.yml b/.github/workflows/pr-test-amd-extra.yml index fb964f03bfdd..029a51970a0f 100644 --- a/.github/workflows/pr-test-amd-extra.yml +++ b/.github/workflows/pr-test-amd-extra.yml @@ -37,18 +37,24 @@ on: - mi300 - mi325 rocm_version: - description: 'ROCm container variant (rocm720 = ROCm 7.2.0, the PR default; rocm700 = ROCm 7.0.0)' + description: 'ROCm container variant (ROCm 7.2.0 by default)' required: false type: choice default: rocm720 options: - rocm720 + - rocm724 - rocm700 aiter_ref: description: 'Override AITER commit (optional, leave empty to use Dockerfile default)' required: false type: string default: '' + amd_ci_image: + description: 'Override AMD CI Docker image (optional, e.g. rocm/sgl-dev:tag)' + required: false + type: string + default: '' continue_on_error: description: 'Continue on error (do not fail the workflow on test failures)' required: false @@ -67,7 +73,7 @@ on: type: string default: mi300 rocm_version: - description: 'ROCm container variant (rocm720 = ROCm 7.2.0, the PR default; rocm700 = ROCm 7.0.0)' + description: 'ROCm container variant (ROCm 7.2.0 by default)' required: false type: string default: rocm720 @@ -76,6 +82,11 @@ on: required: false type: string default: '' + amd_ci_image: + description: 'Override AMD CI Docker image (optional, e.g. rocm/sgl-dev:tag)' + required: false + type: string + default: '' continue_on_error: description: 'Continue on error (do not fail the workflow on test failures)' required: false @@ -84,6 +95,7 @@ on: env: AITER_COMMIT_OVERRIDE: ${{ inputs.aiter_ref }} + AMD_CI_IMAGE: ${{ inputs.amd_ci_image }} DOCKERHUB_AMD_USERNAME: ${{ secrets.DOCKERHUB_AMD_USERNAME }} DOCKERHUB_AMD_TOKEN: ${{ secrets.DOCKERHUB_AMD_TOKEN }} @@ -145,7 +157,7 @@ jobs: - name: Start CI container # `inputs` is empty on pull_request events, so fall back explicitly to - # the ROCm 7.2 PR default. Reusable callers can still request ROCm 7.0. + # the ROCm 7.2 PR default. Reusable callers can request another variant. run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ inputs.rocm_version || 'rocm720' }} env: GITHUB_WORKSPACE: ${{ github.workspace }} diff --git a/.github/workflows/pr-test-amd-rocm720.yml b/.github/workflows/pr-test-amd-rocm720.yml index 699d612114b6..ec6ac69572ea 100644 --- a/.github/workflows/pr-test-amd-rocm720.yml +++ b/.github/workflows/pr-test-amd-rocm720.yml @@ -16,6 +16,14 @@ on: - "docker/rocm.Dockerfile" workflow_dispatch: inputs: + rocm_version: + description: 'ROCm image version' + required: false + type: choice + default: rocm720 + options: + - rocm720 + - rocm724 target_stage_select: description: "Select a stage to run from dropdown (leave empty for auto-detect)" required: false @@ -57,6 +65,11 @@ on: required: false type: string default: '' + amd_ci_image: + description: 'Override AMD CI Docker image (optional, e.g. rocm/sgl-dev:tag)' + required: false + type: string + default: '' continue_on_error: description: 'Continue on error (do not fail the workflow on test failures)' required: false @@ -77,6 +90,11 @@ on: default: false workflow_call: inputs: + rocm_version: + description: 'ROCm image version' + required: false + type: string + default: rocm720 ref: description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.' required: false @@ -97,6 +115,11 @@ on: required: false type: string default: '' + amd_ci_image: + description: 'Override AMD CI Docker image (optional, e.g. rocm/sgl-dev:tag)' + required: false + type: string + default: '' continue_on_error: description: 'Continue on error (do not fail the workflow on test failures)' required: false @@ -116,6 +139,8 @@ permissions: env: AITER_COMMIT_OVERRIDE: ${{ inputs.aiter_ref }} + AMD_CI_IMAGE: ${{ inputs.amd_ci_image }} + ROCM_VERSION: ${{ inputs.rocm_version || 'rocm720' }} DOCKERHUB_AMD_USERNAME: ${{ secrets.DOCKERHUB_AMD_USERNAME }} DOCKERHUB_AMD_TOKEN: ${{ secrets.DOCKERHUB_AMD_TOKEN }} @@ -225,8 +250,9 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.ref || '' }} runner_arch: ${{ inputs.runner_arch || 'mi300' }} - rocm_version: rocm720 + rocm_version: ${{ inputs.rocm_version || 'rocm720' }} aiter_ref: ${{ inputs.aiter_ref }} + amd_ci_image: ${{ inputs.amd_ci_image }} continue_on_error: true secrets: inherit @@ -255,7 +281,7 @@ jobs: run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "$ROCM_VERSION" env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -313,7 +339,7 @@ jobs: run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "$ROCM_VERSION" env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -354,7 +380,7 @@ jobs: run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "$ROCM_VERSION" env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -391,7 +417,7 @@ jobs: run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "$ROCM_VERSION" env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -428,7 +454,7 @@ jobs: run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "$ROCM_VERSION" env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -496,7 +522,7 @@ jobs: run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "$ROCM_VERSION" env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -532,7 +558,7 @@ jobs: run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "$ROCM_VERSION" env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -571,7 +597,7 @@ jobs: run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "$ROCM_VERSION" env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -612,7 +638,7 @@ jobs: run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "$ROCM_VERSION" env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -653,7 +679,7 @@ jobs: run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "$ROCM_VERSION" env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -702,7 +728,7 @@ jobs: pattern: wheel-python3.10-cuda12.9 - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "$ROCM_VERSION" env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -833,7 +859,7 @@ jobs: pattern: wheel-python3.10-cuda12.9 - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "$ROCM_VERSION" env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -963,7 +989,7 @@ jobs: pattern: wheel-python3.10-cuda12.9 - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "$ROCM_VERSION" env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -1040,7 +1066,7 @@ jobs: run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "$ROCM_VERSION" env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -1099,7 +1125,7 @@ jobs: run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "$ROCM_VERSION" env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -1145,7 +1171,7 @@ jobs: run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "$ROCM_VERSION" env: GITHUB_WORKSPACE: ${{ github.workspace }} @@ -1189,7 +1215,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "$ROCM_VERSION" env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" @@ -1243,7 +1269,7 @@ jobs: - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version "$ROCM_VERSION" env: GITHUB_WORKSPACE: ${{ github.workspace }} ENABLE_CACHE_HOST: "1" @@ -1336,7 +1362,7 @@ jobs: echo "=== Host RDMA Check Complete ===" - name: Start Special Container - run: bash scripts/ci/amd/amd_ci_start_container_disagg.sh --rocm-version rocm720 + run: bash scripts/ci/amd/amd_ci_start_container_disagg.sh --rocm-version "$ROCM_VERSION" env: GITHUB_WORKSPACE: ${{ github.workspace }} diff --git a/.github/workflows/release-docker-amd-rocm720-nightly.yml b/.github/workflows/release-docker-amd-rocm720-nightly.yml index 8126ed7d66ed..b87edbad97cd 100644 --- a/.github/workflows/release-docker-amd-rocm720-nightly.yml +++ b/.github/workflows/release-docker-amd-rocm720-nightly.yml @@ -10,6 +10,17 @@ on: options: - 'all' - publish + gpu_arch: + description: 'Select which ROCm 7.2 GPU arch to build' + required: false + type: choice + default: 'all' + options: + - 'all' + - gfx942-rocm724 + - gfx950-rocm724 + - gfx942-rocm720 + - gfx950-rocm720 schedule: - cron: '0 12 * * *' @@ -29,7 +40,7 @@ jobs: strategy: fail-fast: false matrix: - gpu_arch: ['gfx942-rocm720', 'gfx950-rocm720'] + gpu_arch: ${{ fromJson((github.event_name == 'workflow_dispatch' && inputs.gpu_arch != 'all' && inputs.gpu_arch != '') && format('["{0}"]', inputs.gpu_arch) || '["gfx942-rocm720", "gfx950-rocm720", "gfx942-rocm724", "gfx950-rocm724"]') }} build_type: ['all'] steps: - name: Checkout repository @@ -81,7 +92,11 @@ jobs: echo "Version: ${version}" echo "Pretend version: ${pretend_version}" - if [ "${{ matrix.gpu_arch }}" = "gfx942-rocm720" ]; then + if [ "${{ matrix.gpu_arch }}" = "gfx942-rocm724" ]; then + rocm_tag="rocm724-mi30x" + elif [ "${{ matrix.gpu_arch }}" = "gfx950-rocm724" ]; then + rocm_tag="rocm724-mi35x" + elif [ "${{ matrix.gpu_arch }}" = "gfx942-rocm720" ]; then rocm_tag="rocm720-mi30x" elif [ "${{ matrix.gpu_arch }}" = "gfx950-rocm720" ]; then rocm_tag="rocm720-mi35x" @@ -141,7 +156,7 @@ jobs: strategy: fail-fast: false matrix: - gpu_arch: ['gfx942-rocm720', 'gfx950-rocm720'] + gpu_arch: ${{ fromJson((github.event_name == 'workflow_dispatch' && inputs.gpu_arch != 'all' && inputs.gpu_arch != '') && format('["{0}"]', inputs.gpu_arch) || '["gfx942-rocm720", "gfx950-rocm720", "gfx942-rocm724", "gfx950-rocm724"]') }} steps: - name: Download image tag artifact uses: actions/download-artifact@v4 diff --git a/.github/workflows/release-docker-amd.yml b/.github/workflows/release-docker-amd.yml index 6920338014f6..e5cd66ca35b1 100644 --- a/.github/workflows/release-docker-amd.yml +++ b/.github/workflows/release-docker-amd.yml @@ -8,6 +8,16 @@ on: version: description: 'Version to build (without v prefix, e.g., 0.5.7)' required: true + rocm_version: + description: 'ROCm version to build' + required: false + type: choice + default: 'all' + options: + - 'all' + - rocm724 + - rocm720 + - rocm700 jobs: publish: @@ -15,8 +25,11 @@ jobs: runs-on: amd-docker-scale environment: 'prod' strategy: + # Six flavors publish independently: a failure in one must not cancel the + # others mid-push and leave the release tag with a partial image set. + fail-fast: false matrix: - rocm_version: ['rocm700', 'rocm720'] + rocm_version: ${{ fromJson((github.event_name == 'workflow_dispatch' && inputs.rocm_version != 'all' && inputs.rocm_version != '') && format('["{0}"]', inputs.rocm_version) || '["rocm700", "rocm720", "rocm724"]') }} gpu_arch: ['gfx942', 'gfx950'] build_type: ['all'] steps: @@ -66,12 +79,12 @@ jobs: echo "Unsupported gfx arch" exit 1 fi - elif [ "${{ matrix.rocm_version }}" = "rocm720" ]; then + elif [ "${{ matrix.rocm_version }}" = "rocm720" ] || [ "${{ matrix.rocm_version }}" = "rocm724" ]; then gpu_arch_suffix="-${{ matrix.rocm_version }}" if [ "${{ matrix.gpu_arch }}" = "gfx942" ]; then - rocm_tag="rocm720-mi30x" + rocm_tag="${{ matrix.rocm_version }}-mi30x" elif [ "${{ matrix.gpu_arch }}" = "gfx950" ]; then - rocm_tag="rocm720-mi35x" + rocm_tag="${{ matrix.rocm_version }}-mi35x" else echo "Unsupported gfx arch" exit 1 diff --git a/docker/rocm.Dockerfile b/docker/rocm.Dockerfile index b0123cef9562..0e6152588db7 100644 --- a/docker/rocm.Dockerfile +++ b/docker/rocm.Dockerfile @@ -1,8 +1,15 @@ # Usage (to build SGLang ROCm docker image): # docker build --build-arg SGL_BRANCH=v0.5.10.post1 --build-arg GPU_ARCH=gfx942 -t v0.5.10.post1-rocm700-mi30x -f rocm.Dockerfile . # docker build --build-arg SGL_BRANCH=v0.5.10.post1 --build-arg GPU_ARCH=gfx942-rocm720 -t v0.5.10.post1-rocm720-mi30x -f rocm.Dockerfile . +# docker build --build-arg SGL_BRANCH=v0.5.10.post1 --build-arg GPU_ARCH=gfx942-rocm724 -t v0.5.10.post1-rocm724-mi30x -f rocm.Dockerfile . # docker build --build-arg SGL_BRANCH=v0.5.10.post1 --build-arg GPU_ARCH=gfx950 -t v0.5.10.post1-rocm700-mi35x -f rocm.Dockerfile . # docker build --build-arg SGL_BRANCH=v0.5.10.post1 --build-arg GPU_ARCH=gfx950-rocm720 -t v0.5.10.post1-rocm720-mi35x -f rocm.Dockerfile . +# docker build --build-arg SGL_BRANCH=v0.5.10.post1 --build-arg GPU_ARCH=gfx950-rocm724 -t v0.5.10.post1-rocm724-mi35x -f rocm.Dockerfile . +# +# Flavor notes: +# GPU_ARCH=*-rocm724 is built on a Python 3.12 base and upgrades the stack to +# torch 2.11 (+torchvision 0.26 / torchaudio 2.11) and Triton 3.7. +# The ROCm 7.2.0 flavors remain on Python 3.10 and torch 2.9.1. # Usage (to build SGLang ROCm + Mori docker image): # remove --build-arg NIC_BACKEND=ainic since new MoRI JIT will do NIC auto detection on target @@ -23,8 +30,10 @@ # Default base images ARG BASE_IMAGE_942="rocm/sgl-dev:rocm7-vllm-20250904" ARG BASE_IMAGE_942_ROCM720="rocm/pytorch:rocm7.2_ubuntu22.04_py3.10_pytorch_release_2.9.1" +ARG BASE_IMAGE_942_ROCM724="rocm/pytorch:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0" ARG BASE_IMAGE_950="rocm/sgl-dev:rocm7-vllm-20250904" ARG BASE_IMAGE_950_ROCM720="rocm/pytorch:rocm7.2_ubuntu22.04_py3.10_pytorch_release_2.9.1" +ARG BASE_IMAGE_950_ROCM724="rocm/pytorch:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0" # This is necessary for scope purpose ARG GPU_ARCH=gfx950 @@ -49,6 +58,30 @@ ENV BUILD_AITER_ALL="1" ENV BUILD_MOONCAKE="1" ENV AITER_COMMIT_DEFAULT="d9e5ef7ce08ee7045d583aed768cff41aa9210fe" +# =============================== +# Base image 942 with rocm724 and args (Python 3.12 + torch 2.11) +FROM $BASE_IMAGE_942_ROCM724 AS gfx942-rocm724 +ENV BUILD_VLLM="0" +ENV BUILD_TRITON="1" +ENV BUILD_LLVM="0" +ENV BUILD_AITER_ALL="1" +ENV BUILD_MOONCAKE="1" +ENV AITER_COMMIT_DEFAULT="d9e5ef7ce08ee7045d583aed768cff41aa9210fe" +# Pin the ROCm torch stack for every pip invocation in this flavor. The file is +# filled in after the torch 2.11 upgrade below; it must already exist (empty is +# valid) because pip reads PIP_CONSTRAINT from the first pip call onwards. +# Deliberately still set in the shipped image, not just during the build: a later +# `pip install` that resolves torch would otherwise pull the PyPI CUDA build over +# this ROCm one, and the constraint turns that into a resolution error instead. +# It names only torch / torchvision / torchaudio, so nothing else is constrained. +ENV PIP_CONSTRAINT="/etc/sglang/constraints/torch-rocm.txt" +RUN mkdir -p /etc/sglang/constraints && : > /etc/sglang/constraints/torch-rocm.txt +# Work around ROCM-21485: the CUDA/ROCm IPC path leaks GPU memory (a freed IPC +# block is not returned to the driver). Legacy IPC mode releases it. Verified on +# ROCm 7.2.1 and 7.2.4; scoped to this flavor so rocm700 / rocm720 keep current +# IPC behavior. +ENV HSA_ENABLE_IPC_MODE_LEGACY=1 + # =============================== # Base image 950 and args FROM $BASE_IMAGE_950 AS gfx950 @@ -69,6 +102,30 @@ ENV BUILD_AITER_ALL="1" ENV BUILD_MOONCAKE="1" ENV AITER_COMMIT_DEFAULT="d9e5ef7ce08ee7045d583aed768cff41aa9210fe" +# =============================== +# Base image 950 with rocm724 and args (Python 3.12 + torch 2.11) +FROM $BASE_IMAGE_950_ROCM724 AS gfx950-rocm724 +ENV BUILD_VLLM="0" +ENV BUILD_TRITON="1" +ENV BUILD_LLVM="0" +ENV BUILD_AITER_ALL="1" +ENV BUILD_MOONCAKE="1" +ENV AITER_COMMIT_DEFAULT="d9e5ef7ce08ee7045d583aed768cff41aa9210fe" +# Pin the ROCm torch stack for every pip invocation in this flavor. The file is +# filled in after the torch 2.11 upgrade below; it must already exist (empty is +# valid) because pip reads PIP_CONSTRAINT from the first pip call onwards. +# Deliberately still set in the shipped image, not just during the build: a later +# `pip install` that resolves torch would otherwise pull the PyPI CUDA build over +# this ROCm one, and the constraint turns that into a resolution error instead. +# It names only torch / torchvision / torchaudio, so nothing else is constrained. +ENV PIP_CONSTRAINT="/etc/sglang/constraints/torch-rocm.txt" +RUN mkdir -p /etc/sglang/constraints && : > /etc/sglang/constraints/torch-rocm.txt +# Work around ROCM-21485: the CUDA/ROCm IPC path leaks GPU memory (a freed IPC +# block is not returned to the driver). Legacy IPC mode releases it. Verified on +# ROCm 7.2.1 and 7.2.4; scoped to this flavor so rocm700 / rocm720 keep current +# IPC behavior. +ENV HSA_ENABLE_IPC_MODE_LEGACY=1 + # Local source stage: with BRANCH_TYPE=local the build context is copied here and # used instead of git clone (mirrors docker/Dockerfile's local_src stage). FROM scratch AS local_src @@ -80,6 +137,11 @@ FROM ${GPU_ARCH} # This is necessary for scope purpose, again ARG GPU_ARCH=gfx950 +# ARG is build-time only. Stamp the stage name (gfx950-rocm724, gfx942, ...) +# so CI can read which AITER_COMMIT_DEFAULT block to use instead of guessing +# from torch or HIP — 720 may also ship torch 2.11 later, and both 7.2 flavors +# report HIP 7.2*. +ENV GPU_ARCH=${GPU_ARCH} ENV GPU_ARCH_LIST=${GPU_ARCH%-*} ENV PYTORCH_ROCM_ARCH=gfx942;gfx950 @@ -91,6 +153,22 @@ ARG BRANCH_TYPE=remote # Version override for setuptools_scm (used in nightly builds) ARG SETUPTOOLS_SCM_PRETEND_VERSION="" +# ROCm 7.2 Triton (BUILD_TRITON=1 stages only). Both wheels are the same +# upstream revision, triton-lang/triton@89002410. AITER only requires +# triton>=3.6.0 and treats the base image as the owner of the version, so the +# choice is ours; bump these together after checking the index. +ARG TRITON_INDEX_URL="https://pypi.amd.com/triton/release/rocm-7.2.0/simple/" +ARG TRITON_VERSION="3.7.0+amd.rocm7.2.0.git89002410" +ARG TRITON_KERNELS_VERSION="1.0.0+amd.rocm7.2.0.git89002410" + +# ROCm 7.2.4 torch upgrade pins (Python 3.12). Torch 2.11 for ROCm 7.2 is only +# published on the PyTorch Foundation index; AMD's repo.radeon.com wheels top +# out at torch 2.10. +ARG TORCH_ROCM_INDEX_URL="https://download.pytorch.org/whl/rocm7.2" +ARG TORCH_ROCM_VERSION="2.11.0+rocm7.2" +ARG TORCHVISION_ROCM_VERSION="0.26.0+rocm7.2" +ARG TORCHAUDIO_ROCM_VERSION="2.11.0+rocm7.2" + ARG AITER_REPO="https://github.com/ROCm/aiter.git" ARG AITER_COMMIT="" ENV AITER_COMMIT="${AITER_COMMIT:-${AITER_COMMIT_DEFAULT}}" @@ -131,14 +209,16 @@ ARG UBUNTU_CODENAME=jammy # Optional Ubuntu mirror override + apt hardening. # - UBUNTU_MIRROR is empty by default (no behaviour change for local builds). # When set (typically in CI), all http://*archive.ubuntu.com and -# http://*security.ubuntu.com entries in /etc/apt/sources.list are rewritten -# to point at the given base URL, e.g. +# http://*security.ubuntu.com entries in every /etc/apt source file are +# rewritten to point at the given base URL, e.g. # --build-arg UBUNTU_MIRROR=https://archive.ubuntu.com # --build-arg UBUNTU_MIRROR=https://tw.archive.ubuntu.com # --build-arg UBUNTU_MIRROR=http://internal-cache.example.com # This mirrors the pattern already used in docker/Dockerfile (NVIDIA) and # docker/npu.Dockerfile, and lets CI runners that cannot reach Canonical's -# port-80 mirror IPs still complete `apt-get update`. +# port-80 mirror IPs still complete `apt-get update`. Every file, not just +# sources.list: the noble base used by rocm724 keeps its URIs in the deb822 +# /etc/apt/sources.list.d/ubuntu.sources instead. # - The 80-net-hardening apt config adds retries + per-request timeout so that # transient mirror flakes don't immediately fail a build (apt's default is 0 # retries). @@ -146,21 +226,40 @@ ARG UBUNTU_MIRROR= USER root RUN if [ -n "$UBUNTU_MIRROR" ]; then \ - sed -i "s|http://[^[:space:]/]*archive.ubuntu.com|$UBUNTU_MIRROR|g" /etc/apt/sources.list && \ - sed -i "s|http://[^[:space:]/]*security.ubuntu.com|$UBUNTU_MIRROR|g" /etc/apt/sources.list; \ + find /etc/apt -type f \( -name '*.list' -o -name '*.sources' \) \ + -exec sed -i \ + -e "s|http://[^[:space:]/]*archive.ubuntu.com|$UBUNTU_MIRROR|g" \ + -e "s|http://[^[:space:]/]*security.ubuntu.com|$UBUNTU_MIRROR|g" \ + {} + ; \ fi && \ printf 'Acquire::Retries "5";\nAcquire::http::Timeout "30";\nAcquire::https::Timeout "30";\n' \ > /etc/apt/apt.conf.d/80-net-hardening -# Fix hipDeviceGetName returning empty string in ROCm 7.0 docker images. -# The ROCm 7.0 base image is missing libdrm-amdgpu-common which provides the -# amdgpu.ids device-ID-to-marketing-name mapping file. -# ROCm 7.2 base images already ship these packages, so this step is skipped. +# Fix hipDeviceGetName returning empty / generic names. +# amdgpu.ids maps PCI IDs to marketing names. The ROCm 7.0 base is missing it. +# The 7.2.0 base was built with amdgpu-install and already has it. The 7.2.4 +# ubuntu24.04 base installs the `rocm` apt metapackage and does not; noble's +# distro table has MI300 (74A*) but no MI355X (75A3), so gfx950-rocm724 would +# otherwise report "AMD Radeon Graphics" and miss every name-keyed config. # See https://github.com/ROCm/ROCm/issues/5992 RUN set -eux; \ case "${GPU_ARCH}" in \ + *rocm724*) \ + echo "ROCm 7.2.4 (GPU_ARCH=${GPU_ARCH}): installing libdrm-amdgpu from graphics/7.2.4 noble"; \ + curl -fsSL https://repo.radeon.com/rocm/rocm.gpg.key \ + | gpg --dearmor -o /etc/apt/keyrings/amdgpu-graphics.gpg \ + && echo 'deb [arch=amd64,i386 signed-by=/etc/apt/keyrings/amdgpu-graphics.gpg] https://repo.radeon.com/graphics/7.2.4/ubuntu noble main' \ + > /etc/apt/sources.list.d/amdgpu-graphics.list \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + libdrm-amdgpu-common \ + libdrm-amdgpu-amdgpu1 \ + libdrm2-amdgpu \ + && rm -rf /var/lib/apt/lists/* \ + && cp /opt/amdgpu/share/libdrm/amdgpu.ids /usr/share/libdrm/amdgpu.ids; \ + ;; \ *rocm720*) \ - echo "ROCm 7.2 (GPU_ARCH=${GPU_ARCH}): libdrm-amdgpu packages already present, skipping"; \ + echo "ROCm 7.2.0 (GPU_ARCH=${GPU_ARCH}): libdrm-amdgpu packages already present, skipping"; \ ;; \ *) \ echo "ROCm 7.0 (GPU_ARCH=${GPU_ARCH}): installing libdrm-amdgpu packages"; \ @@ -187,7 +286,7 @@ RUN apt-get purge -y sccache; python -m pip uninstall -y sccache; rm -f "$(which # The ROCm 7.2 base image (rocm/pytorch) does not pre-install this package. RUN set -eux; \ case "${GPU_ARCH}" in \ - *rocm720*) \ + *rocm720*|*rocm724*) \ echo "ROCm 7.2 flavor detected from GPU_ARCH=${GPU_ARCH}"; \ cd /opt/rocm/share/amd_smi \ && python3 -m pip install --no-cache-dir . \ @@ -197,6 +296,33 @@ RUN set -eux; \ ;; \ esac +# ----------------------- +# ROCm 7.2.4: upgrade torch 2.10 -> 2.11 (+ vision/audio), which pulls triton-rocm 3.6.0. +# Done here, before AITER / sgl-kernel, so those extensions build against torch 2.11's ABI. +RUN case "${GPU_ARCH}" in \ + *-rocm724) \ + python3 -m pip install --no-cache-dir --index-url "${TORCH_ROCM_INDEX_URL}" \ + "torch==${TORCH_ROCM_VERSION}" \ + "torchvision==${TORCHVISION_ROCM_VERSION}" \ + "torchaudio==${TORCHAUDIO_ROCM_VERSION}" \ + ;; \ + *) \ + echo "Not a ROCm 7.2.4 flavor (GPU_ARCH=${GPU_ARCH}), keep base torch/triton"; \ + ;; \ + esac + +# Populate the PIP_CONSTRAINT file, which only the rocm724 stages define, so that +# resolving AITER and SGLang dependencies cannot replace the torch stack above. +# Triton is left out: the BUILD_TRITON step installs it later. +RUN case "${GPU_ARCH}" in \ + *-rocm724) \ + python3 -m pip freeze \ + | grep -E '^(torch|torchvision|torchaudio)(==| @ )' \ + > /etc/sglang/constraints/torch-rocm.txt \ + && cat /etc/sglang/constraints/torch-rocm.txt \ + ;; \ + esac + WORKDIR /sgl-workspace # ----------------------- @@ -220,7 +346,7 @@ RUN if [ "$BUILD_LLVM" = "1" ]; then \ ENV SETUPTOOLS_SCM_PRETEND_VERSION= # Compile AITER against the base image's Triton; the Triton step at the end of -# this file swaps in AITER's own pin afterwards. +# this file installs the pinned one afterwards. ENV AITER_USE_SYSTEM_TRITON=1 RUN pip uninstall -y aiter # Use `checkout -f` so the smudge-filter-induced "dirty" working tree from @@ -248,6 +374,28 @@ RUN cd aiter \ fi \ && echo "export PYTHONPATH=/sgl-workspace/aiter:\${PYTHONPATH}" >> /etc/bash.bashrc +# torch 2.11 Dynamo may pass a base torch.Stream; drop after ROCm/aiter#4817. +RUN python3 <<'PY' +from pathlib import Path +p = Path("/sgl-workspace/aiter/csrc/cpp_itfs/torch_utils.py") +s = p.read_text() +old = """ elif isinstance(arg, torch.cuda.Stream): + c_args.append(ctypes.cast(arg.cuda_stream, ctypes.c_void_p)) +""" +new = """ elif isinstance(arg, torch.Stream): + handle = getattr(arg, "cuda_stream", None) + if handle is None: + handle = torch.cuda.Stream( + stream_id=arg.stream_id, + device_index=arg.device_index, + device_type=arg.device_type, + ).cuda_stream + c_args.append(ctypes.cast(handle, ctypes.c_void_p)) +""" +if old in s: + p.write_text(s.replace(old, new)) +PY + # ----------------------- # Build Mooncake ENV PATH=$PATH:/usr/local/go/bin @@ -319,14 +467,30 @@ RUN if [ "$BRANCH_TYPE" = "local" ]; then \ && AMDGPU_TARGET=$GPU_ARCH_LIST python setup_rocm.py install \ && cd ../../../.. \ && rm -rf python/pyproject.toml && mv python/pyproject_other.toml python/pyproject.toml \ + # srt_hip pins compressed-tensors==0.15.0, which requires torch<2.11 and so + # cannot be satisfied on the ROCm 7.2.4 torch 2.11 stack. The *_rocm724 extras + # carry a 0.16.0 pin instead; all other flavors keep the extras they used before. + && case "${GPU_ARCH}" in \ + *-rocm724) srt_extras="srt_hip_rocm724,diffusion_hip"; all_extras="all_hip_rocm724" ;; \ + *) srt_extras="srt_hip,diffusion_hip"; all_extras="all_hip" ;; \ + esac \ && if [ "$BUILD_TYPE" = "srt" ]; then \ - export SETUPTOOLS_SCM_PRETEND_VERSION="${SETUPTOOLS_SCM_PRETEND_VERSION}" && python -m pip --no-cache-dir install -e "python[srt_hip,diffusion_hip]"; \ + export SETUPTOOLS_SCM_PRETEND_VERSION="${SETUPTOOLS_SCM_PRETEND_VERSION}" && python -m pip --no-cache-dir install -e "python[${srt_extras}]"; \ else \ - export SETUPTOOLS_SCM_PRETEND_VERSION="${SETUPTOOLS_SCM_PRETEND_VERSION}" && python -m pip --no-cache-dir install -e "python[all_hip]"; \ + export SETUPTOOLS_SCM_PRETEND_VERSION="${SETUPTOOLS_SCM_PRETEND_VERSION}" && python -m pip --no-cache-dir install -e "python[${all_extras}]"; \ fi RUN python -m pip cache purge +RUN if [ "${GPU_ARCH##*-}" = "rocm724" ]; then \ + python3 -m pip check \ + && python3 -c "import torch, torchaudio, torchvision, triton; expected={'torch':'2.11.','torchaudio':'2.11.','torchvision':'0.26.'}; actual={'torch':torch.__version__,'torchaudio':torchaudio.__version__,'torchvision':torchvision.__version__,'triton':triton.__version__}; assert torch.version.hip, actual; assert all(actual[name].startswith(version) for name, version in expected.items()), actual; print('Validated ROCm stack:', actual, 'HIP', torch.version.hip)" \ + && if pip list --format=freeze | grep -Eq '^nvidia-.*-cu[0-9]+'; then \ + echo "ERROR: NVIDIA CUDA runtime packages were installed into the ROCm image"; \ + exit 1; \ + fi; \ + fi + # Copy config files to support MI300X in virtualized environments (MI300X_VF). Symlinks will not be created in image build. RUN find /sgl-workspace/sglang/python/sglang/srt/layers/quantization/configs/ \ /sgl-workspace/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/ \ @@ -536,6 +700,13 @@ RUN /bin/bash -lc 'set -euo pipefail; \ apt-get update && apt-get install -y --no-install-recommends \ build-essential autoconf automake libtool pkg-config git \ libibverbs-dev librdmacm-dev rdma-core && rm -rf /var/lib/apt/lists/*; \ + # Mooncake's dependencies.sh apt-installs Ubuntu's libabsl-dev (20220623 on + # the noble base used by rocm724). NIXL's meson then finds absl_base but no + # absl_log and refuses to fall back to its bundled Abseil -- "that would + # result in a mix of Abseil versions at runtime" -- so nixl fails at metadata + # generation. Drop just the -dev package (headers and pkg-config files); the + # runtime library that already-built components link against stays in place. + case "${GPU_ARCH}" in *-rocm724) apt-get remove -y libabsl-dev ;; esac; \ pip install --no-cache-dir meson ninja pybind11 meson-python patchelf pyyaml; \ git clone --depth=1 -b "${UCX_BRANCH}" "${UCX_REPO}" /sgl-workspace/ucx; \ cd /sgl-workspace/ucx && ./autogen.sh && mkdir build && cd build && \ @@ -624,50 +795,30 @@ RUN cd /tmp/whl \ ;; \ esac -# ----------------------- -# Hot patch: transformers dynamic_module_utils symlink bug (v5.12.1). -# _compute_local_source_files_hash calls Path(...).resolve() on custom-code -# module files, following the HF-cache snapshots//x.py -> blobs/ -# symlink. trust_remote_code models whose custom code uses relative imports -# (e.g. Kimi-K2.6's kimi_k25_vision_processing.py: `from .media_utils import`) -# then crash with FileNotFoundError: .../blobs/.py at processor init. -# Mirrors upstream transformers PR #46618 (merged, not yet released): drop the -# .resolve() on the module file and its relative-import sources so the snapshot -# .py names (not the blob targets) are used. Self-skips once transformers ships -# the fix; fails the build loudly if the pattern is present but unpatched. -RUN python3 - <<'PY' -import pathlib -import transformers.dynamic_module_utils as m - -MARKS = ["Path(resolved_module_file).resolve()", "Path(source_file).resolve()"] -path = pathlib.Path(m.__file__) -src = path.read_text() -if not any(mark in src for mark in MARKS): - print("transformers dynamic_module_utils already fixed; no patch needed") -else: - patched = ( - src.replace("Path(resolved_module_file).resolve()", "Path(resolved_module_file)") - .replace("Path(source_file).resolve()", "Path(source_file)") - ) - assert patched != src, "FATAL: transformers symlink patch matched nothing" - path.write_text(patched) - print("patched transformers dynamic_module_utils.py (symlink hash fix)") -PY +# transformers 5.12.1: don't follow HF-cache symlinks when hashing custom modules +# (transformers#46618, not yet released). +RUN python3 -c "from pathlib import Path; import transformers.dynamic_module_utils as m; p=Path(m.__file__); t=p.read_text(); p.write_text(t.replace('Path(resolved_module_file).resolve()','Path(resolved_module_file)').replace('Path(source_file).resolve()','Path(source_file)'))" # ----------------------- -# Install the Triton AITER pins, replacing the base image's. No version check -# on purpose: the pin is AITER's to move, and its installer enforces a floor. +# Install AMD's ROCm Triton, replacing the base image's. The local version is +# part of the pin: `==3.7.0` alone would accept any revision the index later +# publishes under that number, and pip would choose between them by lexical +# order of the git hash rather than by date. # # Keep this last. Base ROCm Torch pins triton==3.5.1 and the torch patch above # is what drops that pin, so installing Triton any earlier lets the next pip # install pull CUDA torch instead. The hip check below is the tripwire. +# torch 2.11 names this `triton-rocm`; uninstall it so the pin is the only Triton. RUN if [ "$BUILD_TRITON" = "1" ]; then \ - cd /sgl-workspace/aiter \ - && test -f .github/scripts/install_triton.sh \ - && PIP_NO_CACHE_DIR=1 bash .github/scripts/install_triton.sh \ + pip uninstall -y triton-rocm || true \ + && PIP_NO_CACHE_DIR=1 pip install --extra-index-url ${TRITON_INDEX_URL} \ + "triton==${TRITON_VERSION}" "triton-kernels==${TRITON_KERNELS_VERSION}" \ && python3 -c "import torch; from importlib.metadata import version; v = version('triton'); k = version('triton-kernels'); assert torch.version.hip is not None, torch.__version__; print(f'[Triton] ROCm Torch {torch.__version__}, Triton {v}, triton-kernels {k}')"; \ fi +# torch 2.11 still Requires-Dist: triton-rocm after the swap above. +RUN case "${GPU_ARCH}" in *-rocm724) python3 -c "import pathlib,re,importlib.metadata as m; p=pathlib.Path(m.distribution('torch')._path)/'METADATA'; v=m.version('triton'); t,n=re.subn(r'^Requires-Dist: (?:triton|triton-rocm)==[^ ;]+', 'Requires-Dist: triton=='+v, p.read_text(), count=1, flags=re.M); assert n==1, n; p.write_text(t)" ;; esac + # ----------------------- # Performance environment variable. diff --git a/python/pyproject_other.toml b/python/pyproject_other.toml index abd14689af00..e4952cecdbd1 100755 --- a/python/pyproject_other.toml +++ b/python/pyproject_other.toml @@ -107,6 +107,10 @@ tracing = [ # HIP (Heterogeneous-computing Interface for Portability) for AMD # => base docker rocm/vllm-dev:20250114, not from public vllm whl +# +# srt_hip_rocm724 below is the torch 2.11 variant of this list and differs only +# in its compressed-tensors and torch pins. A dependency added here has to be +# added there too; nothing enforces that. srt_hip = [ # Pin to 0.15.0: 0.16.0 needs torch>=2.10 (incompatible with ROCm torch # 2.9.1). An open-ended `<0.16.0` made pip backtrack into an unbuildable @@ -118,6 +122,17 @@ srt_hip = [ "wave-lang==3.8.2", ] +# Kept in step with srt_hip above by hand; only the two pins below differ. +srt_hip_rocm724 = [ + # ROCm 7.2.4 uses torch 2.11. compressed-tensors 0.15.0 requires + # torch<2.11, while 0.16.0 supports torch>=2.10. + "compressed-tensors==0.16.0", + "petit_kernel==0.0.2", + "sglang[runtime_common]", + "torch==2.11.0", + "wave-lang==3.8.2", +] + diffusion_hip = [ "cache-dit==1.3.0", "peft>=0.18.0,<0.19.0", # Pin to <0.19.0 due to torchao incompatibility @@ -189,11 +204,13 @@ test = [ ] all_hip = ["sglang[diffusion_hip]", "sglang[srt_hip]", "sglang[tracing]"] +all_hip_rocm724 = ["sglang[diffusion_hip]", "sglang[srt_hip_rocm724]", "sglang[tracing]"] all_hpu = ["sglang[srt_hpu]"] all_musa = ["sglang[diffusion_musa]", "sglang[srt_musa]"] all_mps = ["sglang[diffusion_mps]", "sglang[srt_mps]"] dev_hip = ["sglang[all_hip]", "sglang[test]"] +dev_hip_rocm724 = ["sglang[all_hip_rocm724]", "sglang[test]"] dev_hpu = ["sglang[all_hpu]", "sglang[test]"] dev_musa = ["sglang[all_musa]", "sglang[test]"] dev_mps = ["sglang[all_mps]", "sglang[test]"] diff --git a/scripts/ci/amd/amd_ci_install_dependency.sh b/scripts/ci/amd/amd_ci_install_dependency.sh index 999b037aa225..b5ad5c231c81 100755 --- a/scripts/ci/amd/amd_ci_install_dependency.sh +++ b/scripts/ci/amd/amd_ci_install_dependency.sh @@ -30,7 +30,6 @@ EXTRAS="dev_hip,tracing" if [ -n "$OPTIONAL_DEPS" ]; then EXTRAS="dev_hip,tracing,${OPTIONAL_DEPS}" fi -echo "Installing python extras: [${EXTRAS}]" # Host names look like: linux-mi35x-gpu-1-xxxxx-runner-zzzzz if [[ "${HOSTNAME_VALUE}" =~ ^linux-(mi[0-9]+[a-z]*)-gpu-[0-9]+ ]]; then @@ -40,7 +39,56 @@ else echo "Warning: could not parse GPU architecture from '${HOSTNAME_VALUE}', defaulting to ${GPU_ARCH}" fi +# Identify the Dockerfile stage that built this image. Both the python extras +# below and the AITER pin lookup further down need the flavor, and they have to +# agree, so detect it once here. +# +# Prefer GPU_ARCH stamped into the image (gfx950-rocm724, gfx942, ...). +# Images built before that ENV existed: 724 stages already set +# PIP_CONSTRAINT and HSA_ENABLE_IPC_MODE_LEGACY; remaining HIP 7.2* is +# 720; else 7.0. Do not key off torch 2.11 — 720 may ship that later. +IMAGE_TORCH_VERSION=$(docker exec ci_sglang python3 -c 'import torch; print(torch.__version__)') +IMAGE_HIP_VERSION=$(docker exec ci_sglang python3 -c 'import torch; print(torch.version.hip or "")') +IMAGE_GPU_ARCH=$(docker exec ci_sglang printenv GPU_ARCH 2>/dev/null || true) +if [[ "${IMAGE_GPU_ARCH}" =~ ^(gfx942|gfx950)(-rocm720|-rocm724)?$ ]]; then + echo "[CI-IMAGE] Image GPU_ARCH=${IMAGE_GPU_ARCH}" + case "${IMAGE_GPU_ARCH}" in + *-rocm724) IMAGE_BASE_ARG_SUFFIX="_ROCM724"; IMAGE_STAGE_SUFFIX="-rocm724" ;; + *-rocm720) IMAGE_BASE_ARG_SUFFIX="_ROCM720"; IMAGE_STAGE_SUFFIX="-rocm720" ;; + *) IMAGE_BASE_ARG_SUFFIX=""; IMAGE_STAGE_SUFFIX="" ;; + esac + IMAGE_GFX="${IMAGE_GPU_ARCH%-*}" +else + IMAGE_PIP_CONSTRAINT=$(docker exec ci_sglang printenv PIP_CONSTRAINT 2>/dev/null || true) + IMAGE_HSA_LEGACY=$(docker exec ci_sglang printenv HSA_ENABLE_IPC_MODE_LEGACY 2>/dev/null || true) + if [[ -n "${IMAGE_PIP_CONSTRAINT}" || "${IMAGE_HSA_LEGACY}" == "1" ]]; then + IMAGE_BASE_ARG_SUFFIX="_ROCM724" + IMAGE_STAGE_SUFFIX="-rocm724" + elif [[ "${IMAGE_HIP_VERSION}" == 7.2* ]]; then + IMAGE_BASE_ARG_SUFFIX="_ROCM720" + IMAGE_STAGE_SUFFIX="-rocm720" + else + IMAGE_BASE_ARG_SUFFIX="" + IMAGE_STAGE_SUFFIX="" + fi + if [[ "${GPU_ARCH}" == "mi35x" ]]; then + IMAGE_GFX="gfx950" + else + IMAGE_GFX="gfx942" + fi + echo "[CI-IMAGE] Image has no GPU_ARCH stamp; inferred ${IMAGE_GFX}${IMAGE_STAGE_SUFFIX} (PIP_CONSTRAINT='${IMAGE_PIP_CONSTRAINT}', HSA_ENABLE_IPC_MODE_LEGACY='${IMAGE_HSA_LEGACY}', HIP=${IMAGE_HIP_VERSION})" + unset IMAGE_PIP_CONSTRAINT IMAGE_HSA_LEGACY +fi +unset IMAGE_GPU_ARCH + # Install the required dependencies in CI. +# ROCm 7.2.4 images ship torch 2.11, which srt_hip cannot satisfy (it pins +# compressed-tensors 0.15.0, requiring torch<2.11). Select the rocm724 extras. +if [[ "${IMAGE_STAGE_SUFFIX}" == "-rocm724" ]]; then + EXTRAS="${EXTRAS/dev_hip/dev_hip_rocm724}" +fi +echo "Image torch ${IMAGE_TORCH_VERSION}, HIP ${IMAGE_HIP_VERSION}; installing python extras: [${EXTRAS}]" + # Fix permissions on pip cache, ignore errors from concurrent access or missing temp files docker exec ci_sglang chown -R root:root /sgl-data/pip-cache 2>/dev/null || true docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache --upgrade pip @@ -221,6 +269,8 @@ if docker exec ci_sglang test -d /sgl-workspace/mori; then cd /sgl-workspace/mori git checkout '${MORI_COMMIT}' git submodule update --init --recursive + apt-get update + apt-get install -y --no-install-recommends libgrpc++-dev 2>/dev/null || true python3 setup.py develop python3 -c 'import os, torch; print(os.path.join(os.path.dirname(torch.__file__), \"lib\"))' > /etc/ld.so.conf.d/torch.conf ldconfig @@ -248,35 +298,23 @@ DOCKERFILE="docker/rocm.Dockerfile" GPU_ARCH="${GPU_ARCH:-mi30x}" echo "[CI-AITER-CHECK] Runner GPU_ARCH=${GPU_ARCH}" -# ROCm 7.0 keeps the Triton its base image ships; later ROCm images run on the -# Triton AITER pins, so a rebuilt AITER has to bring its own along. -IMAGE_HIP_VERSION=$(docker exec ci_sglang python3 -c 'import torch; print(torch.version.hip or "")') -case "${IMAGE_HIP_VERSION}" in - 7.0*) INSTALL_AITER_TRITON="false" ;; - 7.*) INSTALL_AITER_TRITON="true" ;; - *) - echo "[CI-AITER-CHECK] ERROR: Unsupported or empty HIP version: '${IMAGE_HIP_VERSION}'" - exit 1 - ;; -esac -echo "[CI-AITER-CHECK] Container HIP=${IMAGE_HIP_VERSION}, install AITER's Triton on rebuild=${INSTALL_AITER_TRITON}" +# Image owns Triton (pinned in docker/rocm.Dockerfile). Rebuild AITER against it. ############################################# -# 1. Extract AITER_COMMIT from correct Dockerfile block +# 1. Extract AITER_COMMIT from the Dockerfile stage that built this image, as +# identified near the top of this script. ############################################# -if [[ "${GPU_ARCH}" == "mi35x" ]]; then - echo "[CI-AITER-CHECK] Using gfx950 block from Dockerfile..." - REPO_AITER_COMMIT=$(grep -F -A20 'FROM $BASE_IMAGE_950 AS gfx950' docker/rocm.Dockerfile \ - | grep 'AITER_COMMIT_DEFAULT=' \ - | head -n1 \ - | sed 's/.*AITER_COMMIT_DEFAULT="\([^"]*\)".*/\1/') +if [[ "${IMAGE_GFX}" == "gfx950" ]]; then + _from_line="FROM \$BASE_IMAGE_950${IMAGE_BASE_ARG_SUFFIX} AS gfx950${IMAGE_STAGE_SUFFIX}" else - echo "[CI-AITER-CHECK] Using gfx942 block from Dockerfile..." - REPO_AITER_COMMIT=$(grep -F -A20 'FROM $BASE_IMAGE_942 AS gfx942' docker/rocm.Dockerfile \ - | grep 'AITER_COMMIT_DEFAULT=' \ - | head -n1 \ - | sed 's/.*AITER_COMMIT_DEFAULT="\([^"]*\)".*/\1/') + _from_line="FROM \$BASE_IMAGE_942${IMAGE_BASE_ARG_SUFFIX} AS gfx942${IMAGE_STAGE_SUFFIX}" fi +echo "[CI-AITER-CHECK] Using ${_from_line} from Dockerfile..." +REPO_AITER_COMMIT=$(grep -F -A20 "${_from_line}" docker/rocm.Dockerfile \ + | grep 'AITER_COMMIT_DEFAULT=' \ + | head -n1 \ + | sed 's/.*AITER_COMMIT_DEFAULT="\([^"]*\)".*/\1/') +unset _from_line if [[ -z "${REPO_AITER_COMMIT}" ]]; then @@ -344,6 +382,30 @@ if [[ "${NEED_REBUILD}" == "true" ]]; then pip install -r requirements.txt " + # Re-apply the Dockerfile torch.Stream patch after re-clone (ROCm/aiter#4817). + if [[ "${IMAGE_STAGE_SUFFIX}" == "-rocm724" ]]; then + docker exec -i ci_sglang python3 - <<'PY' +from pathlib import Path +p = Path("/sgl-workspace/aiter/csrc/cpp_itfs/torch_utils.py") +s = p.read_text() +old = """ elif isinstance(arg, torch.cuda.Stream): + c_args.append(ctypes.cast(arg.cuda_stream, ctypes.c_void_p)) +""" +new = """ elif isinstance(arg, torch.Stream): + handle = getattr(arg, "cuda_stream", None) + if handle is None: + handle = torch.cuda.Stream( + stream_id=arg.stream_id, + device_index=arg.device_index, + device_type=arg.device_type, + ).cuda_stream + c_args.append(ctypes.cast(handle, ctypes.c_void_p)) +""" +if old in s: + p.write_text(s.replace(old, new)) +PY + fi + if [[ "${GPU_ARCH}" == "mi35x" ]]; then GPU_ARCH_LIST="gfx950" else @@ -351,19 +413,7 @@ if [[ "${NEED_REBUILD}" == "true" ]]; then fi echo "[CI-AITER-CHECK] GPU_ARCH_LIST=${GPU_ARCH_LIST}" - # Run the installer here rather than letting setup.py do it: setup.py - # swallows its errors, and the AITER_USE_SYSTEM_TRITON=1 below then keeps - # whatever Triton is already installed. Doing it up front fails closed. - if [[ "${INSTALL_AITER_TRITON}" == "true" ]]; then - docker exec ci_sglang bash -c " - set -euo pipefail - cd /sgl-workspace/aiter - test -f .github/scripts/install_triton.sh - bash .github/scripts/install_triton.sh - " - fi - - # build AITER + # The image already has the Dockerfile-pinned Triton; compile against it. docker exec ci_sglang bash -c " cd /sgl-workspace/aiter && \ AITER_USE_SYSTEM_TRITON=1 GPU_ARCHS=${GPU_ARCH_LIST} python3 setup.py develop diff --git a/scripts/ci/amd/amd_ci_start_container.sh b/scripts/ci/amd/amd_ci_start_container.sh index 0715e325964a..3eee603da50d 100755 --- a/scripts/ci/amd/amd_ci_start_container.sh +++ b/scripts/ci/amd/amd_ci_start_container.sh @@ -28,7 +28,7 @@ LOCAL_DOCKER_REGISTRY="10.44.14.109:5000" # Parse command line arguments MI30X_BASE_TAG="${DEFAULT_MI30X_BASE_TAG}" MI35X_BASE_TAG="${DEFAULT_MI35X_BASE_TAG}" -CUSTOM_IMAGE="" +CUSTOM_IMAGE="${AMD_CI_IMAGE:-}" BUILD_FROM_DOCKERFILE="" GPU_ARCH_BUILD="" @@ -50,7 +50,7 @@ while [[ $# -gt 0 ]]; do echo "Options:" echo " --mi30x-base-tag TAG Override MI30x base image tag" echo " --mi35x-base-tag TAG Override MI35x base image tag" - echo " --custom-image IMAGE Use a specific Docker image directly" + echo " --custom-image IMAGE Use a specific Docker image directly (or set AMD_CI_IMAGE)" echo " --build-from-dockerfile Build image from docker/rocm.Dockerfile" echo " --gpu-arch ARCH GPU architecture for Dockerfile build (e.g., gfx950-rocm720)" echo " --rocm-version VERSION Override ROCm version for image lookup (e.g., rocm720)" diff --git a/scripts/ci/amd/amd_ci_start_container_disagg.sh b/scripts/ci/amd/amd_ci_start_container_disagg.sh index 0c6e8a1dbdac..079015076fe5 100755 --- a/scripts/ci/amd/amd_ci_start_container_disagg.sh +++ b/scripts/ci/amd/amd_ci_start_container_disagg.sh @@ -28,11 +28,13 @@ LOCAL_DOCKER_REGISTRY="10.44.14.109:5000" # Parse command line arguments MI30X_BASE_TAG="${DEFAULT_MI30X_BASE_TAG}" MI35X_BASE_TAG="${DEFAULT_MI35X_BASE_TAG}" +CUSTOM_IMAGE="${AMD_CI_IMAGE:-}" while [[ $# -gt 0 ]]; do case $1 in --mi30x-base-tag) MI30X_BASE_TAG="$2"; shift 2;; --mi35x-base-tag) MI35X_BASE_TAG="$2"; shift 2;; + --custom-image) CUSTOM_IMAGE="$2"; shift 2;; --rocm-version) ROCM_VERSION="$2" MI30X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi30x" @@ -40,7 +42,7 @@ while [[ $# -gt 0 ]]; do echo "Using ROCm version override: ${ROCM_VERSION}" shift 2;; -h|--help) - echo "Usage: $0 [--mi30x-base-tag TAG] [--mi35x-base-tag TAG] [--rocm-version VERSION]" + echo "Usage: $0 [--mi30x-base-tag TAG] [--mi35x-base-tag TAG] [--custom-image IMAGE] [--rocm-version VERSION]" exit 0 ;; *) echo "Unknown option $1"; exit 1;; @@ -212,20 +214,30 @@ find_latest_image() { esac } -# Pull and run the latest image -IMAGE=$(find_latest_image "${GPU_ARCH}") -# Try the local docker registry first (avoids Docker Hub rate limits and is -# faster on the LAN); if that fails for any reason, fall back to the -# public registry with exponential-backoff retries. Capture stderr so the -# real failure reason (TLS handshake, 404, connection refused, etc.) is -# visible in the job log instead of being silently swallowed. -if local_pull_output=$(docker pull "${LOCAL_DOCKER_REGISTRY}/${IMAGE}" 2>&1); then - echo "Pulled from local docker registry: ${LOCAL_DOCKER_REGISTRY}/${IMAGE}" - docker tag "${LOCAL_DOCKER_REGISTRY}/${IMAGE}" "${IMAGE}" +# Determine which image to use +if [[ -n "${CUSTOM_IMAGE}" ]]; then + IMAGE="${CUSTOM_IMAGE}" + echo "Using custom image: ${IMAGE}" + if [[ "${IMAGE}" == "${LOCAL_DOCKER_REGISTRY}/"* ]]; then + docker pull "${IMAGE}" + else + retry_with_backoff 6 docker pull "${IMAGE}" + fi else - echo "Local docker registry pull failed; falling back to public registry: ${IMAGE}" >&2 - printf '%s\n' "${local_pull_output}" | sed 's/^/ [local-pull] /' >&2 - retry_with_backoff 6 docker pull "${IMAGE}" + IMAGE=$(find_latest_image "${GPU_ARCH}") + # Try the local docker registry first (avoids Docker Hub rate limits and is + # faster on the LAN); if that fails for any reason, fall back to the + # public registry with exponential-backoff retries. Capture stderr so the + # real failure reason (TLS handshake, 404, connection refused, etc.) is + # visible in the job log instead of being silently swallowed. + if local_pull_output=$(docker pull "${LOCAL_DOCKER_REGISTRY}/${IMAGE}" 2>&1); then + echo "Pulled from local docker registry: ${LOCAL_DOCKER_REGISTRY}/${IMAGE}" + docker tag "${LOCAL_DOCKER_REGISTRY}/${IMAGE}" "${IMAGE}" + else + echo "Local docker registry pull failed; falling back to public registry: ${IMAGE}" >&2 + printf '%s\n' "${local_pull_output}" | sed 's/^/ [local-pull] /' >&2 + retry_with_backoff 6 docker pull "${IMAGE}" + fi fi CACHE_HOST=/home/runner/sglang-data