diff --git a/.github/workflows/release-docker-amd-rocm10.yml b/.github/workflows/release-docker-amd-rocm10.yml index 7e3038efb132..4dc1bd442cdc 100644 --- a/.github/workflows/release-docker-amd-rocm10.yml +++ b/.github/workflows/release-docker-amd-rocm10.yml @@ -39,7 +39,8 @@ concurrency: jobs: publish: - if: github.repository == 'sgl-project/sglang' && (github.event_name != 'workflow_dispatch' || inputs.job_select == 'all' || inputs.job_select == 'publish') + # TEMPORARY: only the gfx1250 ROCm 10.1 test image is built. + if: false runs-on: amd-docker-scale environment: 'prod' strategy: @@ -179,7 +180,8 @@ jobs: PRETEND_VERSION: ${{ steps.version.outputs.pretend_version }} VERSION: ${{ steps.version.outputs.version }} run: | - image_tag="v${VERSION}-rocm101-mi45x-${DATE}" + # TEMPORARY: test tag. Remove before merge. + image_tag="v${VERSION}-rocm101-mi45x-${DATE}_8066cd93b__test" echo "IMAGE_TAG=${image_tag}" >> "$GITHUB_ENV" docker build . \ @@ -199,24 +201,14 @@ jobs: echo "Build-only run: not pushing rocm/sgl-dev:${image_tag}" fi - - name: Login to Docker Hub (lmsys) - if: env.PUSH_IMAGES == 'true' - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Push to lmsysorg/sglang-rocm - if: env.PUSH_IMAGES == 'true' - run: | - docker tag "rocm/sgl-dev:${IMAGE_TAG}" "lmsysorg/sglang-rocm:${IMAGE_TAG}" - docker push "lmsysorg/sglang-rocm:${IMAGE_TAG}" + # TEMPORARY: no lmsysorg/sglang-rocm push for the test image. # gfx1250 (MI45x) runs on the same schedule as the pair above but stays a # separate job: no MI450 runner pool consumes the image, so a bring-up build # failure must not gate the released gfx942/gfx950 images. publish-gfx1250: - if: github.repository == 'sgl-project/sglang' && (github.event_name != 'workflow_dispatch' || inputs.job_select == 'all' || inputs.job_select == 'publish-gfx1250') + # TEMPORARY: only the gfx1250 ROCm 10.1 test image is built. + if: false runs-on: amd-docker-scale environment: 'prod' steps: diff --git a/docker/rocm.Dockerfile b/docker/rocm.Dockerfile index 39e1c4830f56..9b6037e5713d 100644 --- a/docker/rocm.Dockerfile +++ b/docker/rocm.Dockerfile @@ -23,7 +23,8 @@ # GPU_ARCH=*-rocm1010 is ROCm 10.1 + torch 2.12 off the same channel, and # keeps the SDK's Triton instead of building one. # GPU_ARCH=gfx1250-rocm1010 also replaces the SDK's RCCL with a newer build -# from rocm-systems; see BUILD_RCCL. +# from rocm-systems (see BUILD_RCCL), and its ROCr and rocprofiler-sdk with +# patched builds (see BUILD_ROCPROFILER). # Usage (to build SGLang ROCm + Mori docker image): # remove --build-arg NIC_BACKEND=ainic since new MoRI JIT will do NIC auto detection on target @@ -385,6 +386,11 @@ ENV BUILD_RCCL="1" # (ROCm/rocm-systems#14254) has landed on the mainline, so pin develop instead # of the PR branch. ENV RCCL_COMMIT_DEFAULT="d2e0de357464cd34cabaea859b71c10da88e7ae9" +# torch.profiler hangs on TP workloads in the queue interposition of the 10.1 +# rocprofiler-sdk ("Async signal handler still waiting on signal"). The fix +# (ROCm/rocm-systems#9984) needs ROCr's hsa_amd_signal_wait_any fix (#8564), and +# 10.1 has neither. +ENV BUILD_ROCPROFILER="1" # hipBLASLt 10.1 looks for TensileLibrary_lazy_gfx1250.dat directly under # library/, not library//, and every GEMM fails without this. ENV HIPBLASLT_TENSILE_LIBPATH="/opt/venv/lib/python3.12/site-packages/_rocm_sdk_libraries/lib/hipblaslt/library/gfx1250" @@ -459,6 +465,11 @@ ARG RCCL_REPO="https://github.com/ROCm/rocm-systems.git" ARG RCCL_COMMIT="" ENV RCCL_COMMIT="${RCCL_COMMIT:-${RCCL_COMMIT_DEFAULT}}" +ARG ROCPROFILER_REPO="https://github.com/ROCm/rocm-systems.git" +# Cherry-picked in order onto the rocm-systems commit the SDK was built from: +# ROCr #8564, then rocprofiler-sdk #9984 and #12957. +ARG ROCPROFILER_CHERRY_PICKS="80696bf14d3ce2acf4fa5480488be4b6dba8d73c 6cdcc6a08813ca7940f0ba5b5fb48ae70116b712 93bd837e4d641b8243491331e25df2ff6f61c371" + ARG LLVM_REPO="https://github.com/jrbyrnes/llvm-project.git" ARG LLVM_BRANCH="MainOpSelV2" ARG LLVM_COMMIT="6520ace8227ffe2728148d5f3b9872a870b0a560" @@ -1163,6 +1174,59 @@ RUN if [ "${BUILD_RCCL:-0}" = "1" ]; then \ python3 -c "import ctypes, torch; v = ctypes.c_int(); ctypes.CDLL('librccl.so.1').ncclGetVersion(ctypes.byref(v)); maps = sorted({l.split()[-1] for l in open('/proc/self/maps') if 'librccl' in l}); print('[RCCL] ncclGetVersion', v.value, maps); assert len(maps) == 1, maps"; \ fi +# ----------------------- +# ROCr + rocprofiler-sdk (BUILD_ROCPROFILER=1 stages only): rebuild both from +# the rocm-systems commit the SDK was built from (TheRock manifest) with +# ROCPROFILER_CHERRY_PICKS applied, and overwrite the SDK's libhsa-runtime64.so.1 +# and librocprofiler-sdk.so.1. ROCr links against the SDK's own sysdeps (elf, +# drm, numa), and both files keep the stock RPATH, as in the RCCL block. +RUN if [ "${BUILD_ROCPROFILER:-0}" = "1" ]; then \ + set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends xxd libdw-dev libelf-dev; \ + rm -rf /var/lib/apt/lists/*; \ + pip install --no-cache-dir patchelf; \ + SP="$VIRTUAL_ENV/lib/python3.12/site-packages"; \ + SYSDEPS="${ROCM_HOME}/lib/rocm_sysdeps"; \ + PIN="$(python3 -c "import json; m = json.load(open('$SP/_rocm_sdk_core/share/therock/therock_manifest.json')); print(next(s['pin_sha'] for s in m['submodules'] if s['submodule_name'] == 'rocm-systems'))")"; \ + echo "[ROCPROFILER] rocm-systems ${PIN} + ${ROCPROFILER_CHERRY_PICKS}"; \ + mkdir -p /sgl-workspace/rocm-systems && cd /sgl-workspace/rocm-systems; \ + git init -q; \ + git remote add origin "${ROCPROFILER_REPO}"; \ + git sparse-checkout init --cone; \ + git sparse-checkout set projects/rocr-runtime projects/rocprofiler-sdk; \ + git fetch --depth 1 --filter=blob:none origin "${PIN}"; \ + git checkout -q FETCH_HEAD; \ + # depth 2 also fetches the parent of each commit, which cherry-pick diffs against + for c in ${ROCPROFILER_CHERRY_PICKS}; do git fetch --depth 2 --filter=blob:none origin "$c"; done; \ + git -c user.name=sglang -c user.email=sglang@localhost cherry-pick ${ROCPROFILER_CHERRY_PICKS}; \ + git submodule update --init --depth 1 --jobs 8 -- $(git config --file .gitmodules --get-regexp path \ + | awk '{print $2}' | grep '^projects/rocprofiler-sdk/external/' | grep -vE 'googletest|doxygen'); \ + PKG_CONFIG_PATH="${SYSDEPS}/lib/pkgconfig" cmake -S projects/rocr-runtime -B /tmp/rocr-build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="${ROCM_HOME};${SYSDEPS}" \ + -DLIBELF_INCLUDE_DIRS="${SYSDEPS}/include" -DLIBELF_LIBRARIES="${SYSDEPS}/lib/libelf.so"; \ + cmake --build /tmp/rocr-build --parallel "$(nproc)"; \ + cmake -S projects/rocprofiler-sdk -B /tmp/rocprofiler-sdk-build -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH="${ROCM_HOME}" -DROCPROFILER_BUILD_TESTS=OFF \ + -DROCPROFILER_BUILD_SAMPLES=OFF -DROCPROFILER_BUILD_DOCS=OFF; \ + cmake --build /tmp/rocprofiler-sdk-build --parallel "$(nproc)"; \ + for spec in libhsa-runtime64:/tmp/rocr-build librocprofiler-sdk:/tmp/rocprofiler-sdk-build; do \ + lib="${spec%%:*}"; build="${spec#*:}"; \ + cp -L "$(find "$build" -name "$lib.so.1" -path '*/lib/*' | head -1)" "/tmp/$lib.so.1"; \ + patchelf --force-rpath --set-rpath "$(patchelf --print-rpath "$SP/_rocm_sdk_core/lib/$lib.so.1")" "/tmp/$lib.so.1"; \ + for d in "$SP/_rocm_sdk_core/lib" "$SP/_rocm_sdk_devel/lib"; do \ + # _rocm_sdk_devel also ships $lib.so and $lib.so.1. as hard links to the stock file + names="$(cd "$d" && ls "$lib".so*)"; \ + rm -f "$d/$lib".so*; \ + cp "/tmp/$lib.so.1" "$d/$lib.so.1"; \ + for n in $names; do [ "$n" = "$lib.so.1" ] || ln -sf "$lib.so.1" "$d/$n"; done; \ + done; \ + done; \ + cd / && rm -rf /sgl-workspace/rocm-systems /tmp/rocr-build /tmp/rocprofiler-sdk-build \ + /tmp/libhsa-runtime64.so.1 /tmp/librocprofiler-sdk.so.1; \ + python3 -c "import ctypes, torch; [ctypes.CDLL(n) for n in ('libhsa-runtime64.so.1', 'librocprofiler-sdk.so.1')]; maps = sorted({l.split()[-1] for l in open('/proc/self/maps') if 'libhsa-runtime64' in l or 'librocprofiler-sdk.so' in l}); print('[ROCPROFILER]', maps); assert len(maps) == 2, maps"; \ + fi + # ----------------------- # Hot patch: torch-ROCm # The artifact hardcoded the supported triton version to be 3.5.1.