yctseng0211 · Nov 7, 2026 · Nov 7, 2026
@@ -39,7 +39,8 @@ concurrency:
 
 jobs:
   publish:
-    if: github.repository == 'sgl-project/sglang' && (github.event_name != 'workflow_dispatch' || inputs.job_select == 'all' || inputs.job_select == 'publish')
+    # TEMPORARY: only the gfx1250 ROCm 10.1 test image is built.
+    if: false
     runs-on: amd-docker-scale
     environment: 'prod'
     strategy:
@@ -179,7 +180,8 @@ jobs:
           PRETEND_VERSION: ${{ steps.version.outputs.pretend_version }}
           VERSION: ${{ steps.version.outputs.version }}
         run: |
-          image_tag="v${VERSION}-rocm101-mi45x-${DATE}"
+          # TEMPORARY: test tag. Remove before merge.
+          image_tag="v${VERSION}-rocm101-mi45x-${DATE}_8066cd93b__test"
           echo "IMAGE_TAG=${image_tag}" >> "$GITHUB_ENV"
 
           docker build . \
@@ -199,24 +201,14 @@ jobs:
             echo "Build-only run: not pushing rocm/sgl-dev:${image_tag}"
           fi
 
-      - name: Login to Docker Hub (lmsys)
-        if: env.PUSH_IMAGES == 'true'
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Push to lmsysorg/sglang-rocm
-        if: env.PUSH_IMAGES == 'true'
-        run: |
-          docker tag "rocm/sgl-dev:${IMAGE_TAG}" "lmsysorg/sglang-rocm:${IMAGE_TAG}"
-          docker push "lmsysorg/sglang-rocm:${IMAGE_TAG}"
+      # TEMPORARY: no lmsysorg/sglang-rocm push for the test image.
 
   # gfx1250 (MI45x) runs on the same schedule as the pair above but stays a
   # separate job: no MI450 runner pool consumes the image, so a bring-up build
   # failure must not gate the released gfx942/gfx950 images.
   publish-gfx1250:
-    if: github.repository == 'sgl-project/sglang' && (github.event_name != 'workflow_dispatch' || inputs.job_select == 'all' || inputs.job_select == 'publish-gfx1250')
+    # TEMPORARY: only the gfx1250 ROCm 10.1 test image is built.
+    if: false
     runs-on: amd-docker-scale
     environment: 'prod'
     steps:

@@ -23,7 +23,8 @@
 #   GPU_ARCH=*-rocm1010 is ROCm 10.1 + torch 2.12 off the same channel, and
 #   keeps the SDK's Triton instead of building one.
 #   GPU_ARCH=gfx1250-rocm1010 also replaces the SDK's RCCL with a newer build
-#   from rocm-systems; see BUILD_RCCL.
+#   from rocm-systems (see BUILD_RCCL), and its ROCr and rocprofiler-sdk with
+#   patched builds (see BUILD_ROCPROFILER).
 
 # Usage (to build SGLang ROCm + Mori docker image):
 # remove --build-arg NIC_BACKEND=ainic since new MoRI JIT will do NIC auto detection on target
@@ -385,6 +386,11 @@ ENV BUILD_RCCL="1"
 # (ROCm/rocm-systems#14254) has landed on the mainline, so pin develop instead
 # of the PR branch.
 ENV RCCL_COMMIT_DEFAULT="d2e0de357464cd34cabaea859b71c10da88e7ae9"
+# torch.profiler hangs on TP workloads in the queue interposition of the 10.1
+# rocprofiler-sdk ("Async signal handler still waiting on signal"). The fix
+# (ROCm/rocm-systems#9984) needs ROCr's hsa_amd_signal_wait_any fix (#8564), and
+# 10.1 has neither.
+ENV BUILD_ROCPROFILER="1"
 # hipBLASLt 10.1 looks for TensileLibrary_lazy_gfx1250.dat directly under
 # library/, not library/<arch>/, and every GEMM fails without this.
 ENV HIPBLASLT_TENSILE_LIBPATH="/opt/venv/lib/python3.12/site-packages/_rocm_sdk_libraries/lib/hipblaslt/library/gfx1250"
@@ -459,6 +465,11 @@ ARG RCCL_REPO="https://github.com/ROCm/rocm-systems.git"
 ARG RCCL_COMMIT=""
 ENV RCCL_COMMIT="${RCCL_COMMIT:-${RCCL_COMMIT_DEFAULT}}"
 
+ARG ROCPROFILER_REPO="https://github.com/ROCm/rocm-systems.git"
+# Cherry-picked in order onto the rocm-systems commit the SDK was built from:
+# ROCr #8564, then rocprofiler-sdk #9984 and #12957.
+ARG ROCPROFILER_CHERRY_PICKS="80696bf14d3ce2acf4fa5480488be4b6dba8d73c 6cdcc6a08813ca7940f0ba5b5fb48ae70116b712 93bd837e4d641b8243491331e25df2ff6f61c371"
+
 ARG LLVM_REPO="https://github.com/jrbyrnes/llvm-project.git"
 ARG LLVM_BRANCH="MainOpSelV2"
 ARG LLVM_COMMIT="6520ace8227ffe2728148d5f3b9872a870b0a560"
@@ -1163,6 +1174,59 @@ RUN if [ "${BUILD_RCCL:-0}" = "1" ]; then \
         python3 -c "import ctypes, torch; v = ctypes.c_int(); ctypes.CDLL('librccl.so.1').ncclGetVersion(ctypes.byref(v)); maps = sorted({l.split()[-1] for l in open('/proc/self/maps') if 'librccl' in l}); print('[RCCL] ncclGetVersion', v.value, maps); assert len(maps) == 1, maps"; \
     fi
 
+# -----------------------
+# ROCr + rocprofiler-sdk (BUILD_ROCPROFILER=1 stages only): rebuild both from
+# the rocm-systems commit the SDK was built from (TheRock manifest) with
+# ROCPROFILER_CHERRY_PICKS applied, and overwrite the SDK's libhsa-runtime64.so.1
+# and librocprofiler-sdk.so.1. ROCr links against the SDK's own sysdeps (elf,
+# drm, numa), and both files keep the stock RPATH, as in the RCCL block.
+RUN if [ "${BUILD_ROCPROFILER:-0}" = "1" ]; then \
+        set -eux; \
+        apt-get update; \
+        apt-get install -y --no-install-recommends xxd libdw-dev libelf-dev; \
+        rm -rf /var/lib/apt/lists/*; \
+        pip install --no-cache-dir patchelf; \
+        SP="$VIRTUAL_ENV/lib/python3.12/site-packages"; \
+        SYSDEPS="${ROCM_HOME}/lib/rocm_sysdeps"; \
+        PIN="$(python3 -c "import json; m = json.load(open('$SP/_rocm_sdk_core/share/therock/therock_manifest.json')); print(next(s['pin_sha'] for s in m['submodules'] if s['submodule_name'] == 'rocm-systems'))")"; \
+        echo "[ROCPROFILER] rocm-systems ${PIN} + ${ROCPROFILER_CHERRY_PICKS}"; \
+        mkdir -p /sgl-workspace/rocm-systems && cd /sgl-workspace/rocm-systems; \
+        git init -q; \
+        git remote add origin "${ROCPROFILER_REPO}"; \
+        git sparse-checkout init --cone; \
+        git sparse-checkout set projects/rocr-runtime projects/rocprofiler-sdk; \
+        git fetch --depth 1 --filter=blob:none origin "${PIN}"; \
+        git checkout -q FETCH_HEAD; \
+        # depth 2 also fetches the parent of each commit, which cherry-pick diffs against
+        for c in ${ROCPROFILER_CHERRY_PICKS}; do git fetch --depth 2 --filter=blob:none origin "$c"; done; \
+        git -c user.name=sglang -c user.email=sglang@localhost cherry-pick ${ROCPROFILER_CHERRY_PICKS}; \
+        git submodule update --init --depth 1 --jobs 8 -- $(git config --file .gitmodules --get-regexp path \
+            | awk '{print $2}' | grep '^projects/rocprofiler-sdk/external/' | grep -vE 'googletest|doxygen'); \
+        PKG_CONFIG_PATH="${SYSDEPS}/lib/pkgconfig" cmake -S projects/rocr-runtime -B /tmp/rocr-build -G Ninja \
+            -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="${ROCM_HOME};${SYSDEPS}" \
+            -DLIBELF_INCLUDE_DIRS="${SYSDEPS}/include" -DLIBELF_LIBRARIES="${SYSDEPS}/lib/libelf.so"; \
+        cmake --build /tmp/rocr-build --parallel "$(nproc)"; \
+        cmake -S projects/rocprofiler-sdk -B /tmp/rocprofiler-sdk-build -G Ninja -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_PREFIX_PATH="${ROCM_HOME}" -DROCPROFILER_BUILD_TESTS=OFF \
+            -DROCPROFILER_BUILD_SAMPLES=OFF -DROCPROFILER_BUILD_DOCS=OFF; \
+        cmake --build /tmp/rocprofiler-sdk-build --parallel "$(nproc)"; \
+        for spec in libhsa-runtime64:/tmp/rocr-build librocprofiler-sdk:/tmp/rocprofiler-sdk-build; do \
+            lib="${spec%%:*}"; build="${spec#*:}"; \
+            cp -L "$(find "$build" -name "$lib.so.1" -path '*/lib/*' | head -1)" "/tmp/$lib.so.1"; \
+            patchelf --force-rpath --set-rpath "$(patchelf --print-rpath "$SP/_rocm_sdk_core/lib/$lib.so.1")" "/tmp/$lib.so.1"; \
+            for d in "$SP/_rocm_sdk_core/lib" "$SP/_rocm_sdk_devel/lib"; do \
+                # _rocm_sdk_devel also ships $lib.so and $lib.so.1.<version> as hard links to the stock file
+                names="$(cd "$d" && ls "$lib".so*)"; \
+                rm -f "$d/$lib".so*; \
+                cp "/tmp/$lib.so.1" "$d/$lib.so.1"; \
+                for n in $names; do [ "$n" = "$lib.so.1" ] || ln -sf "$lib.so.1" "$d/$n"; done; \
+            done; \
+        done; \
+        cd / && rm -rf /sgl-workspace/rocm-systems /tmp/rocr-build /tmp/rocprofiler-sdk-build \
+            /tmp/libhsa-runtime64.so.1 /tmp/librocprofiler-sdk.so.1; \
+        python3 -c "import ctypes, torch; [ctypes.CDLL(n) for n in ('libhsa-runtime64.so.1', 'librocprofiler-sdk.so.1')]; maps = sorted({l.split()[-1] for l in open('/proc/self/maps') if 'libhsa-runtime64' in l or 'librocprofiler-sdk.so' in l}); print('[ROCPROFILER]', maps); assert len(maps) == 2, maps"; \
+    fi
+
 # -----------------------
 # Hot patch: torch-ROCm
 # The artifact hardcoded the supported triton version to be 3.5.1.