test(rccl): guard cache bits on LL flag-poll loads (AICOMRCCL-2742)

.github/workflows/therock-rccl-ci-linux.yml

@@ -83,6 +83,8 @@ jobs:
       id-token: write
     container:
       image: ghcr.io/rocm/therock_build_manylinux_x86_64@sha256:cf4f6d9909056906e4a1c0bc4c245658a0af4c6a6174082a86adfa2d518b7aeb # 2026-08-18
+    outputs:
+      ll_cache_bits: ${{ steps.ll_cache_bits.outcome }}
     env:
       AMDGPU_FAMILIES: ${{ inputs.amdgpu_families }}
       TEATIME_FORCE_INTERACTIVE: 0
@@ -158,6 +160,19 @@ jobs:
           ccache -s -v
           tail -v -n +1 .ccache/compiler_check_cache/* > TheRock/build/logs/ccache_compiler_check_cache.log
 
+      # Reported by ll-cache-bit-gate, so a failure doesn't skip the test
+      # suites. dist/ keeps the unsplit lib.
+      - name: LL cache-bit guard
+        id: ll_cache_bits
+        continue-on-error: true
+        env:
+          ROCM_PATH: ${{ github.workspace }}/TheRock/build/dist/rocm
+          RCCL_LIB: ${{ github.workspace }}/TheRock/build/dist/rocm/lib/librccl.so
+          RCCL_LL_GUARD_REQUIRED: 1
+        run: |
+          python3 -m pip install -r projects/rccl/test/ll-cache-bits/requirements.txt
+          python3 -m pytest projects/rccl/test/ll-cache-bits/tests -v -rs
+
       - name: Configure AWS Credentials for non-forked repos
         if: ${{ always() && !github.event.pull_request.head.repo.fork }}
         uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1
@@ -181,6 +196,16 @@ jobs:
             --build-dir TheRock/build \
             --upload
 
+  ll-cache-bit-gate:
+    name: LL cache-bit gate
+    needs: [therock-build-linux]
+    if: ${{ !cancelled() && needs.therock-build-linux.outputs.ll_cache_bits == 'failure' }}
+    runs-on: ubuntu-24.04
+    steps:
+      - run: |
+          echo "LL cache-bit guard failed; see its step in Build Linux Packages." >&2
+          exit 1
+
   therock-test-linux-multi-node:
     name: "Test multi-node"
     # TODO: Re-enable gfx94X-dcgpu and gfx950-dcgpu multi-node tests once the clusters are stable again.

projects/rccl/test/host/run_host_tests.sh

@@ -23,8 +23,9 @@
 #                   prerequisite the host tests compile against
 #   configure       configure test/host
 #   build           build all host binaries (default target)
-#   guards          device-table unittest, the kernel-count and accl-reporter
-#                   pytest suites, the tools/scripts/test_runner unittests
+#   guards          device-table unittest, the kernel-count, accl-reporter and
+#                   ll-cache-bits detector pytest suites, the
+#                   tools/scripts/test_runner unittests
 #                   (which include the rccl-device-compile driver cases),
 #                   src/include/test_poison_hip_atomics.py,
 #                   test/test_makefile_depflags.py, and the
@@ -279,6 +280,15 @@ do_accl_report_guards() {
   "$venv/bin/python" -m pytest "$RCCL_ROOT/test/test_accl_report.py" -v
 }
 
+# Run the ll-cache-bits detector tests. The ISA guard itself needs a built
+# librccl.so, so it runs in the TheRock RCCL build job instead.
+do_ll_cache_bits_guards() {
+  echo "==> LL cache-bit detector (pytest: test/ll-cache-bits)"
+  local venv
+  venv=$(guard_venv) || return 1
+  "$venv/bin/python" -m pytest "$RCCL_ROOT/test/ll-cache-bits/tests" -m detector -v
+}
+
 # Run the RcclReplayer host checks (g++ only, no MPI or HIP); nothing else builds tools/.
 do_replayer_guards() {
   echo "==> RcclReplayer host checks (make -C tools/RcclReplayer test)"
@@ -297,19 +307,20 @@ do_test_runner_guards() {
 }
 
 # All CPU-only guards: makefile -MM scan, the device-table unittest, the
-# kernel-count and accl-reporter pytest suites, the tools/scripts/test_runner
-# unittests, the __hip_atomic_* poison compile probe, then the RcclReplayer host
-# checks. Collected with `|| rc=1` rather than run back to back so that under
-# `set -e` an early failure still leaves the later guards running and reported,
-# instead of aborting the phase at the first one. Same idiom as do_host_tests
-# above.
+# kernel-count, accl-reporter and ll-cache-bits pytest suites, the
+# tools/scripts/test_runner unittests, the __hip_atomic_* poison compile probe,
+# then the RcclReplayer host checks. Collected with `|| rc=1` rather than run
+# back to back so that under `set -e` an early failure still leaves the later
+# guards running and reported, instead of aborting the phase at the first one.
+# Same idiom as do_host_tests above.
 do_guards() {
   local rc=0
   do_makefile_depflags || rc=1
   do_device_table_guards || rc=1
   do_kernel_count_guards || rc=1
   do_tuner_embed_guards || rc=1
   do_accl_report_guards || rc=1
+  do_ll_cache_bits_guards || rc=1
   do_test_runner_guards || rc=1
   do_poison_hip_atomics || rc=1
   do_replayer_guards || rc=1

projects/rccl/test/ll-cache-bits/.gitignore

@@ -0,0 +1,3 @@
+__pycache__/
+.pytest_cache/
+*.pyc

projects/rccl/test/ll-cache-bits/README.md

@@ -0,0 +1,74 @@
+# RCCL LL Cache-Bit Guard
+
+A [pytest](https://pytest.org) guard that checks the cache-policy bits on the
+LL protocol's flag-poll loads in a built `librccl.so`.
+
+## Why
+
+An LL receiver re-reads one 16-byte line `{data1, flag1, data2, flag2}` until
+both flags change: the LL primitives (`readLL` in `src/device/prims_ll.h`), the
+DDA LL kernels (`ddaLLWaitLineFlags`) and LLA2A in the symmetric kernels. A
+peer GPU or a NIC writes that line. If the load can be served from a stale
+cached copy, the receiver never sees the new flag and the collective hangs.
+
+The source asks for the right behaviour (`__builtin_nontemporal_load`, or a
+system-scope load on some arches), but the compiler picks the bits that end up
+in the instruction. A compiler update, a code change, or a new arch can drop
+them with no build error. This guard reads the final ISA and fails if that
+happens.
+
+## What it checks
+
+1. Pulls the device code for each arch out of `librccl.so`, from every offload
+   bundle (the device linker adds one per fat object).
+2. Disassembles the functions that can hold LL poll loads.
+3. Finds every poll load by its shape: a 128-bit load whose two flag words are
+   both compared against the expected value (directly or via each other). Cache
+   bits are not used to find it, so a poll that lost them is still found.
+4. Asserts each poll carries one of the arch's expected modifier sets, e.g.
+   `nt` or `sc0 sc1` on gfx942.
+
+It also fails if some LL function (`ProtoLL`, DDA LL or symmetric LL kernel)
+has no recognisable poll at all, so a change in code shape can't make the
+guard pass by finding nothing.
+
+| File | Purpose |
+|------|---------|
+| `tests/ll_isa.py` | poll-load detector and ROCm LLVM tool wrappers |
+| `tests/test_ll_poll_cache_bits.py` | the guard, run against a built `librccl.so` |
+| `tests/test_ll_isa_detector.py` | detector unit tests on hand-written ISA (no ROCm needed) |
+
+Arches with an expectation: **gfx942, gfx950, gfx1250**. Other arches in the
+library are reported as skipped, not passed.
+
+## Running locally
+
+Needs a built RCCL and the ROCm LLVM tools. No GPU.
+
+```bash
+cd projects/rccl/test/ll-cache-bits
+python3 -m pip install -r requirements.txt
+python3 -m pytest -v -rs
+```
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `RCCL_LIB` | `$RCCL_BUILD/librccl.so` | library to inspect |
+| `RCCL_BUILD` | `<rccl>/build/release`, else `build/debug` | RCCL build dir |
+| `ROCM_PATH` | `/opt/rocm` | ROCm install root |
+| `RCCL_LL_GUARD_REQUIRED` | unset | `1`: fail instead of skip if the library, tools or all checked arches are missing |
+
+The whole suite skips if the library or tools are missing.
+
+## CI
+
+- The TheRock RCCL build job (`.github/workflows/therock-rccl-ci-linux.yml`)
+  runs the full suite on the built `librccl.so` with `RCCL_LL_GUARD_REQUIRED=1`.
+  That job runs nightly and on manual dispatch, not per PR.
+- `test/host/run_host_tests.sh guards` runs the detector tests (`-m detector`).
+
+## When it fails
+
+The failure lists the offending loads and the functions they're in. Find out
+why the bits changed (compiler, source, or build flags) and fix that. Do not
+just edit `EXPECTED_POLL_MODIFIERS` to match.

projects/rccl/test/ll-cache-bits/pytest.ini

@@ -0,0 +1,7 @@
+# pytest.ini
+[pytest]
+testpaths = tests
+addopts = --import-mode=importlib
+markers =
+    detector: unit tests for the LL poll-load detector (no ROCm or build needed)
+    isa: checks the device ISA inside a built librccl.so

projects/rccl/test/ll-cache-bits/requirements.txt

@@ -0,0 +1,4 @@
+# Testing dependencies for the LL cache-bit guard.
+# The guard itself only uses the Python standard library plus the ROCm LLVM
+# tools; pytest is the only external dependency.
+pytest==9.1.1