From e8feaa61c68f87367c8f7ab70c70106907402461 Mon Sep 17 00:00:00 2001 From: Larry Meadows Date: Thu, 24 Sep 2026 20:55:36 -0500 Subject: [PATCH 01/10] fix(rccl): fix ASAN GPU target selection, and propagate ASAN to rocSHMEM An ASAN build rewrote every requested GPU target to :xnack+. That is right for gfx9, where xnack is a selectable target-ID feature and a bare arch name compiles with -fsanitize=address silently ignored. It is wrong for everything else: - gfx1250 has xnack always on, so the feature is not selectable and "gfx1250:xnack+" is an invalid target ID. rocm_check_target_ids then dropped it without comment, so asking for gfx1250 in an ASAN build quietly built nothing for gfx1250. - gfx10xx/gfx11xx/gfx12xx have no xnack at all. Appending :xnack+ made them invalid too, so they were also silently dropped. - the test for an existing suffix looked for the substring ":xnack+", which does not match ":xnack-". A requested gfx942:xnack- became "gfx942:xnack-:xnack+". Replace the rewrite with an explicit per-family policy in cmake/AsanGpuTargets.cmake: normalise gfx9 to :xnack+, pass always-on-xnack parts through untouched, and drop the rest with a message rather than leaving a target that would produce an uninstrumented device image inside an ASAN build. Fail the configure if nothing survives. Each rule was verified against amdclang++ and is pinned by tests/test_asan_gpu_targets.py. Also pass -DASAN=ON to the rocSHMEM ExternalProject. Under ENABLE_ROCSHMEM, librocshmem.a is linked into librccl.so, so an ASAN build of rccl was shipping an uninstrumented rocSHMEM inside an instrumented library -- precisely the allocations ASAN is there to police. rocSHMEM's own ASAN block gets the same gfx9-only restriction, since it cannot include rccl's module: rccl configures it as a separate cmake process, and it also builds standalone. --- projects/rccl/CMakeLists.txt | 23 ++-- projects/rccl/cmake/AsanGpuTargets.cmake | 65 ++++++++++ projects/rccl/cmake/ROCSHMEM.cmake | 8 +- .../tests/test_asan_gpu_targets.py | 122 ++++++++++++++++++ projects/rocshmem/CMakeLists.txt | 87 +++++++------ 5 files changed, 251 insertions(+), 54 deletions(-) create mode 100644 projects/rccl/cmake/AsanGpuTargets.cmake create mode 100644 projects/rccl/tools/scripts/test_runner/tests/test_asan_gpu_targets.py diff --git a/projects/rccl/CMakeLists.txt b/projects/rccl/CMakeLists.txt index 4900545606f5..75133e1e6f9a 100644 --- a/projects/rccl/CMakeLists.txt +++ b/projects/rccl/CMakeLists.txt @@ -199,8 +199,10 @@ endif() set(GPU_TARGETS "${DEFAULT_GPUS}" CACHE STRING "Target default GPUs if GPU_TARGETS is not defined.") -# Modify GPU architectures for Address Sanitizer builds by appending "xnack+" +# Adjust GPU architectures for Address Sanitizer builds; per-family rules are +# in cmake/AsanGpuTargets.cmake. if (BUILD_ADDRESS_SANITIZER) + include(cmake/AsanGpuTargets.cmake) set(_suffix "asan") set(_clang_rt_names "libclang_rt.${_suffix}.so" "libclang_rt.${_suffix}-x86_64.so") foreach(_clang_rt_name ${_clang_rt_names}) @@ -220,18 +222,13 @@ if (BUILD_ADDRESS_SANITIZER) endif() message(STATUS "ASAN runtime directory: ${ASAN_RUNTIME_DIR}") - SET(amdgpu_targets "") - foreach(amdgpu_target IN LISTS GPU_TARGETS) - if(NOT amdgpu_target STREQUAL "") - string(FIND "${amdgpu_target}" ":xnack+" HAS_XNACK_SUFFIX) - if(HAS_XNACK_SUFFIX EQUAL -1) - list(APPEND amdgpu_targets "${amdgpu_target}:xnack+") - else() - list(APPEND amdgpu_targets "${amdgpu_target}") - endif() - endif() - endforeach() - SET(GPU_TARGETS "${amdgpu_targets}") + # Guarded so that "no targets at all" still reaches the curated diagnostic + # below. rocm_local_targets() reports failure as the literal "NOTFOUND", + # which if() treats as false along with the empty case. + if(GPU_TARGETS) + rccl_asan_adjust_gpu_targets(GPU_TARGETS ${GPU_TARGETS}) + message(STATUS "ASAN: GPU targets = ${GPU_TARGETS}") + endif() endif() # Check if clang compiler can offload to GPU_TARGETS diff --git a/projects/rccl/cmake/AsanGpuTargets.cmake b/projects/rccl/cmake/AsanGpuTargets.cmake new file mode 100644 index 000000000000..86cdb8929749 --- /dev/null +++ b/projects/rccl/cmake/AsanGpuTargets.cmake @@ -0,0 +1,65 @@ +# GPU target selection for device AddressSanitizer builds. Device ASAN needs +# xnack, which gfx9 must request explicitly, gfx1250 has always on (and rejects +# the suffix), and no other family has. Cases are pinned in +# tools/scripts/test_runner/tests/test_asan_gpu_targets.py; +# projects/rocshmem/CMakeLists.txt repeats the policy inline, being a separate +# cmake process. + +# Parts where xnack is always on, so naming it is an error. +set(_RCCL_ASAN_XNACK_ALWAYS_ON gfx1250 gfx1250-strict) + +# rccl_asan_adjust_gpu_targets(OUT_VAR gfx942 gfx1100 gfx1250) +# -> OUT_VAR = gfx942:xnack+;gfx1250 +# Uninstrumentable targets are dropped rather than left to produce an +# uninstrumented image inside an ASAN build. +function(rccl_asan_adjust_gpu_targets OUT_VAR) + set(_result "") + set(_dropped "") + + foreach(_target IN LISTS ARGN) + if(_target STREQUAL "") + continue() + endif() + + # A requested suffix says what the caller wants, not what the part + # supports, so classify on the bare name. + string(REGEX REPLACE ":.*" "" _base "${_target}") + # list(FIND) not IN_LIST: the test loads this via cmake -P, where CMP0057 + # is unset. + list(FIND _RCCL_ASAN_XNACK_ALWAYS_ON "${_base}" _always_on) + + # Drop any xnack term the caller wrote before deciding what to do with it: + # it may be ":xnack-", which contradicts the build. Other feature terms + # (e.g. ":sramecc+") are part of the requested target and are kept; xnack + # sorts last, so re-appending it keeps the target ID canonical. + string(REGEX REPLACE ":xnack[+-]" "" _without_xnack "${_target}") + + if(_base MATCHES "^gfx9") + list(APPEND _result "${_without_xnack}:xnack+") + elseif(NOT _always_on EQUAL -1) + # Always on here, so naming xnack at all is an invalid target ID. + list(APPEND _result "${_without_xnack}") + else() + list(APPEND _dropped "${_target}") + endif() + endforeach() + + # Both xnack spellings of one arch normalise together, and a duplicate + # becomes a second add_library() in DeviceLinker.cmake. + if(_result) + list(REMOVE_DUPLICATES _result) + endif() + + if(_dropped) + message(STATUS + "ASAN: dropping GPU target(s) with no xnack support: ${_dropped}") + endif() + if(NOT _result) + message(FATAL_ERROR + "ASAN requires a GPU target that supports xnack (gfx9* with :xnack+, " + "or gfx1250 where it is always on). None of the requested GPU targets " + "do: ${ARGN}") + endif() + + set(${OUT_VAR} "${_result}" PARENT_SCOPE) +endfunction() diff --git a/projects/rccl/cmake/ROCSHMEM.cmake b/projects/rccl/cmake/ROCSHMEM.cmake index 13fc8eee1d7f..577af31afca6 100644 --- a/projects/rccl/cmake/ROCSHMEM.cmake +++ b/projects/rccl/cmake/ROCSHMEM.cmake @@ -124,6 +124,12 @@ function(add_rocshmem_targets) if(ENABLE_ROCSHMEM_GIN) set(_rocshmem_sdma_opt "-DUSE_SDMA=ON") endif() + # librocshmem.a is linked into librccl.so, so without this an ASAN + # build gets an uninstrumented rocSHMEM inside an instrumented library. + set(_rocshmem_asan_opt "") + if(BUILD_ADDRESS_SANITIZER) + set(_rocshmem_asan_opt "-DASAN=ON") + endif() message(STATUS "rocSHMEM: building from ${ROCSHMEM_SOURCE_DIR}") ExternalProject_Add(rocshmem_ext @@ -141,7 +147,7 @@ function(add_rocshmem_targets) CONFIGURE_COMMAND "" BUILD_COMMAND ${CMAKE_COMMAND} -E make_directory build - && ${CMAKE_COMMAND} -E chdir build bash -lc "INSTALL_PREFIX=${ROCSHMEM_INSTALL_DIR} ../scripts/build_configs/gda -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DUSE_EXTERNAL_MPI=OFF -DGDA_MLX5=ON -DGDA_BNXT=ON -DGDA_IONIC=ON -DBUILD_EXAMPLES=OFF -DBUILD_FUNCTIONAL_TESTS=OFF -DBUILD_UNIT_TESTS=OFF -DBUILD_CTESTS=OFF -DBUILD_TOOLS=OFF -DGPU_TARGETS=${_rocshmem_gpu_targets} ${_rocshmem_sdma_opt} ${_rocshmem_cmake_opts} " + && ${CMAKE_COMMAND} -E chdir build bash -lc "INSTALL_PREFIX=${ROCSHMEM_INSTALL_DIR} ../scripts/build_configs/gda -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DUSE_EXTERNAL_MPI=OFF -DGDA_MLX5=ON -DGDA_BNXT=ON -DGDA_IONIC=ON -DBUILD_EXAMPLES=OFF -DBUILD_FUNCTIONAL_TESTS=OFF -DBUILD_UNIT_TESTS=OFF -DBUILD_CTESTS=OFF -DBUILD_TOOLS=OFF -DGPU_TARGETS=${_rocshmem_gpu_targets} ${_rocshmem_sdma_opt} ${_rocshmem_asan_opt} ${_rocshmem_cmake_opts} " INSTALL_COMMAND "" ) diff --git a/projects/rccl/tools/scripts/test_runner/tests/test_asan_gpu_targets.py b/projects/rccl/tools/scripts/test_runner/tests/test_asan_gpu_targets.py new file mode 100644 index 000000000000..9b303993aa45 --- /dev/null +++ b/projects/rccl/tools/scripts/test_runner/tests/test_asan_gpu_targets.py @@ -0,0 +1,122 @@ +"""Guards the ASAN GPU-target policy in cmake/AsanGpuTargets.cmake. + +Each case pins one family's observed clang behaviour, so reclassifying a +family fails here rather than producing an ASAN build with a silently +uninstrumented device image. Driven through `cmake -P`, like +test_device_coverage_cmake.py. +""" + +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + + +RCCL_ROOT = Path(__file__).resolve().parents[4] +MODULE = RCCL_ROOT / "cmake" / "AsanGpuTargets.cmake" + + +@unittest.skipUnless(shutil.which("cmake"), "cmake not available on PATH") +class AsanGpuTargetsTest(unittest.TestCase): + def adjust(self, targets): + """Run rccl_asan_adjust_gpu_targets over the given targets. + + Returns (result_list, status_messages). Raises on FATAL_ERROR. + """ + quoted = " ".join(f'"{t}"' for t in targets) + with tempfile.TemporaryDirectory() as temp_dir: + out_file = Path(temp_dir) / "out.txt" + script = Path(temp_dir) / "probe.cmake" + script.write_text( + f'include("{MODULE}")\n' + f"rccl_asan_adjust_gpu_targets(RESULT {quoted})\n" + f'file(WRITE "{out_file}" "${{RESULT}}")\n' + ) + completed = subprocess.run( + ["cmake", "-P", str(script)], + capture_output=True, text=True) + if completed.returncode != 0: + raise AssertionError(completed.stderr.strip()) + result = out_file.read_text() + # message(STATUS) goes to stdout under cmake -P, FATAL_ERROR to stderr. + return ([t for t in result.split(";") if t], completed.stdout) + + def test_gfx9_gets_xnack_plus_requested(self): + """A bare gfx9 name compiles with the sanitizer ignored.""" + result, _ = self.adjust(["gfx90a", "gfx942", "gfx950"]) + + self.assertEqual(result, + ["gfx90a:xnack+", "gfx942:xnack+", "gfx950:xnack+"]) + + def test_gfx9_already_requesting_xnack_plus_is_unchanged(self): + result, _ = self.adjust(["gfx942:xnack+"]) + + self.assertEqual(result, ["gfx942:xnack+"]) + + def test_gfx9_requesting_xnack_minus_is_corrected(self): + """The old substring test for ":xnack+" produced gfx942:xnack-:xnack+.""" + result, _ = self.adjust(["gfx942:xnack-"]) + + self.assertEqual(result, ["gfx942:xnack+"]) + + def test_gfx9_keeps_its_other_target_features(self): + """Only the xnack term is ours to rewrite; sramecc+ is what was asked for.""" + result, _ = self.adjust(["gfx942:sramecc+:xnack-"]) + + self.assertEqual(result, ["gfx942:sramecc+:xnack+"]) + + def test_both_xnack_variants_of_one_arch_collapse_to_one_entry(self): + """A duplicate becomes a second add_library() in DeviceLinker.cmake.""" + result, _ = self.adjust(["gfx942:xnack+", "gfx942:xnack-"]) + + self.assertEqual(result, ["gfx942:xnack+"]) + + def test_gfx1250_is_kept_untouched(self): + result, _ = self.adjust(["gfx1250"]) + + self.assertEqual(result, ["gfx1250"]) + + def test_gfx1250_loses_an_explicitly_requested_xnack(self): + """clang rejects 'gfx1250:xnack+' outright: xnack is not selectable there.""" + result, _ = self.adjust(["gfx1250:xnack+"]) + + self.assertEqual(result, ["gfx1250"]) + + def test_gfx1250_with_and_without_xnack_collapse_to_one_entry(self): + result, _ = self.adjust(["gfx1250", "gfx1250:xnack+"]) + + self.assertEqual(result, ["gfx1250"]) + + def test_archs_without_xnack_are_dropped(self): + result, messages = self.adjust(["gfx942", "gfx1100", "gfx1201"]) + + self.assertEqual(result, ["gfx942:xnack+"]) + self.assertIn("gfx1100", messages) + self.assertIn("gfx1201", messages) + + def test_full_default_target_list(self): + """Matches the old effective set, except gfx1250 is no longer lost. + + The input is a copy of DEFAULT_GPUS in the root CMakeLists.txt and has + to be updated by hand when a family is added there. + """ + result, _ = self.adjust([ + "gfx906", "gfx908", "gfx90a", "gfx942", "gfx950", + "gfx1030", "gfx1100", "gfx1101", "gfx1102", "gfx1151", + "gfx1200", "gfx1201", "gfx1250"]) + + self.assertEqual(result, [ + "gfx906:xnack+", "gfx908:xnack+", "gfx90a:xnack+", + "gfx942:xnack+", "gfx950:xnack+", "gfx1250"]) + + def test_empty_result_is_fatal(self): + with self.assertRaises(AssertionError) as cm: + self.adjust(["gfx1100"]) + + self.assertIn("ASAN requires a GPU target that supports xnack", + str(cm.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/projects/rocshmem/CMakeLists.txt b/projects/rocshmem/CMakeLists.txt index 140ccaeb8577..ff92a71d3c2b 100644 --- a/projects/rocshmem/CMakeLists.txt +++ b/projects/rocshmem/CMakeLists.txt @@ -339,52 +339,59 @@ if (NOT BUILD_TESTS_ONLY) endif() if (ASAN) - message(STATUS "Removing xnack- entries from GPU_TARGETS because they are invalid with AddressSanitizer (ASAN)") - list(FILTER GPU_TARGETS EXCLUDE REGEX ".*:xnack-$") - if(GPU_TARGETS_FULL) - list(FILTER GPU_TARGETS_FULL EXCLUDE REGEX ".*:xnack-$") - endif() - - # RDNA archs (gfx11xx/gfx12xx) have no xnack+ variant at all, so device - # ASAN can never be enabled for them - clang just ignores - # -fsanitize=address for that offload arch with a warning. Drop them - # instead of leaving a target in GPU_TARGETS that silently gets no - # instrumentation. - set(_ASAN_INCOMPATIBLE_ARCHS gfx1100 gfx1201 gfx1250 gfx1250-strict) - foreach(_asan_incompatible_arch ${_ASAN_INCOMPATIBLE_ARCHS}) - list(FILTER GPU_TARGETS EXCLUDE REGEX "^${_asan_incompatible_arch}(:.*)?$") - if(GPU_TARGETS_FULL) - list(FILTER GPU_TARGETS_FULL EXCLUDE REGEX "^${_asan_incompatible_arch}(:.*)?$") + # Device ASAN needs xnack. gfx9 must request it explicitly, or a bare arch + # name compiles with the sanitizer silently ignored; gfx1250 has it always + # on and rejects an explicit xnack term as an invalid target ID; no other + # family has it at all. Drop what cannot be instrumented rather than ship + # an uninstrumented target inside an ASAN build. Same policy as + # projects/rccl/cmake/AsanGpuTargets.cmake, repeated here because rccl + # builds rocSHMEM as a separate cmake process. + set(_ASAN_XNACK_ALWAYS_ON gfx1250 gfx1250-strict) + + # GPU_TARGETS_FULL mirrors GPU_TARGETS, so only the first pass reports. + set(_asan_report ON) + foreach(_asan_list GPU_TARGETS GPU_TARGETS_FULL) + if("${${_asan_list}}" STREQUAL "") + continue() endif() - endforeach() - if(NOT GPU_TARGETS) - message(FATAL_ERROR "ASAN requires a GPU target that supports xnack+ (e.g. gfx90a, gfx942, gfx950). " - "None of the requested/default GPU_TARGETS support it.") - endif() + set(_asan_kept "") + foreach(_asan_target ${${_asan_list}}) + string(REGEX REPLACE ":.*" "" _asan_base "${_asan_target}") + list(FIND _ASAN_XNACK_ALWAYS_ON "${_asan_base}" _asan_always_on) + # Other feature terms (e.g. sramecc+) are part of what was asked for + # and are kept; xnack sorts last, so re-appending it stays canonical. + string(REGEX REPLACE ":xnack[+-]" "" _asan_stripped "${_asan_target}") + + if(_asan_base MATCHES "^gfx9") + set(_asan_fixed "${_asan_stripped}:xnack+") + elseif(NOT _asan_always_on EQUAL -1) + set(_asan_fixed "${_asan_stripped}") + else() + if(_asan_report) + message(STATUS "ASAN: dropping GPU target '${_asan_target}' - no xnack support") + endif() + continue() + endif() - # CDNA archs (gfx90a/gfx942/gfx950) only get ASAN instrumentation with the - # xnack+ feature explicitly enabled; a bare arch name compiles without it. - # Force :xnack+ onto any entry that doesn't already request it. - set(_ASAN_GPU_TARGETS "") - foreach(_asan_target ${GPU_TARGETS}) - if(NOT _asan_target MATCHES "xnack\\+") - message(STATUS "ASAN: appending :xnack+ to GPU target '${_asan_target}'") - set(_asan_target "${_asan_target}:xnack+") + if(_asan_report AND NOT _asan_fixed STREQUAL _asan_target) + message(STATUS "ASAN: GPU target '${_asan_target}' -> '${_asan_fixed}'") + endif() + list(APPEND _asan_kept "${_asan_fixed}") + endforeach() + + # Both xnack spellings of one arch normalise to the same string. + if(_asan_kept) + list(REMOVE_DUPLICATES _asan_kept) endif() - list(APPEND _ASAN_GPU_TARGETS "${_asan_target}") + set(${_asan_list} "${_asan_kept}") + set(_asan_report OFF) endforeach() - set(GPU_TARGETS "${_ASAN_GPU_TARGETS}") - if(GPU_TARGETS_FULL) - set(_ASAN_GPU_TARGETS_FULL "") - foreach(_asan_target ${GPU_TARGETS_FULL}) - if(NOT _asan_target MATCHES "xnack\\+") - set(_asan_target "${_asan_target}:xnack+") - endif() - list(APPEND _ASAN_GPU_TARGETS_FULL "${_asan_target}") - endforeach() - set(GPU_TARGETS_FULL "${_ASAN_GPU_TARGETS_FULL}") + if(NOT GPU_TARGETS) + message(FATAL_ERROR "ASAN requires a GPU target that supports xnack (gfx9* with :xnack+, " + "or gfx1250 where it is always on). " + "None of the requested/default GPU_TARGETS do.") endif() endif() From 82d941cdc2b284c2f0ceb139e54aa7e3e1ba409c Mon Sep 17 00:00:00 2001 From: Larry Meadows Date: Sat, 26 Sep 2026 15:33:57 -0500 Subject: [PATCH 02/10] fix(rocshmem): apply the ASAN target policy before target-ID validation The policy ran after rocm_check_target_ids, so the one input it exists to rewrite never reached it: gfx1250:xnack+ is a hard "invalid target ID" error from clang, not a warning, so validation dropped it and an ASAN configure died with "ASAN requires a GPU target that supports xnack" instead of normalising to gfx1250. Reproduced with a standalone rocSHMEM configure before the change. Moving the policy above validation also collapses it: SUPPORTED_GPUS now comes back already normalised, so GPU_TARGETS and GPU_TARGETS_FULL both inherit it and the two-list foreach goes away. It is a function now, matching the shape of rccl's AsanGpuTargets.cmake. It also no longer sits inside NOT BUILD_TESTS_ONLY, where an ASAN build previously got unnormalised targets. BITCODE_GPU_ARCHS_FULL gets the same FORCE treatment as GPU_BUILD_TARGETS just above it. cmake/DeviceBitcode.cmake caches it without FORCE, so a build directory first configured without ASAN kept a bare gfx942 while GPU_TARGETS had correctly become gfx942:xnack+, and the bitcode compile lost the feature. Verified by standalone rocSHMEM configures: gfx1250:xnack+ -> gfx1250 (was fatal), gfx942 -> gfx942:xnack+, gfx942:sramecc+:xnack- -> gfx942:sramecc+:xnack+, defaults drop gfx1100 and gfx1201, a non-ASAN configure is untouched, and a no-ASAN-then-ASAN reconfigure of one build directory now updates BITCODE_GPU_ARCHS_FULL where it previously stayed stale. --- projects/rocshmem/CMakeLists.txt | 131 +++++++++++++++++-------------- 1 file changed, 74 insertions(+), 57 deletions(-) diff --git a/projects/rocshmem/CMakeLists.txt b/projects/rocshmem/CMakeLists.txt index ff92a71d3c2b..2c814dda34fb 100644 --- a/projects/rocshmem/CMakeLists.txt +++ b/projects/rocshmem/CMakeLists.txt @@ -241,6 +241,71 @@ if(GPU_TARGETS) list(REMOVE_DUPLICATES GPU_TARGETS) endif() +# Device ASAN needs xnack. gfx9 must request it explicitly, or a bare arch name +# compiles with the sanitizer silently ignored; gfx1250 has it always on and +# rejects an explicit xnack term as an invalid target ID; no other family has +# it at all. Drop what cannot be instrumented rather than ship an +# uninstrumented target inside an ASAN build. Same policy as +# projects/rccl/cmake/AsanGpuTargets.cmake, repeated here because rccl builds +# rocSHMEM as a separate cmake process. +# +# This runs before rocm_check_target_ids, not after. gfx1250:xnack+ is a hard +# "invalid target ID" error from clang, so a target this policy exists to +# normalise to gfx1250 would otherwise fail validation and be dropped before +# the policy ever saw it. Normalising first also means SUPPORTED_GPUS, and so +# both GPU_TARGETS and GPU_TARGETS_FULL below, are already correct. +function(rocshmem_asan_adjust_gpu_targets OUT_VAR) + set(_always_on gfx1250 gfx1250-strict) + set(_kept "") + + foreach(_target ${ARGN}) + if(_target STREQUAL "") + continue() + endif() + # A requested suffix says what the caller wants, not what the part + # supports, so classify on the bare name. + string(REGEX REPLACE ":.*" "" _base "${_target}") + list(FIND _always_on "${_base}" _is_always_on) + # Other feature terms (e.g. sramecc+) are part of what was asked for and + # are kept; xnack sorts last, so re-appending it stays canonical. + string(REGEX REPLACE ":xnack[+-]" "" _stripped "${_target}") + + if(_base MATCHES "^gfx9") + set(_fixed "${_stripped}:xnack+") + elseif(NOT _is_always_on EQUAL -1) + set(_fixed "${_stripped}") + else() + message(STATUS "ASAN: dropping GPU target '${_target}' - no xnack support") + continue() + endif() + + if(NOT _fixed STREQUAL _target) + message(STATUS "ASAN: GPU target '${_target}' -> '${_fixed}'") + endif() + list(APPEND _kept "${_fixed}") + endforeach() + + # Both xnack spellings of one arch normalise to the same string. + if(_kept) + list(REMOVE_DUPLICATES _kept) + endif() + if(NOT _kept) + message(FATAL_ERROR "ASAN requires a GPU target that supports xnack (gfx9* with :xnack+, " + "or gfx1250 where it is always on). " + "None of the requested/default GPU_TARGETS do: ${ARGN}") + endif() + + set(${OUT_VAR} "${_kept}" PARENT_SCOPE) +endfunction() + +if(ASAN) + if(GPU_TARGETS) + rocshmem_asan_adjust_gpu_targets(GPU_TARGETS ${GPU_TARGETS}) + else() + rocshmem_asan_adjust_gpu_targets(DEFAULT_GPU_TARGETS ${DEFAULT_GPU_TARGETS}) + endif() +endif() + if(COMMAND rocm_check_target_ids) if(GPU_TARGETS) message(STATUS "Checking for ROCm support for GPU targets: " "${GPU_TARGETS}") @@ -338,63 +403,6 @@ if (NOT BUILD_TESTS_ONLY) endif() endif() - if (ASAN) - # Device ASAN needs xnack. gfx9 must request it explicitly, or a bare arch - # name compiles with the sanitizer silently ignored; gfx1250 has it always - # on and rejects an explicit xnack term as an invalid target ID; no other - # family has it at all. Drop what cannot be instrumented rather than ship - # an uninstrumented target inside an ASAN build. Same policy as - # projects/rccl/cmake/AsanGpuTargets.cmake, repeated here because rccl - # builds rocSHMEM as a separate cmake process. - set(_ASAN_XNACK_ALWAYS_ON gfx1250 gfx1250-strict) - - # GPU_TARGETS_FULL mirrors GPU_TARGETS, so only the first pass reports. - set(_asan_report ON) - foreach(_asan_list GPU_TARGETS GPU_TARGETS_FULL) - if("${${_asan_list}}" STREQUAL "") - continue() - endif() - - set(_asan_kept "") - foreach(_asan_target ${${_asan_list}}) - string(REGEX REPLACE ":.*" "" _asan_base "${_asan_target}") - list(FIND _ASAN_XNACK_ALWAYS_ON "${_asan_base}" _asan_always_on) - # Other feature terms (e.g. sramecc+) are part of what was asked for - # and are kept; xnack sorts last, so re-appending it stays canonical. - string(REGEX REPLACE ":xnack[+-]" "" _asan_stripped "${_asan_target}") - - if(_asan_base MATCHES "^gfx9") - set(_asan_fixed "${_asan_stripped}:xnack+") - elseif(NOT _asan_always_on EQUAL -1) - set(_asan_fixed "${_asan_stripped}") - else() - if(_asan_report) - message(STATUS "ASAN: dropping GPU target '${_asan_target}' - no xnack support") - endif() - continue() - endif() - - if(_asan_report AND NOT _asan_fixed STREQUAL _asan_target) - message(STATUS "ASAN: GPU target '${_asan_target}' -> '${_asan_fixed}'") - endif() - list(APPEND _asan_kept "${_asan_fixed}") - endforeach() - - # Both xnack spellings of one arch normalise to the same string. - if(_asan_kept) - list(REMOVE_DUPLICATES _asan_kept) - endif() - set(${_asan_list} "${_asan_kept}") - set(_asan_report OFF) - endforeach() - - if(NOT GPU_TARGETS) - message(FATAL_ERROR "ASAN requires a GPU target that supports xnack (gfx9* with :xnack+, " - "or gfx1250 where it is always on). " - "None of the requested/default GPU_TARGETS do.") - endif() - endif() - # hip-config-amd.cmake caches GPU_BUILD_TARGETS without FORCE, so on a # reconfigure of an existing build directory it can retain a stale value # (e.g. from before ASAN was enabled, or from a previous GPU_TARGETS) even @@ -402,6 +410,15 @@ if (NOT BUILD_TESTS_ONLY) # on every configure so hip::device's --offload-arch flags stay in sync. set(GPU_BUILD_TARGETS "${GPU_TARGETS}" CACHE STRING "GPU targets to compile for" FORCE) + # cmake/DeviceBitcode.cmake derives BITCODE_GPU_ARCHS_FULL from + # GPU_TARGETS_FULL, also without FORCE, so it goes stale the same way: a + # build directory first configured without ASAN keeps the bare gfx942 and the + # bitcode compile loses the :xnack+ this policy just added. Same treatment. + if(GPU_TARGETS_FULL) + set(BITCODE_GPU_ARCHS_FULL "${GPU_TARGETS_FULL}" CACHE STRING + "Full GPU arch strings with feature suffixes for device bitcode" FORCE) + endif() + find_package(hip REQUIRED) find_package(hsa-runtime64 REQUIRED) From 5ad863a53b6ab80e68e50b723f3a144ac014a98f Mon Sep 17 00:00:00 2001 From: Larry Meadows Date: Sat, 26 Sep 2026 15:44:05 -0500 Subject: [PATCH 03/10] fix(rocshmem): carry ASAN's feature suffixes into the defaults path too GPU_TARGETS_FULL was set only when the user passed -DGPU_TARGETS. That was right when the defaults path could not produce a feature suffix, but the ASAN policy now gives it one, so a default-target ASAN configure left GPU_TARGETS_FULL unset and DeviceBitcode.cmake fell back to the bare _BITCODE_DEFAULT_ARCHS -- rocSHMEM bitcode compiled without :xnack+ and therefore uninstrumented, the exact failure this PR exists to close. Key the branch on whether SUPPORTED_GPUS actually carries a suffix instead. The clearing arm keeps its stale-value protection and loses nothing, since DeviceBitcode.cmake's fallback is the same bare list it would have been given. Before: defaults + ASAN gave GPU_TARGETS=gfx90a:xnack+;gfx942:xnack+;gfx950:xnack+;gfx1250 but BITCODE_GPU_ARCHS_FULL=gfx90a;gfx942;gfx950;gfx1250. Both now match. --- projects/rocshmem/CMakeLists.txt | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/projects/rocshmem/CMakeLists.txt b/projects/rocshmem/CMakeLists.txt index 2c814dda34fb..60a3ce4bb3fe 100644 --- a/projects/rocshmem/CMakeLists.txt +++ b/projects/rocshmem/CMakeLists.txt @@ -327,9 +327,14 @@ endif() # Consumed by DeviceBitcode.cmake to embed correct amdhsa.target metadata in # the device bitcode — HIP error 209 occurs if the suffix in the bitcode # metadata does not match the active feature configuration of the GPU. -# Explicitly cleared when GPU_TARGETS is not set so a stale cached value from a -# previous -DGPU_TARGETS=... configure run cannot persist into the defaults path. -if(GPU_TARGETS) +# Keyed on whether SUPPORTED_GPUS actually carries a suffix rather than on +# whether the user set GPU_TARGETS: under ASAN the defaults path produces +# :xnack+ as well, and testing GPU_TARGETS would skip it and leave the bitcode +# compile on DeviceBitcode.cmake's bare _BITCODE_DEFAULT_ARCHS fallback. +# Explicitly cleared otherwise so a stale cached value from a previous +# -DGPU_TARGETS=... configure run cannot persist into the defaults path; that +# arm loses nothing, since the fallback is the same bare list. +if(SUPPORTED_GPUS MATCHES ":") set(GPU_TARGETS_FULL "${SUPPORTED_GPUS}" CACHE STRING "Validated GPU arch strings with feature suffixes (consumed by DeviceBitcode.cmake)" FORCE) else() From c8060eb68a43f44993a29dbdacde6a2088c03720 Mon Sep 17 00:00:00 2001 From: Larry Meadows Date: Sat, 26 Sep 2026 23:02:02 -0500 Subject: [PATCH 04/10] fix(rocshmem): stop the ASAN policy from feeding on its own output Moving the policy above the GPU_TARGETS cache write made its result the value that gets FORCE-cached, so the next configure of the same build directory read it back as input. A later -DASAN=OFF then kept the :xnack+ and never brought back the dropped families: develop restores gfx1100 and gfx1201, this branch did not. Remember GPU_TARGETS as it arrived and re-derive from that each time; GPU_TARGETS still matching what we wrote last time means nobody has edited it, anything else is a user edit and becomes the new request. BITCODE_GPU_ARCHS_FULL gets the else arm its GPU_TARGETS_FULL counterpart already had, so a suffixed value cannot outlive a configure that resolves to bare archs. The policy function now mirrors cmake/AsanGpuTargets.cmake name for name. _always_on held the list here and the list(FIND) index there, so one name meant opposite things in two copies that have to stay in sync. On the rccl side, ExternalProject stamps its build step on completion rather than on BUILD_COMMAND text, so flipping ASAN over an existing tree kept the previous uninstrumented librocshmem.a. Writing the options to a file and hanging the build step off it re-runs the sub-build exactly when they change, which BUILD_ALWAYS would also do but at the cost of every invocation. Verified on both Unix Makefiles and Ninja. The pre-built ROCSHMEM_INSTALL_DIR path returns before the ASAN propagation, so it now warns instead of silently linking a possibly uninstrumented archive into an ASAN rccl. Tests: gfx1250-strict is pinned (it was the one entry of the always-on list no case reached, so deleting it left all cases green), and adjust() returns stderr as well as stdout so the tests do not pin the policy to a message level. --- projects/rccl/cmake/ROCSHMEM.cmake | 22 +++++ .../tests/test_asan_gpu_targets.py | 14 +++- projects/rocshmem/CMakeLists.txt | 80 +++++++++++++------ 3 files changed, 90 insertions(+), 26 deletions(-) diff --git a/projects/rccl/cmake/ROCSHMEM.cmake b/projects/rccl/cmake/ROCSHMEM.cmake index 577af31afca6..e1124ea2bdbb 100644 --- a/projects/rccl/cmake/ROCSHMEM.cmake +++ b/projects/rccl/cmake/ROCSHMEM.cmake @@ -97,6 +97,16 @@ function(add_rocshmem_targets) endif() find_package(rocshmem_static) if(rocshmem_static_FOUND) + # This path returns before the -DASAN=ON propagation below, so an + # ASAN rccl links whatever the pre-built tree happens to contain. + if(BUILD_ADDRESS_SANITIZER) + message(WARNING + "ASAN build using pre-built rocSHMEM at ${ROCSHMEM_INSTALL_DIR}. " + "It is linked into librccl.so as-is; if it was not itself built " + "with -DASAN=ON, rccl gets an uninstrumented rocSHMEM inside an " + "instrumented library. Unset ROCSHMEM_INSTALL_DIR to build it " + "from source with ASAN propagated.") + endif() set(ROCSHMEM_INCLUDE_DIR "${ROCSHMEM_INCLUDE_DIR}" PARENT_SCOPE) set(ROCSHMEM_LIBRARY "${ROCSHMEM_LIBRARY}" PARENT_SCOPE) set(ROCSHMEM_SOURCE_DIR "${ROCSHMEM_SOURCE_DIR}" CACHE INTERNAL "rocSHMEM source directory") @@ -151,6 +161,18 @@ function(add_rocshmem_targets) INSTALL_COMMAND "" ) + # ExternalProject stamps its build step on completion, not on the text + # of BUILD_COMMAND, so flipping ASAN over an existing tree leaves the + # step up to date and keeps the previous, uninstrumented + # librocshmem.a. Record the options in a file and hang the build step + # off it: file(GENERATE) rewrites only when the content differs, so the + # sub-build re-runs exactly when its options change, without the + # every-invocation cost of BUILD_ALWAYS. + set(_rocshmem_opts_stamp "${CMAKE_CURRENT_BINARY_DIR}/rocshmem_ext_options.txt") + file(GENERATE OUTPUT "${_rocshmem_opts_stamp}" + CONTENT "${CMAKE_BUILD_TYPE} ${_rocshmem_gpu_targets} ${_rocshmem_sdma_opt} ${_rocshmem_asan_opt} ${_rocshmem_cmake_opts}\n") + ExternalProject_Add_StepDependencies(rocshmem_ext build "${_rocshmem_opts_stamp}") + set(ROCSHMEM_INSTALL_DIR "${ROCSHMEM_INSTALL_DIR}" PARENT_SCOPE) set(ROCSHMEM_INCLUDE_DIR "${ROCSHMEM_INSTALL_DIR}/include" PARENT_SCOPE) set(ROCSHMEM_LIBRARY "${ROCSHMEM_INSTALL_DIR}/lib/librocshmem.a" PARENT_SCOPE) diff --git a/projects/rccl/tools/scripts/test_runner/tests/test_asan_gpu_targets.py b/projects/rccl/tools/scripts/test_runner/tests/test_asan_gpu_targets.py index 9b303993aa45..353249a729a6 100644 --- a/projects/rccl/tools/scripts/test_runner/tests/test_asan_gpu_targets.py +++ b/projects/rccl/tools/scripts/test_runner/tests/test_asan_gpu_targets.py @@ -39,8 +39,10 @@ def adjust(self, targets): if completed.returncode != 0: raise AssertionError(completed.stderr.strip()) result = out_file.read_text() - # message(STATUS) goes to stdout under cmake -P, FATAL_ERROR to stderr. - return ([t for t in result.split(";") if t], completed.stdout) + # cmake -P sends STATUS to stdout and WARNING/FATAL_ERROR to stderr. + # Both, so the tests below do not pin the policy to a message level. + return ([t for t in result.split(";") if t], + completed.stdout + completed.stderr) def test_gfx9_gets_xnack_plus_requested(self): """A bare gfx9 name compiles with the sanitizer ignored.""" @@ -77,6 +79,14 @@ def test_gfx1250_is_kept_untouched(self): self.assertEqual(result, ["gfx1250"]) + def test_gfx1250_strict_is_kept_untouched(self): + """The other entry in _RCCL_ASAN_XNACK_ALWAYS_ON, and the one no other + case reaches: drop it from that list and every test here stays green + while the target silently moves to the drop arm.""" + result, _ = self.adjust(["gfx1250-strict", "gfx1250-strict:xnack+"]) + + self.assertEqual(result, ["gfx1250-strict"]) + def test_gfx1250_loses_an_explicitly_requested_xnack(self): """clang rejects 'gfx1250:xnack+' outright: xnack is not selectable there.""" result, _ = self.adjust(["gfx1250:xnack+"]) diff --git a/projects/rocshmem/CMakeLists.txt b/projects/rocshmem/CMakeLists.txt index 60a3ce4bb3fe..1d9b36b2f4b4 100644 --- a/projects/rocshmem/CMakeLists.txt +++ b/projects/rocshmem/CMakeLists.txt @@ -254,50 +254,75 @@ endif() # normalise to gfx1250 would otherwise fail validation and be dropped before # the policy ever saw it. Normalising first also means SUPPORTED_GPUS, and so # both GPU_TARGETS and GPU_TARGETS_FULL below, are already correct. +# +# Variable names deliberately mirror the rccl copy so the two can be diffed. +set(_ROCSHMEM_ASAN_XNACK_ALWAYS_ON gfx1250 gfx1250-strict) + function(rocshmem_asan_adjust_gpu_targets OUT_VAR) - set(_always_on gfx1250 gfx1250-strict) - set(_kept "") + set(_result "") + set(_dropped "") - foreach(_target ${ARGN}) + foreach(_target IN LISTS ARGN) if(_target STREQUAL "") continue() endif() + # A requested suffix says what the caller wants, not what the part # supports, so classify on the bare name. string(REGEX REPLACE ":.*" "" _base "${_target}") - list(FIND _always_on "${_base}" _is_always_on) - # Other feature terms (e.g. sramecc+) are part of what was asked for and - # are kept; xnack sorts last, so re-appending it stays canonical. - string(REGEX REPLACE ":xnack[+-]" "" _stripped "${_target}") + list(FIND _ROCSHMEM_ASAN_XNACK_ALWAYS_ON "${_base}" _always_on) + + # Drop any xnack term the caller wrote before deciding what to do with it: + # it may be ":xnack-", which contradicts the build. Other feature terms + # (e.g. ":sramecc+") are part of the requested target and are kept; xnack + # sorts last, so re-appending it keeps the target ID canonical. + string(REGEX REPLACE ":xnack[+-]" "" _without_xnack "${_target}") if(_base MATCHES "^gfx9") - set(_fixed "${_stripped}:xnack+") - elseif(NOT _is_always_on EQUAL -1) - set(_fixed "${_stripped}") + list(APPEND _result "${_without_xnack}:xnack+") + elseif(NOT _always_on EQUAL -1) + # Always on here, so naming xnack at all is an invalid target ID. + list(APPEND _result "${_without_xnack}") else() - message(STATUS "ASAN: dropping GPU target '${_target}' - no xnack support") - continue() - endif() - - if(NOT _fixed STREQUAL _target) - message(STATUS "ASAN: GPU target '${_target}' -> '${_fixed}'") + list(APPEND _dropped "${_target}") endif() - list(APPEND _kept "${_fixed}") endforeach() # Both xnack spellings of one arch normalise to the same string. - if(_kept) - list(REMOVE_DUPLICATES _kept) + if(_result) + list(REMOVE_DUPLICATES _result) endif() - if(NOT _kept) - message(FATAL_ERROR "ASAN requires a GPU target that supports xnack (gfx9* with :xnack+, " - "or gfx1250 where it is always on). " - "None of the requested/default GPU_TARGETS do: ${ARGN}") + + if(_dropped) + message(STATUS + "ASAN: dropping GPU target(s) with no xnack support: ${_dropped}") + endif() + if(NOT _result) + message(FATAL_ERROR + "ASAN requires a GPU target that supports xnack (gfx9* with :xnack+, " + "or gfx1250 where it is always on). None of the requested GPU targets " + "do: ${ARGN}") endif() - set(${OUT_VAR} "${_kept}" PARENT_SCOPE) + message(STATUS "ASAN: GPU targets = ${_result}") + set(${OUT_VAR} "${_result}" PARENT_SCOPE) endfunction() +# GPU_TARGETS is FORCE-written from SUPPORTED_GPUS below, so whatever the +# policy produces is what the next configure of this build directory reads back +# as its input. Applying the policy to its own previous output makes it stick: +# a later -DASAN=OFF would keep the :xnack+ and would never bring back the +# families ASAN dropped. Remember the request as it arrived and re-derive from +# that every time. GPU_TARGETS matching the value we wrote last time means no +# one has touched it since, so the remembered request still stands; anything +# else is a user edit and becomes the new request. +if(DEFINED ROCSHMEM_GPU_TARGETS_DERIVED AND GPU_TARGETS STREQUAL ROCSHMEM_GPU_TARGETS_DERIVED) + set(GPU_TARGETS "${ROCSHMEM_GPU_TARGETS_REQUESTED}") +else() + set(ROCSHMEM_GPU_TARGETS_REQUESTED "${GPU_TARGETS}" CACHE INTERNAL + "GPU_TARGETS as requested, before ASAN normalisation and validation") +endif() + if(ASAN) if(GPU_TARGETS) rocshmem_asan_adjust_gpu_targets(GPU_TARGETS ${GPU_TARGETS}) @@ -342,6 +367,8 @@ else() endif() set(GPU_TARGETS "${SUPPORTED_GPUS}" CACHE STRING "GPU architectures to compile for" FORCE) +set(ROCSHMEM_GPU_TARGETS_DERIVED "${GPU_TARGETS}" CACHE INTERNAL + "Last value written to GPU_TARGETS above, so the next configure can tell a user edit from this project's own output") message(STATUS "Compiling for ${GPU_TARGETS}") @@ -419,9 +446,14 @@ if (NOT BUILD_TESTS_ONLY) # GPU_TARGETS_FULL, also without FORCE, so it goes stale the same way: a # build directory first configured without ASAN keeps the bare gfx942 and the # bitcode compile loses the :xnack+ this policy just added. Same treatment. + # The else arm mirrors the unset above: without it a suffixed value from an + # earlier configure outlives the configure that resolved to bare archs, and + # the bitcode metadata stops matching the targets being built. if(GPU_TARGETS_FULL) set(BITCODE_GPU_ARCHS_FULL "${GPU_TARGETS_FULL}" CACHE STRING "Full GPU arch strings with feature suffixes for device bitcode" FORCE) + else() + unset(BITCODE_GPU_ARCHS_FULL CACHE) endif() find_package(hip REQUIRED) From 0d178bf3d38a86bcfe9a59f31b5c3b1ce6ad5bf1 Mon Sep 17 00:00:00 2001 From: Larry Meadows Date: Sat, 26 Sep 2026 23:19:10 -0500 Subject: [PATCH 05/10] fix(rocshmem): instrument the device bitcode under ASAN cmake/DeviceBitcode.cmake invokes clang directly with its own BITCODE_COMPILE_FLAGS_BASE, so it inherited none of the ASAN flags the rest of the build picks up from CMAKE_CXX_FLAGS. Propagating -DASAN=ON therefore left librocshmem_device_*.bc uninstrumented, and rccl's device linker injects that bitcode for ENABLE_ROCSHMEM -- uninstrumented device code inside an ASAN build, which is the case the propagation exists to close. Measured on a standalone gfx950 ASAN build: librocshmem_device_gfx950.bc goes from 0 to 72 asan symbols, keeping target-features=+xnack. End to end, ./install.sh --ninja --rocshmem --address-sanitizer --amdgpu_targets gfx950 still reaches 4010/4010 with the instrumented bitcode: device.elf links at 0x74f (gfx950, xnack+, sramecc) and librccl.so is unchanged on the host side. --- projects/rocshmem/cmake/DeviceBitcode.cmake | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/projects/rocshmem/cmake/DeviceBitcode.cmake b/projects/rocshmem/cmake/DeviceBitcode.cmake index 49c73dddc6c6..0b918af4069c 100644 --- a/projects/rocshmem/cmake/DeviceBitcode.cmake +++ b/projects/rocshmem/cmake/DeviceBitcode.cmake @@ -106,6 +106,14 @@ if(${ROCM_MAJOR_VERSION} LESS 7) list(APPEND BITCODE_COMPILE_FLAGS_BASE -DHIP_ENABLE_WARP_SYNC_BUILTINS=1) endif() +# This path invokes clang directly, so it inherits none of the ASAN flags the +# rest of the build gets from CMAKE_CXX_FLAGS. Without this the bitcode rccl's +# device linker injects for ENABLE_ROCSHMEM is uninstrumented device code +# inside an ASAN build -- the case the -DASAN=ON propagation exists to close. +if(ASAN) + list(APPEND BITCODE_COMPILE_FLAGS_BASE -fsanitize=address) +endif() + # Add MPI include directories — rocshmem_config.h defines HAVE_EXTERNAL_MPI # when MPI is found, causing rocshmem_mpi.hpp to #include transitively. if(MPI_CXX_FOUND) From f197e41706c6259ff0991a159100fad542359855 Mon Sep 17 00:00:00 2001 From: Larry Meadows Date: Sun, 27 Sep 2026 21:40:11 -0500 Subject: [PATCH 06/10] fix(rocshmem): refresh BITCODE_GPU_ARCHS, move the policy to a module BITCODE_GPU_ARCHS is the list DeviceBitcode.cmake iterates, and it was the one of the three arch caches still left without FORCE. A directory configured without ASAN and then reconfigured with -DASAN=ON kept gfx1100 and gfx1201 -- the families the policy had just dropped as uninstrumentable -- and compiled them with -fsanitize=address on the bare _FULL_ARCH fallback. All three now move together in both directions. The policy moves to projects/rocshmem/cmake/AsanGpuTargets.cmake, included where it used to sit, following the cmake/rocm_local_targets.cmake precedent. Inline it could not be loaded under cmake -P, so the rocSHMEM copy had no coverage at all, which is how the earlier _always_on collision survived. The existing suite is now parameterised over both copies, 67 tests up from 55; deleting gfx1250-strict from the rocSHMEM copy turns two of them red. The rocshmem_ext options stamp moves beside the install it describes. ext/rocshmem is shared by the release and debug trees while the stamp was per-config, so with --no_clean this sequence linked an uninstrumented archive into an ASAN build: --rocshmem --address-sanitizer 1181 __asan syms in librocshmem.a --rocshmem --debug 0, debug overwrote the shared install --rocshmem --address-sanitizer 0, release stamp unchanged, step skipped With the stamp shared, the third step is 1181 again. It needs --no_clean to arm, since install.sh otherwise removes the build directory each run. --- projects/rccl/cmake/AsanGpuTargets.cmake | 6 +- projects/rccl/cmake/ROCSHMEM.cmake | 18 +-- .../tests/test_asan_gpu_targets.py | 30 ++++- projects/rocshmem/CMakeLists.txt | 112 ++++-------------- projects/rocshmem/cmake/AsanGpuTargets.cmake | 56 +++++++++ projects/rocshmem/cmake/DeviceBitcode.cmake | 7 +- 6 files changed, 123 insertions(+), 106 deletions(-) create mode 100644 projects/rocshmem/cmake/AsanGpuTargets.cmake diff --git a/projects/rccl/cmake/AsanGpuTargets.cmake b/projects/rccl/cmake/AsanGpuTargets.cmake index 86cdb8929749..2f3d7c5eb969 100644 --- a/projects/rccl/cmake/AsanGpuTargets.cmake +++ b/projects/rccl/cmake/AsanGpuTargets.cmake @@ -1,9 +1,7 @@ # GPU target selection for device AddressSanitizer builds. Device ASAN needs # xnack, which gfx9 must request explicitly, gfx1250 has always on (and rejects -# the suffix), and no other family has. Cases are pinned in -# tools/scripts/test_runner/tests/test_asan_gpu_targets.py; -# projects/rocshmem/CMakeLists.txt repeats the policy inline, being a separate -# cmake process. +# the suffix), and no other family has. Cases pinned in +# tests/test_asan_gpu_targets.py; projects/rocshmem repeats the policy inline. # Parts where xnack is always on, so naming it is an error. set(_RCCL_ASAN_XNACK_ALWAYS_ON gfx1250 gfx1250-strict) diff --git a/projects/rccl/cmake/ROCSHMEM.cmake b/projects/rccl/cmake/ROCSHMEM.cmake index e1124ea2bdbb..c5fa606a55de 100644 --- a/projects/rccl/cmake/ROCSHMEM.cmake +++ b/projects/rccl/cmake/ROCSHMEM.cmake @@ -161,14 +161,16 @@ function(add_rocshmem_targets) INSTALL_COMMAND "" ) - # ExternalProject stamps its build step on completion, not on the text - # of BUILD_COMMAND, so flipping ASAN over an existing tree leaves the - # step up to date and keeps the previous, uninstrumented - # librocshmem.a. Record the options in a file and hang the build step - # off it: file(GENERATE) rewrites only when the content differs, so the - # sub-build re-runs exactly when its options change, without the - # every-invocation cost of BUILD_ALWAYS. - set(_rocshmem_opts_stamp "${CMAKE_CURRENT_BINARY_DIR}/rocshmem_ext_options.txt") + # The build step is stamped on completion, not on BUILD_COMMAND text, + # so flipping ASAN would otherwise keep the previous librocshmem.a. + # file(GENERATE) rewrites only on a content change, so this re-runs the + # sub-build exactly then, unlike BUILD_ALWAYS. + # + # The stamp sits beside the install it describes, not in this build + # dir: ext/rocshmem is shared by the release and debug trees, so a + # per-config stamp goes unchanged while the other config overwrites + # the archive underneath it. + set(_rocshmem_opts_stamp "${ROCSHMEM_INSTALL_DIR}/rccl_build_options.txt") file(GENERATE OUTPUT "${_rocshmem_opts_stamp}" CONTENT "${CMAKE_BUILD_TYPE} ${_rocshmem_gpu_targets} ${_rocshmem_sdma_opt} ${_rocshmem_asan_opt} ${_rocshmem_cmake_opts}\n") ExternalProject_Add_StepDependencies(rocshmem_ext build "${_rocshmem_opts_stamp}") diff --git a/projects/rccl/tools/scripts/test_runner/tests/test_asan_gpu_targets.py b/projects/rccl/tools/scripts/test_runner/tests/test_asan_gpu_targets.py index 353249a729a6..61cea733343b 100644 --- a/projects/rccl/tools/scripts/test_runner/tests/test_asan_gpu_targets.py +++ b/projects/rccl/tools/scripts/test_runner/tests/test_asan_gpu_targets.py @@ -15,10 +15,18 @@ RCCL_ROOT = Path(__file__).resolve().parents[4] MODULE = RCCL_ROOT / "cmake" / "AsanGpuTargets.cmake" +# The same policy, duplicated because rccl builds rocSHMEM as a separate cmake +# process. Drift between the two shows up only as librocshmem.a built for a +# different arch set than the librccl.so it links into, so run the cases +# against both. +ROCSHMEM_MODULE = (RCCL_ROOT.parent / "rocshmem" / "cmake" / "AsanGpuTargets.cmake") @unittest.skipUnless(shutil.which("cmake"), "cmake not available on PATH") class AsanGpuTargetsTest(unittest.TestCase): + module = MODULE + function = "rccl_asan_adjust_gpu_targets" + def adjust(self, targets): """Run rccl_asan_adjust_gpu_targets over the given targets. @@ -29,8 +37,8 @@ def adjust(self, targets): out_file = Path(temp_dir) / "out.txt" script = Path(temp_dir) / "probe.cmake" script.write_text( - f'include("{MODULE}")\n' - f"rccl_asan_adjust_gpu_targets(RESULT {quoted})\n" + f'include("{self.module}")\n' + f"{self.function}(RESULT {quoted})\n" f'file(WRITE "{out_file}" "${{RESULT}}")\n' ) completed = subprocess.run( @@ -128,5 +136,23 @@ def test_empty_result_is_fatal(self): str(cm.exception)) +@unittest.skipUnless(shutil.which("cmake"), "cmake not available on PATH") +class RocshmemAsanGpuTargetsTest(AsanGpuTargetsTest): + """Every case above, run against the rocSHMEM copy of the policy.""" + + module = ROCSHMEM_MODULE + function = "rocshmem_asan_adjust_gpu_targets" + + def test_full_default_target_list(self): + """rocSHMEM's DEFAULT_GPUS differs from rccl's; see its CMakeLists.txt.""" + result, _ = self.adjust([ + "gfx90a", "gfx1100", "gfx1201", "gfx942", "gfx950", + "gfx1250", "gfx1250-strict"]) + + self.assertEqual(result, [ + "gfx90a:xnack+", "gfx942:xnack+", "gfx950:xnack+", + "gfx1250", "gfx1250-strict"]) + + if __name__ == "__main__": unittest.main() diff --git a/projects/rocshmem/CMakeLists.txt b/projects/rocshmem/CMakeLists.txt index 1d9b36b2f4b4..844a373a7512 100644 --- a/projects/rocshmem/CMakeLists.txt +++ b/projects/rocshmem/CMakeLists.txt @@ -241,81 +241,12 @@ if(GPU_TARGETS) list(REMOVE_DUPLICATES GPU_TARGETS) endif() -# Device ASAN needs xnack. gfx9 must request it explicitly, or a bare arch name -# compiles with the sanitizer silently ignored; gfx1250 has it always on and -# rejects an explicit xnack term as an invalid target ID; no other family has -# it at all. Drop what cannot be instrumented rather than ship an -# uninstrumented target inside an ASAN build. Same policy as -# projects/rccl/cmake/AsanGpuTargets.cmake, repeated here because rccl builds -# rocSHMEM as a separate cmake process. -# -# This runs before rocm_check_target_ids, not after. gfx1250:xnack+ is a hard -# "invalid target ID" error from clang, so a target this policy exists to -# normalise to gfx1250 would otherwise fail validation and be dropped before -# the policy ever saw it. Normalising first also means SUPPORTED_GPUS, and so -# both GPU_TARGETS and GPU_TARGETS_FULL below, are already correct. -# -# Variable names deliberately mirror the rccl copy so the two can be diffed. -set(_ROCSHMEM_ASAN_XNACK_ALWAYS_ON gfx1250 gfx1250-strict) - -function(rocshmem_asan_adjust_gpu_targets OUT_VAR) - set(_result "") - set(_dropped "") - - foreach(_target IN LISTS ARGN) - if(_target STREQUAL "") - continue() - endif() - - # A requested suffix says what the caller wants, not what the part - # supports, so classify on the bare name. - string(REGEX REPLACE ":.*" "" _base "${_target}") - list(FIND _ROCSHMEM_ASAN_XNACK_ALWAYS_ON "${_base}" _always_on) - - # Drop any xnack term the caller wrote before deciding what to do with it: - # it may be ":xnack-", which contradicts the build. Other feature terms - # (e.g. ":sramecc+") are part of the requested target and are kept; xnack - # sorts last, so re-appending it keeps the target ID canonical. - string(REGEX REPLACE ":xnack[+-]" "" _without_xnack "${_target}") - - if(_base MATCHES "^gfx9") - list(APPEND _result "${_without_xnack}:xnack+") - elseif(NOT _always_on EQUAL -1) - # Always on here, so naming xnack at all is an invalid target ID. - list(APPEND _result "${_without_xnack}") - else() - list(APPEND _dropped "${_target}") - endif() - endforeach() +include(cmake/AsanGpuTargets.cmake) - # Both xnack spellings of one arch normalise to the same string. - if(_result) - list(REMOVE_DUPLICATES _result) - endif() - - if(_dropped) - message(STATUS - "ASAN: dropping GPU target(s) with no xnack support: ${_dropped}") - endif() - if(NOT _result) - message(FATAL_ERROR - "ASAN requires a GPU target that supports xnack (gfx9* with :xnack+, " - "or gfx1250 where it is always on). None of the requested GPU targets " - "do: ${ARGN}") - endif() - - message(STATUS "ASAN: GPU targets = ${_result}") - set(${OUT_VAR} "${_result}" PARENT_SCOPE) -endfunction() - -# GPU_TARGETS is FORCE-written from SUPPORTED_GPUS below, so whatever the -# policy produces is what the next configure of this build directory reads back -# as its input. Applying the policy to its own previous output makes it stick: -# a later -DASAN=OFF would keep the :xnack+ and would never bring back the -# families ASAN dropped. Remember the request as it arrived and re-derive from -# that every time. GPU_TARGETS matching the value we wrote last time means no -# one has touched it since, so the remembered request still stands; anything -# else is a user edit and becomes the new request. +# GPU_TARGETS is FORCE-written from SUPPORTED_GPUS below, so the policy would +# otherwise feed on its own output and a later -DASAN=OFF would never bring +# back the dropped families. Re-derive from the request instead; still matching +# what we wrote last time means nobody edited it. if(DEFINED ROCSHMEM_GPU_TARGETS_DERIVED AND GPU_TARGETS STREQUAL ROCSHMEM_GPU_TARGETS_DERIVED) set(GPU_TARGETS "${ROCSHMEM_GPU_TARGETS_REQUESTED}") else() @@ -352,13 +283,9 @@ endif() # Consumed by DeviceBitcode.cmake to embed correct amdhsa.target metadata in # the device bitcode — HIP error 209 occurs if the suffix in the bitcode # metadata does not match the active feature configuration of the GPU. -# Keyed on whether SUPPORTED_GPUS actually carries a suffix rather than on -# whether the user set GPU_TARGETS: under ASAN the defaults path produces -# :xnack+ as well, and testing GPU_TARGETS would skip it and leave the bitcode -# compile on DeviceBitcode.cmake's bare _BITCODE_DEFAULT_ARCHS fallback. -# Explicitly cleared otherwise so a stale cached value from a previous -# -DGPU_TARGETS=... configure run cannot persist into the defaults path; that -# arm loses nothing, since the fallback is the same bare list. +# Keyed on whether SUPPORTED_GPUS carries a suffix, not on whether the user set +# GPU_TARGETS: under ASAN the defaults path produces :xnack+ too. Cleared +# otherwise, which costs nothing since the fallback is the same bare list. if(SUPPORTED_GPUS MATCHES ":") set(GPU_TARGETS_FULL "${SUPPORTED_GPUS}" CACHE STRING "Validated GPU arch strings with feature suffixes (consumed by DeviceBitcode.cmake)" FORCE) @@ -442,13 +369,9 @@ if (NOT BUILD_TESTS_ONLY) # on every configure so hip::device's --offload-arch flags stay in sync. set(GPU_BUILD_TARGETS "${GPU_TARGETS}" CACHE STRING "GPU targets to compile for" FORCE) - # cmake/DeviceBitcode.cmake derives BITCODE_GPU_ARCHS_FULL from - # GPU_TARGETS_FULL, also without FORCE, so it goes stale the same way: a - # build directory first configured without ASAN keeps the bare gfx942 and the - # bitcode compile loses the :xnack+ this policy just added. Same treatment. - # The else arm mirrors the unset above: without it a suffixed value from an - # earlier configure outlives the configure that resolved to bare archs, and - # the bitcode metadata stops matching the targets being built. + # DeviceBitcode.cmake caches this without FORCE too, so it goes stale the + # same way in both directions: a pre-ASAN directory loses the :xnack+, and + # without the else arm a suffixed value outlives a bare-arch configure. if(GPU_TARGETS_FULL) set(BITCODE_GPU_ARCHS_FULL "${GPU_TARGETS_FULL}" CACHE STRING "Full GPU arch strings with feature suffixes for device bitcode" FORCE) @@ -456,6 +379,19 @@ if (NOT BUILD_TESTS_ONLY) unset(BITCODE_GPU_ARCHS_FULL CACHE) endif() + # Same staleness, and this is the list DeviceBitcode.cmake iterates: a + # pre-ASAN directory keeps the dropped families and builds them instrumented. + set(_bitcode_archs "") + foreach(_target ${GPU_TARGETS}) + string(REGEX REPLACE ":.*" "" _bare "${_target}") + list(APPEND _bitcode_archs "${_bare}") + endforeach() + if(_bitcode_archs) + list(REMOVE_DUPLICATES _bitcode_archs) + set(BITCODE_GPU_ARCHS "${_bitcode_archs}" CACHE STRING + "GPU architectures for device bitcode (semicolon-separated)" FORCE) + endif() + find_package(hip REQUIRED) find_package(hsa-runtime64 REQUIRED) diff --git a/projects/rocshmem/cmake/AsanGpuTargets.cmake b/projects/rocshmem/cmake/AsanGpuTargets.cmake new file mode 100644 index 000000000000..95a02eb762b0 --- /dev/null +++ b/projects/rocshmem/cmake/AsanGpuTargets.cmake @@ -0,0 +1,56 @@ +# Device ASAN needs xnack: gfx9 must request it explicitly, gfx1250 has it +# always on and rejects the term, no other family has it. Mirrors +# projects/rccl/cmake/AsanGpuTargets.cmake, repeated because rccl builds +# rocSHMEM as a separate cmake process. Runs before rocm_check_target_ids, +# which rejects gfx1250:xnack+ before the policy could normalise it. +set(_ROCSHMEM_ASAN_XNACK_ALWAYS_ON gfx1250 gfx1250-strict) + +function(rocshmem_asan_adjust_gpu_targets OUT_VAR) + set(_result "") + set(_dropped "") + + foreach(_target IN LISTS ARGN) + if(_target STREQUAL "") + continue() + endif() + + # A requested suffix says what the caller wants, not what the part + # supports, so classify on the bare name. + string(REGEX REPLACE ":.*" "" _base "${_target}") + list(FIND _ROCSHMEM_ASAN_XNACK_ALWAYS_ON "${_base}" _always_on) + + # Drop any xnack term the caller wrote before deciding what to do with it: + # it may be ":xnack-", which contradicts the build. Other feature terms + # (e.g. ":sramecc+") are part of the requested target and are kept; xnack + # sorts last, so re-appending it keeps the target ID canonical. + string(REGEX REPLACE ":xnack[+-]" "" _without_xnack "${_target}") + + if(_base MATCHES "^gfx9") + list(APPEND _result "${_without_xnack}:xnack+") + elseif(NOT _always_on EQUAL -1) + # Always on here, so naming xnack at all is an invalid target ID. + list(APPEND _result "${_without_xnack}") + else() + list(APPEND _dropped "${_target}") + endif() + endforeach() + + # Both xnack spellings of one arch normalise to the same string. + if(_result) + list(REMOVE_DUPLICATES _result) + endif() + + if(_dropped) + message(STATUS + "ASAN: dropping GPU target(s) with no xnack support: ${_dropped}") + endif() + if(NOT _result) + message(FATAL_ERROR + "ASAN requires a GPU target that supports xnack (gfx9* with :xnack+, " + "or gfx1250 where it is always on). None of the requested GPU targets " + "do: ${ARGN}") + endif() + + message(STATUS "ASAN: GPU targets = ${_result}") + set(${OUT_VAR} "${_result}" PARENT_SCOPE) +endfunction() diff --git a/projects/rocshmem/cmake/DeviceBitcode.cmake b/projects/rocshmem/cmake/DeviceBitcode.cmake index 0b918af4069c..9ec2c10e27cd 100644 --- a/projects/rocshmem/cmake/DeviceBitcode.cmake +++ b/projects/rocshmem/cmake/DeviceBitcode.cmake @@ -106,10 +106,9 @@ if(${ROCM_MAJOR_VERSION} LESS 7) list(APPEND BITCODE_COMPILE_FLAGS_BASE -DHIP_ENABLE_WARP_SYNC_BUILTINS=1) endif() -# This path invokes clang directly, so it inherits none of the ASAN flags the -# rest of the build gets from CMAKE_CXX_FLAGS. Without this the bitcode rccl's -# device linker injects for ENABLE_ROCSHMEM is uninstrumented device code -# inside an ASAN build -- the case the -DASAN=ON propagation exists to close. +# This path invokes clang directly and so inherits nothing from +# CMAKE_CXX_FLAGS. Without this, the bitcode rccl's device linker injects for +# ENABLE_ROCSHMEM is uninstrumented device code inside an ASAN build. if(ASAN) list(APPEND BITCODE_COMPILE_FLAGS_BASE -fsanitize=address) endif() From dba3612444fe6acbde806b632398e3e8ba274e97 Mon Sep 17 00:00:00 2001 From: Larry Meadows Date: Mon, 28 Sep 2026 12:32:45 -0500 Subject: [PATCH 07/10] fix(rocshmem): run the asan pass on the device bitcode The bitcode compile passes -Xclang -disable-llvm-passes, so -fsanitize=address there only marked functions sanitize_address and linked the runtime in; no pass ever instrumented them. The __asan symbols counted before were the runtime's own definitions. Run asan in the opt step instead. Unix Makefiles, flipping ASAN in place: OFF: 0 __asan_report call sites ON: 127798, all 12 sources recompiled and opt re-run Also skip the rocSHMEM policy tests when projects/rocshmem is absent. --- .../scripts/test_runner/tests/test_asan_gpu_targets.py | 1 + projects/rocshmem/cmake/DeviceBitcode.cmake | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/projects/rccl/tools/scripts/test_runner/tests/test_asan_gpu_targets.py b/projects/rccl/tools/scripts/test_runner/tests/test_asan_gpu_targets.py index 61cea733343b..b00d33b0a0c4 100644 --- a/projects/rccl/tools/scripts/test_runner/tests/test_asan_gpu_targets.py +++ b/projects/rccl/tools/scripts/test_runner/tests/test_asan_gpu_targets.py @@ -137,6 +137,7 @@ def test_empty_result_is_fatal(self): @unittest.skipUnless(shutil.which("cmake"), "cmake not available on PATH") +@unittest.skipUnless(ROCSHMEM_MODULE.exists(), "projects/rocshmem not in this checkout") class RocshmemAsanGpuTargetsTest(AsanGpuTargetsTest): """Every case above, run against the rocSHMEM copy of the policy.""" diff --git a/projects/rocshmem/cmake/DeviceBitcode.cmake b/projects/rocshmem/cmake/DeviceBitcode.cmake index 9ec2c10e27cd..1f2c245f467e 100644 --- a/projects/rocshmem/cmake/DeviceBitcode.cmake +++ b/projects/rocshmem/cmake/DeviceBitcode.cmake @@ -107,10 +107,12 @@ if(${ROCM_MAJOR_VERSION} LESS 7) endif() # This path invokes clang directly and so inherits nothing from -# CMAKE_CXX_FLAGS. Without this, the bitcode rccl's device linker injects for -# ENABLE_ROCSHMEM is uninstrumented device code inside an ASAN build. +# CMAKE_CXX_FLAGS. The compile below skips the pass pipeline, so this only +# marks functions sanitize_address; the asan pass runs in the opt step. +set(_BITCODE_OPT_PASSES -O3) if(ASAN) list(APPEND BITCODE_COMPILE_FLAGS_BASE -fsanitize=address) + set(_BITCODE_OPT_PASSES "-passes=default,asan") endif() # Add MPI include directories — rocshmem_config.h defines HAVE_EXTERNAL_MPI @@ -233,7 +235,7 @@ foreach(gpu_arch ${BITCODE_GPU_ARCHS}) add_custom_command( OUTPUT ${BITCODE_OUTPUT_${gpu_arch}} - COMMAND ${LLVM_OPT} -O3 -mtriple=amdgcn-amd-amdhsa -mcpu=${gpu_arch} + COMMAND ${LLVM_OPT} ${_BITCODE_OPT_PASSES} -mtriple=amdgcn-amd-amdhsa -mcpu=${gpu_arch} ${_UNOPT_BC} -o ${BITCODE_OUTPUT_${gpu_arch}} DEPENDS ${_UNOPT_BC} COMMENT "Optimizing device bitcode for ${gpu_arch}" From 45c37bd0b8f96247e95b6e54293d4c3c1efdc497 Mon Sep 17 00:00:00 2001 From: Larry Meadows Date: Mon, 28 Sep 2026 17:03:30 -0500 Subject: [PATCH 08/10] docs(rocshmem): describe the ASAN target policy in DEBUG.md --- projects/rocshmem/DEBUG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/rocshmem/DEBUG.md b/projects/rocshmem/DEBUG.md index 3ad385b084d5..f7db6ab818ba 100644 --- a/projects/rocshmem/DEBUG.md +++ b/projects/rocshmem/DEBUG.md @@ -20,7 +20,7 @@ Refer to [General documentation for ASAN on AMD GPUs][1]. Add `-DASAN=ON` to the `cmake` invocation, whether configuring a fresh build directory or reconfiguring an existing one. `cmake . <...> -DASAN=ON` -`xnack-` targets are silently invalid under ASAN (the compiler skips instrumenting device code for them without a hard error), so rocSHMEM automatically drops any `:xnack-` entries from `GPU_TARGETS` when `ASAN=ON`, and re-derives HIP's offload targets from the result on every configure. Simply toggling `-DASAN=ON`/`-DASAN=OFF` on an existing build directory is enough; no manual CMake cache surgery is required. +Device ASAN needs xnack, and the compiler skips instrumenting device code for a target without it rather than failing. So with `ASAN=ON`, rocSHMEM builds gfx9 targets as `:xnack+` (keeping other features such as `:sramecc+`), builds gfx1250 as is since xnack is always on there, and drops every other target. It re-derives HIP's offload targets from the result on every configure. Simply toggling `-DASAN=ON`/`-DASAN=OFF` on an existing build directory is enough; no manual CMake cache surgery is required. ### Running with ASAN From 3a16cf82ba994c605b44dfc35c9d3ae30a7f2366 Mon Sep 17 00:00:00 2001 From: Larry Meadows Date: Mon, 28 Sep 2026 19:28:01 -0500 Subject: [PATCH 09/10] build(rccl): shorten the rocSHMEM options-stamp comment --- projects/rccl/cmake/ROCSHMEM.cmake | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/projects/rccl/cmake/ROCSHMEM.cmake b/projects/rccl/cmake/ROCSHMEM.cmake index c5fa606a55de..cf8e1c7fe28f 100644 --- a/projects/rccl/cmake/ROCSHMEM.cmake +++ b/projects/rccl/cmake/ROCSHMEM.cmake @@ -161,15 +161,9 @@ function(add_rocshmem_targets) INSTALL_COMMAND "" ) - # The build step is stamped on completion, not on BUILD_COMMAND text, - # so flipping ASAN would otherwise keep the previous librocshmem.a. - # file(GENERATE) rewrites only on a content change, so this re-runs the - # sub-build exactly then, unlike BUILD_ALWAYS. - # - # The stamp sits beside the install it describes, not in this build - # dir: ext/rocshmem is shared by the release and debug trees, so a - # per-config stamp goes unchanged while the other config overwrites - # the archive underneath it. + # Rebuild when the sub-build's options change (e.g. ASAN): its build + # stamp ignores BUILD_COMMAND. The stamp sits beside the shared install, + # so the other config tree overwriting librocshmem.a invalidates it too. set(_rocshmem_opts_stamp "${ROCSHMEM_INSTALL_DIR}/rccl_build_options.txt") file(GENERATE OUTPUT "${_rocshmem_opts_stamp}" CONTENT "${CMAKE_BUILD_TYPE} ${_rocshmem_gpu_targets} ${_rocshmem_sdma_opt} ${_rocshmem_asan_opt} ${_rocshmem_cmake_opts}\n") From 6b0431017713ccb70237179dd3343a04d0641e04 Mon Sep 17 00:00:00 2001 From: Larry Meadows Date: Mon, 28 Sep 2026 23:32:41 -0500 Subject: [PATCH 10/10] build(rocshmem): fail an ASAN configure that has no supported GPU target left --- projects/rocshmem/CMakeLists.txt | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/projects/rocshmem/CMakeLists.txt b/projects/rocshmem/CMakeLists.txt index 844a373a7512..db026da7bac2 100644 --- a/projects/rocshmem/CMakeLists.txt +++ b/projects/rocshmem/CMakeLists.txt @@ -262,21 +262,26 @@ if(ASAN) endif() endif() +if(GPU_TARGETS) + set(_requested_gpus ${GPU_TARGETS}) +else() + set(_requested_gpus ${DEFAULT_GPU_TARGETS}) +endif() + if(COMMAND rocm_check_target_ids) - if(GPU_TARGETS) - message(STATUS "Checking for ROCm support for GPU targets: " "${GPU_TARGETS}") - rocm_check_target_ids(SUPPORTED_GPUS TARGETS ${GPU_TARGETS}) - else() - message(STATUS "Checking for ROCm support for GPU targets: " "${DEFAULT_GPU_TARGETS}") - rocm_check_target_ids(SUPPORTED_GPUS TARGETS ${DEFAULT_GPU_TARGETS}) - endif() + message(STATUS "Checking for ROCm support for GPU targets: " "${_requested_gpus}") + rocm_check_target_ids(SUPPORTED_GPUS TARGETS ${_requested_gpus}) else() message(WARNING "Unable to check for supported GPU targets.") - if(GPU_TARGETS) - set(SUPPORTED_GPUS ${GPU_TARGETS}) - else() - set(SUPPORTED_GPUS ${DEFAULT_GPU_TARGETS}) - endif() + set(SUPPORTED_GPUS ${_requested_gpus}) +endif() + +# The ASAN policy already failed if it kept nothing; this catches the compiler +# rejecting everything it kept, e.g. because of conflicting CXXFLAGS. +if(ASAN AND NOT SUPPORTED_GPUS) + message(FATAL_ERROR + "ASAN requires a GPU target that supports xnack, but none of the " + "requested GPU targets passed rocm_check_target_ids: ${_requested_gpus}") endif() # GPU_TARGETS_FULL: validated arch strings with feature suffixes intact.