feat(compiler): Enable LLVM_ENABLE_PER_TARGET_RUNTIME_DIR

build_tools/build_python_packages.py

@@ -23,6 +23,7 @@
 import argparse
 import functools
 import json
+import subprocess
 from pathlib import Path
 import sys
 
@@ -79,6 +80,42 @@ def ensure_profiler_library_symlinks(profiler: PopulatedDistPackage) -> None:
                 link.symlink_to(target.name)
 
 
+def discover_llvm_host_triple(artifacts: ArtifactCatalog) -> str | None:
+    """Discover the LLVM host triple for libomp's per-target runtime dir.
+
+    With LLVM_ENABLE_PER_TARGET_RUNTIME_DIR=ON, libomp.so is installed under
+    lib/llvm/lib/<host-triple>/ instead of lib/llvm/lib/.
+    """
+    for an, basedir in artifacts.artifact_basedirs:
+        if an.name != "amd-llvm" or an.component != "lib":
+            continue
+
+        llvm_lib = basedir / "lib" / "llvm" / "lib"
+        for libomp in sorted(llvm_lib.glob("*/libomp.so")):
+            return libomp.parent.name
+
+        for clang_name in ("amdclang", "clang"):
+            clang = basedir / "lib" / "llvm" / "bin" / clang_name
+            if not clang.is_file():
+                clang = basedir / "bin" / clang_name
+            if not clang.is_file():
+                continue
+            try:
+                triple = subprocess.run(
+                    [str(clang), "--print-target-triple"],
+                    capture_output=True,
+                    text=True,
+                    timeout=10,
+                    check=True,
+                ).stdout.strip()
+            except (subprocess.SubprocessError, OSError):
+                continue
+            if triple and (llvm_lib / triple / "libomp.so").is_file():
+                return triple
+
+    return None
+
+
 def _platform_targets(
     *,
     linux_targets: list[str] | None,
@@ -233,9 +270,13 @@ def run(args: argparse.Namespace):
         windows_target_families=windows_targets,
     )
 
+    host_triple = discover_llvm_host_triple(artifacts)
+
     # Populate each target neutral library package.
     core = PopulatedDistPackage(params, logical_name="core")
     core.rpath_dep(core, "lib/llvm/lib")
+    if host_triple:
+        core.rpath_dep(core, f"lib/llvm/lib/{host_triple}")
     core.rpath_dep(core, "lib/rocm_sysdeps/lib")
     core.populate_runtime_files(
         params.filter_artifacts(
@@ -265,6 +306,8 @@ def run(args: argparse.Namespace):
         profiler = PopulatedDistPackage(params, logical_name="profiler")
         profiler.rpath_dep(core, "lib")
         profiler.rpath_dep(core, "lib/llvm/lib")
+        if host_triple:
+            profiler.rpath_dep(core, f"lib/llvm/lib/{host_triple}")
         profiler.rpath_dep(core, "lib/rocm_sysdeps/lib")
         profiler.populate_runtime_files(profiler_artifacts)
         ensure_profiler_library_symlinks(profiler)
@@ -295,9 +338,9 @@ def run(args: argparse.Namespace):
         )
 
     if kpack_split:
-        _run_kpack_split(args, params, core)
+        _run_kpack_split(args, params, core, host_triple)
     else:
-        _run_legacy(args, params, core)
+        _run_legacy(args, params, core, host_triple)
 
     if args.build_packages:
         validate_required_dist_packages(
@@ -315,7 +358,10 @@ def run(args: argparse.Namespace):
 
 
 def _run_kpack_split(
-    args: argparse.Namespace, params: Parameters, core: PopulatedDistPackage
+    args: argparse.Namespace,
+    params: Parameters,
+    core: PopulatedDistPackage,
+    host_triple: str | None,
 ):
     """Kpack-split mode: arch-neutral host libraries + per-ISA device wheels."""
 
@@ -324,8 +370,10 @@ def _run_kpack_split(
     lib.rpath_dep(core, "lib")
     lib.rpath_dep(core, "lib/rocm_sysdeps/lib")
     lib.rpath_dep(core, "lib/host-math/lib")
-    # rpp needs libomp, which ships in core under lib/llvm/lib.
+    # rpp needs libomp, which ships in core under lib/llvm/lib[/<host-triple>].
     lib.rpath_dep(core, "lib/llvm/lib")
+    if host_triple:
+        lib.rpath_dep(core, f"lib/llvm/lib/{host_triple}")
     lib.populate_runtime_files(
         params.filter_artifacts(
             filter=functools.partial(libraries_artifact_filter, "generic"),
@@ -402,7 +450,10 @@ def _run_kpack_split(
 
 
 def _run_legacy(
-    args: argparse.Namespace, params: Parameters, core: PopulatedDistPackage
+    args: argparse.Namespace,
+    params: Parameters,
+    core: PopulatedDistPackage,
+    host_triple: str | None,
 ):
     """Legacy mode: per-family libraries wheels with embedded device code."""
 
@@ -414,8 +465,10 @@ def _run_legacy(
         lib.rpath_dep(core, "lib")
         lib.rpath_dep(core, "lib/rocm_sysdeps/lib")
         lib.rpath_dep(core, "lib/host-math/lib")
-        # rpp needs libomp, which ships in core under lib/llvm/lib.
+        # rpp needs libomp, which ships in core under lib/llvm/lib[/<host-triple>].
         lib.rpath_dep(core, "lib/llvm/lib")
+        if host_triple:
+            lib.rpath_dep(core, f"lib/llvm/lib/{host_triple}")
         lib.populate_runtime_files(
             params.filter_artifacts(
                 filter=functools.partial(libraries_artifact_filter, target_family),

build_tools/github_actions/test_executable_scripts/test_rocprofiler_sdk.py

@@ -129,7 +129,7 @@ def setup_env():
     environ_vars["HIP_PLATFORM"] = "amd"
 
     ld_lib_paths = [f"{THEROCK_LIB_PATH}", f"{THEROCK_SYSDEPS_LIB_PATH}"]
-    if host_triple:
+    if host_triple and THEROCK_LLVM_LIB_HOST_TRIPLE_PATH.exists():
         ld_lib_paths += [f"{THEROCK_LLVM_LIB_HOST_TRIPLE_PATH}"]
 
     if is_asan():

build_tools/github_actions/test_executable_scripts/test_rocprofiler_systems.py

@@ -10,9 +10,30 @@
 THEROCK_BIN_DIR = os.getenv("THEROCK_BIN_DIR")
 THEROCK_BIN_PATH = Path(THEROCK_BIN_DIR).resolve()
 THEROCK_PATH = THEROCK_BIN_PATH.parent
-THEROCK_LIB_PATH = str(THEROCK_PATH / "lib")
+THEROCK_LIB_PATH = THEROCK_PATH / "lib"
+THEROCK_CLANG_PATH = THEROCK_LIB_PATH / "llvm" / "bin" / "amdclang"
 ROCPROFSYS_TEST_DIR = THEROCK_PATH / "share" / "rocprofiler-systems" / "tests"
 
+# Determine host triple
+host_triple = ""
+if THEROCK_CLANG_PATH.exists():
+    try:
+        host_triple = subprocess.run(
+            [str(THEROCK_CLANG_PATH), "--print-target-triple"],
+            capture_output=True,
+            text=True,
+            timeout=10,
+            check=True,
+        ).stdout.strip()
+    except (subprocess.SubprocessError, OSError) as exc:
+        raise RuntimeError(
+            f"'{THEROCK_CLANG_PATH} --print-target-triple' failed; "
+            "this suggests a broken toolchain."
+        ) from exc
+
+if host_triple:
+    THEROCK_LLVM_LIB_HOST_TRIPLE_PATH = THEROCK_LIB_PATH / "llvm" / "lib" / host_triple
+
 # These tests are always excluded until the relevant issue is fixed (AIPROFSYST-441)
 EXCLUDED_TESTS = [
     "transferbench-sys-run",
@@ -71,6 +92,8 @@ def setup_env():
     ld_paths = [
         str(THEROCK_PATH / "share" / "rocprofiler-systems" / "examples" / "lib"),
     ]
+    if host_triple and THEROCK_LLVM_LIB_HOST_TRIPLE_PATH.exists():
+        ld_paths += [f"{THEROCK_LLVM_LIB_HOST_TRIPLE_PATH}"]
     ld_paths_str = ":".join(ld_paths)
     old_ld_path = os.getenv("LD_LIBRARY_PATH", "")
     environ_vars["LD_LIBRARY_PATH"] = (

build_tools/github_actions/test_executable_scripts/test_rpp.py

@@ -15,10 +15,31 @@
     )
     sys.exit(1)
 THEROCK_BIN_DIR = Path(THEROCK_BIN_DIR_STR)
+THEROCK_LIB_DIR = THEROCK_BIN_DIR.resolve().parent / "lib"
+THEROCK_CLANG_PATH = THEROCK_LIB_DIR / "llvm" / "bin" / "amdclang"
 SCRIPT_DIR = Path(__file__).resolve().parent
 THEROCK_DIR = SCRIPT_DIR.parent.parent.parent
 THEROCK_TEST_DIR = Path(THEROCK_DIR) / "build"
 
+# Determine host triple
+host_triple = ""
+if THEROCK_CLANG_PATH.exists():
+    try:
+        host_triple = subprocess.run(
+            [str(THEROCK_CLANG_PATH), "--print-target-triple"],
+            capture_output=True,
+            text=True,
+            timeout=10,
+            check=True,
+        ).stdout.strip()
+    except (subprocess.SubprocessError, OSError) as exc:
+        raise RuntimeError(
+            f"'{THEROCK_CLANG_PATH} --print-target-triple' failed; "
+            "this suggests a broken toolchain."
+        ) from exc
+if host_triple:
+    THEROCK_LLVM_LIB_HOST_TRIPLE_PATH = THEROCK_LIB_DIR / "llvm" / "lib" / host_triple
+
 RPP_TEST_PATH = str(Path(THEROCK_BIN_DIR).resolve().parent / "share" / "rpp" / "test")
 if not os.path.isdir(RPP_TEST_PATH):
     logging.info(f"++ Error: rpp tests not found in {RPP_TEST_PATH}")
@@ -62,6 +83,13 @@ def setup_env(env):
             env["LD_LIBRARY_PATH"] = f"{HIP_LIB_PATH}:{env['LD_LIBRARY_PATH']}"
         else:
             env["LD_LIBRARY_PATH"] = str(HIP_LIB_PATH)
+        if host_triple and THEROCK_LLVM_LIB_HOST_TRIPLE_PATH.exists():
+            logging.info(
+                f"++ rpp prepending LD_LIBRARY_PATH with {THEROCK_LLVM_LIB_HOST_TRIPLE_PATH}"
+            )
+            env["LD_LIBRARY_PATH"] = (
+                f"{THEROCK_LLVM_LIB_HOST_TRIPLE_PATH}:{env['LD_LIBRARY_PATH']}"
+            )
     else:
         logging.info("++ rpp tests only supported on Linux")
         sys.exit(0)

cmake/therock_subproject_dep_provider.cmake

@@ -142,14 +142,36 @@ if(THEROCK_INCLUDE_CLANG_RESOURCE_DIR_RPATH)
     # This will yield an absolute path like: /some/long/path/lib/llvm/lib/clang/20
     # The version number on the end is all we want as we will form the correct
     # prefix relative path from that.
-    # The actual libraries live under lib/linux in that directory. We shamelessly
-    # just hardcode "linux" since that is the only system we do RPATH munging for.
     cmake_path(GET _abs_resource_dir FILENAME _clang_version)
     set(_prefix_resource_dir "lib/llvm/lib/clang/${_clang_version}")
-    list(APPEND THEROCK_PRIVATE_INSTALL_RPATH_DIRS "${_prefix_resource_dir}/lib/linux")
+    # Which sub-directory of the resource dir holds the runtimes depends on how
+    # the toolchain was configured: LLVM_ENABLE_PER_TARGET_RUNTIME_DIR (which
+    # TheRock enables for its own amd-llvm) installs them to lib/<target-triple>
+    # and leaves no lib/linux at all, so ask clang rather than assuming. A
+    # toolchain can also report a per-target dir that only holds non-sanitizer
+    # runtimes (i.e. flang_rt) while compiler-rt still lives in lib/linux, so
+    # only trust the reported dir if it actually has shared sanitizer runtimes.
+    execute_process(
+      COMMAND "${CMAKE_CXX_COMPILER}" --print-runtime-dir
+      OUTPUT_VARIABLE _abs_runtime_dir
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+      RESULT_VARIABLE _runtime_dir_status
+    )
+    set(_runtime_libs)
+    if(_runtime_dir_status EQUAL 0 AND _abs_runtime_dir)
+      file(GLOB _runtime_libs "${_abs_runtime_dir}/libclang_rt.*.so")
+    endif()
+    if(NOT _runtime_libs)
+      set(_abs_runtime_dir "${_abs_resource_dir}/lib/linux")
+      message(STATUS "No shared sanitizer runtimes found via ${CMAKE_CXX_COMPILER} --print-runtime-dir: falling back to ${_abs_runtime_dir}")
+    endif()
+    # The install RPATH is prefix relative, so mirror the same sub-directory.
+    cmake_path(RELATIVE_PATH _abs_runtime_dir BASE_DIRECTORY "${_abs_resource_dir}"
+      OUTPUT_VARIABLE _rel_runtime_dir)
+    list(APPEND THEROCK_PRIVATE_INSTALL_RPATH_DIRS "${_prefix_resource_dir}/${_rel_runtime_dir}")
     # Build tree needs absolute paths to the resource dir.
-    list(APPEND THEROCK_PRIVATE_BUILD_RPATH_DIRS "${_abs_resource_dir}/lib/linux")
-    message(STATUS "Added clang resource dir to RPATH: ${_prefix_resource_dir} (since sanitizer enabled)")
+    list(APPEND THEROCK_PRIVATE_BUILD_RPATH_DIRS "${_abs_runtime_dir}")
+    message(STATUS "Added clang resource dir to RPATH: ${_prefix_resource_dir}/${_rel_runtime_dir} (since sanitizer enabled)")
   endblock()
 endif()
 
@@ -158,7 +180,7 @@ endif()
 # Needed for both ASAN and HOST_ASAN modes.
 block(PROPAGATE THEROCK_SANITIZER_LAUNCHER)
   set(THEROCK_SANITIZER_LAUNCHER)
-  if(LINUX AND THEROCK_SANITIZER)
+  if(LINUX AND THEROCK_SANITIZER AND CMAKE_CXX_COMPILER_ID MATCHES "Clang")
     if(THEROCK_SANITIZER STREQUAL "ASAN" OR THEROCK_SANITIZER STREQUAL "HOST_ASAN")
       set(_suffix "asan")
     elseif(THEROCK_SANITIZER STREQUAL "TSAN" OR THEROCK_SANITIZER STREQUAL "HOST_TSAN")
@@ -171,12 +193,57 @@ block(PROPAGATE THEROCK_SANITIZER_LAUNCHER)
     else()
       message(FATAL_ERROR "Unknown processor ${CMAKE_SYSTEM_PROCESSOR}")
     endif()
-    execute_process(
-      COMMAND ${CMAKE_CXX_COMPILER} "--print-file-name=libclang_rt.${_suffix}-${_arch_suffix}.so"
-      OUTPUT_VARIABLE _rt_path
-      OUTPUT_STRIP_TRAILING_WHITESPACE
-      COMMAND_ERROR_IS_FATAL ANY
-    )
+    # With LLVM_ENABLE_PER_TARGET_RUNTIME_DIR the runtime is named
+    # libclang_rt.<sanitizer>.so with no arch suffix, so probe both names. Since
+    # clang echoes the query back verbatim when it cannot find the file, only an
+    # absolute path that exists counts as resolved: a bare soname would make the
+    # LD_PRELOAD silently fail and run the tool with no sanitizer at all.
+    set(_rt_names "libclang_rt.${_suffix}.so" "libclang_rt.${_suffix}-${_arch_suffix}.so")
+    set(_rt_path)
+    set(_rt_probe_log)
+    foreach(_rt_name ${_rt_names})
+      execute_process(
+        COMMAND ${CMAKE_CXX_COMPILER} "--print-file-name=${_rt_name}"
+        OUTPUT_VARIABLE _rt_candidate
+        OUTPUT_STRIP_TRAILING_WHITESPACE
+        COMMAND_ERROR_IS_FATAL ANY
+      )
+      string(APPEND _rt_probe_log "\n      --print-file-name=${_rt_name} -> ${_rt_candidate}")
+      if(IS_ABSOLUTE "${_rt_candidate}" AND EXISTS "${_rt_candidate}")
+        set(_rt_path "${_rt_candidate}")
+        break()
+      endif()
+    endforeach()
+    if(NOT _rt_path)
+      # Only pay for the extra diagnostic queries on the failure path.
+      execute_process(
+        COMMAND "${CMAKE_CXX_COMPILER}" --print-resource-dir
+        OUTPUT_VARIABLE _rt_resource_dir
+        OUTPUT_STRIP_TRAILING_WHITESPACE
+        ERROR_QUIET
+      )
+      execute_process(
+        COMMAND "${CMAKE_CXX_COMPILER}" --print-runtime-dir
+        OUTPUT_VARIABLE _rt_runtime_dir
+        OUTPUT_STRIP_TRAILING_WHITESPACE
+        ERROR_QUIET
+      )
+      message(FATAL_ERROR
+        "Could not resolve the shared ${THEROCK_SANITIZER} runtime library required to "
+        "run build time tools under the sanitizer.\n"
+        "    Sanitizer   : ${THEROCK_SANITIZER}\n"
+        "    Compiler    : ${CMAKE_CXX_COMPILER}\n"
+        "    Resource dir: ${_rt_resource_dir}\n"
+        "    Runtime dir : ${_rt_runtime_dir}\n"
+        "    Probed sonames (clang echoes the query back verbatim when it cannot find "
+        "the file, so a bare soname below means NOT FOUND):${_rt_probe_log}\n"
+        "Provide a shared sanitizer runtime for this toolchain (compiler-rt built with "
+        "COMPILER_RT_BUILD_SANITIZERS=ON) or disable the sanitizer for this build. "
+        "Refusing to continue: an unresolvable LD_PRELOAD is ignored by the loader, so "
+        "carrying on here would silently run build time tools with no sanitizer at all "
+        "rather than failing."
+      )
+    endif()
     set(THEROCK_SANITIZER_LAUNCHER
       "${CMAKE_COMMAND}" -E env "LD_PRELOAD=${_rt_path}" --
     )