diff --git a/pkgs/development/rocm-modules/10/aqlprofile/default.nix b/pkgs/development/rocm-modules/10/aqlprofile/default.nix new file mode 100644 index 0000000000000..84e13ff75271a --- /dev/null +++ b/pkgs/development/rocm-modules/10/aqlprofile/default.nix @@ -0,0 +1,42 @@ +{ + lib, + stdenv, + clr, + cmake, + fetchFromGitHub, + rocmUpdateScript, +}: + +stdenv.mkDerivation (finalAttrs: { + pname = "aqlprofile"; + version = "10.1"; + + src = fetchFromGitHub { + owner = "ROCm"; + repo = "rocm-systems"; + tag = "therock-${finalAttrs.version}"; + sparseCheckout = [ + "projects/aqlprofile" + "shared" + ]; + hash = "sha256-7yeEJoyShSCwy1MTCeArYBabh5qTVm5uZL8N74rvxPA="; + }; + sourceRoot = "${finalAttrs.src.name}/projects/aqlprofile"; + + env.CXXFLAGS = "-DROCP_LD_AQLPROFILE=1"; + + nativeBuildInputs = [ + cmake + clr + ]; + + passthru.updateScript = rocmUpdateScript { inherit finalAttrs; }; + + meta = { + description = "AQLPROFILE library for AMD HSA runtime API extension support"; + homepage = "https://github.com/ROCm/rocm-systems/tree/develop/projects/aqlprofile"; + license = lib.licenses.mit; + teams = [ lib.teams.rocm ]; + platforms = lib.platforms.linux; + }; +}) diff --git a/pkgs/development/rocm-modules/10/default.nix b/pkgs/development/rocm-modules/10/default.nix index fc71188cf36b6..ca6fa75dbb7ff 100644 --- a/pkgs/development/rocm-modules/10/default.nix +++ b/pkgs/development/rocm-modules/10/default.nix @@ -63,6 +63,10 @@ let inherit (self) rocmUpdateScript; }; + aqlprofile = self.callPackage ./aqlprofile { }; + + rdc = self.callPackage ./rdc { }; + rocm-docs-core = python3Packages.callPackage ../rocm-docs-core { }; hip-common = self.callPackage ./hip-common { }; @@ -78,12 +82,33 @@ let # hsakmt was merged into rocm-runtime hsakmt = self.rocm-runtime; + + rocprofiler = self.callPackage ./rocprofiler { + inherit (llvm) clang; + }; rocprofiler-register = self.callPackage ./rocprofiler-register { inherit (llvm) clang; }; + rocprofiler-sdk = self.callPackage ./rocprofiler-sdk { }; + + rocprof-compute-viewer = self.callPackage ../rocprof-compute-viewer { }; + + rocprof-trace-decoder = self.callPackage ./rocprof-trace-decoder { }; roctracer = self.callPackage ./roctracer { }; + rocgdb = self.callPackage ./rocgdb { }; + + rocdbgapi = self.callPackage ./rocdbgapi { }; + + rocr-debug-agent = self.callPackage ./rocr-debug-agent { }; + + mscclpp = self.callPackage ../mscclpp { }; + + rccl = self.callPackage ./rccl { }; + + rocshmem = self.callPackage ./rocshmem { }; + half = self.callPackage ./half { }; # Even if config.rocmSupport is false we need rocmSupport true diff --git a/pkgs/development/rocm-modules/10/rccl/default.nix b/pkgs/development/rocm-modules/10/rccl/default.nix new file mode 100644 index 0000000000000..2269b178b9283 --- /dev/null +++ b/pkgs/development/rocm-modules/10/rccl/default.nix @@ -0,0 +1,153 @@ +{ + lib, + stdenv, + fetchFromGitHub, + rocmUpdateScript, + cmake, + rocm-cmake, + amdsmi, + rocm-core, + pkg-config, + clr, + perl, + hipify, + python3, + fmt, + gtest, + chrpath, + roctracer, + rocprofiler, + rocprofiler-register, + autoPatchelfHook, + buildTests ? false, + gpuTargets ? (clr.localGpuTargets or [ ]), + # for passthru.tests + rccl, +}: + +let + useAsan = buildTests; + useUbsan = buildTests; + san = lib.optionalString (useAsan || useUbsan) ( + "-fno-gpu-sanitize -fsanitize=undefined " + + (lib.optionalString useAsan "-fsanitize=address -shared-libsan ") + ); +in +# Note: we can't properly test or make use of multi-node collective ops +# https://github.com/NixOS/nixpkgs/issues/366242 tracks kernel support +# kfd_peerdirect support which is on out-of-tree amdkfd in ROCm/ROCK-Kernel-Driver +# infiniband ib_peer_mem support isn't in the mainline kernel but is carried by some distros +stdenv.mkDerivation (finalAttrs: { + pname = "rccl${clr.gpuArchSuffix}"; + version = "10.1"; + + outputs = [ + "out" + ] + ++ lib.optionals buildTests [ + "test" + ]; + + patches = [ + ./rccl-test-missing-iomanip.diff + ]; + + src = fetchFromGitHub { + owner = "ROCm"; + repo = "rocm-systems"; + tag = "therock-${finalAttrs.version}"; + sparseCheckout = [ + "projects/rccl" + "shared" + ]; + hash = "sha256-OkX15/P1wQ9E8QfAOklXvxmVw700kvIyi6CwdorqNhA="; + }; + sourceRoot = "${finalAttrs.src.name}/projects/rccl"; + + requiredSystemFeatures = [ "big-parallel" ]; # Very resource intensive LTO + + nativeBuildInputs = [ + cmake + rocm-cmake + clr + perl + hipify + python3 + pkg-config + autoPatchelfHook # ASAN doesn't add rpath without this + ]; + + buildInputs = [ + # Headers only; RCCL dlopens amd_smi at runtime. + amdsmi + fmt + gtest + roctracer + rocprofiler + rocprofiler-register + ] + ++ lib.optionals buildTests [ + chrpath + ]; + + cmakeFlags = [ + "-DHIP_CLANG_NUM_PARALLEL_JOBS=4" + "-DROCM_PATH=${clr}" + "-DHIP_COMPILER=${clr}/bin/amdclang++" + "-DCMAKE_CXX_COMPILER=${clr}/bin/amdclang++" + "-DROCM_PATCH_VERSION=${rocm-core.ROCM_LIBPATCH_VERSION}" + "-DROCM_VERSION=${rocm-core.ROCM_LIBPATCH_VERSION}" + "-DBUILD_BFD=OFF" # Can't get it to detect bfd.h + # Manually define CMAKE_INSTALL_ + # See: https://github.com/NixOS/nixpkgs/pull/197838 + "-DCMAKE_INSTALL_BINDIR=bin" + "-DCMAKE_INSTALL_LIBDIR=lib" + "-DCMAKE_INSTALL_INCLUDEDIR=include" + ] + ++ lib.optionals (gpuTargets != [ ]) [ + # AMD can't make up their minds and keep changing which one is used in different projects. + "-DAMDGPU_TARGETS=${lib.concatStringsSep ";" gpuTargets}" + "-DGPU_TARGETS=${lib.concatStringsSep ";" gpuTargets}" + ] + ++ lib.optionals buildTests [ + "-DBUILD_TESTS=ON" + ]; + + # -O2 and -fno-strict-aliasing due to UB issues in RCCL :c + # Reported upstream + env.CFLAGS = "-I${clr}/include -I${roctracer}/include -O2 -fno-strict-aliasing ${san}-fno-omit-frame-pointer -momit-leaf-frame-pointer"; + env.CXXFLAGS = "-I${clr}/include -I${roctracer}/include -O2 -fno-strict-aliasing ${san}-fno-omit-frame-pointer -momit-leaf-frame-pointer"; + env.LDFLAGS = "${san}"; + postPatch = '' + patchShebangs src tools + ''; + + postInstall = + lib.optionalString useAsan '' + patchelf --add-needed ${clr}/llvm/lib/linux/libclang_rt.asan-${stdenv.hostPlatform.parsed.cpu.name}.so $out/lib/librccl.so + '' + + lib.optionalString buildTests '' + mkdir -p $test/bin + mv $out/bin/* $test/bin + rmdir $out/bin + ''; + + passthru.updateScript = rocmUpdateScript { inherit finalAttrs; }; + + # This package with sanitizers + manual integration test binaries built + # must be ran manually + passthru.tests.rccl = rccl.override { + buildTests = true; + }; + + meta = { + description = "ROCm communication collectives library"; + homepage = "https://github.com/ROCm/rccl"; + license = with lib.licenses; [ + bsd2 + bsd3 + ]; + teams = [ lib.teams.rocm ]; + platforms = lib.platforms.linux; + }; +}) diff --git a/pkgs/development/rocm-modules/10/rccl/rccl-test-missing-iomanip.diff b/pkgs/development/rocm-modules/10/rccl/rccl-test-missing-iomanip.diff new file mode 100644 index 0000000000000..3cc54a83c1429 --- /dev/null +++ b/pkgs/development/rocm-modules/10/rccl/rccl-test-missing-iomanip.diff @@ -0,0 +1,10 @@ +--- a/test/common/TestBed.cpp ++++ b/test/common/TestBed.cpp +@@ -4,6 +4,7 @@ + * See LICENSE.txt for license information + ************************************************************************/ + #include ++#include + #include "TestBed.hpp" + #include + diff --git a/pkgs/development/rocm-modules/10/rdc/default.nix b/pkgs/development/rocm-modules/10/rdc/default.nix new file mode 100644 index 0000000000000..fe03b7f0510fd --- /dev/null +++ b/pkgs/development/rocm-modules/10/rdc/default.nix @@ -0,0 +1,148 @@ +{ + lib, + stdenv, + fetchFromGitHub, + rocmUpdateScript, + cmake, + pkg-config, + amdsmi, + rocm-smi, + rocm-runtime, + libcap, + libdrm, + grpc, + protobuf, + openssl, + doxygen, + graphviz, + texliveSmall, + gtest, + buildDocs ? true, + buildTests ? false, +}: + +let + latex = lib.optionalAttrs buildDocs ( + texliveSmall.withPackages ( + ps: with ps; [ + changepage + latexmk + varwidth + multirow + hanging + adjustbox + collectbox + stackengine + enumitem + alphalph + wasysym + sectsty + tocloft + newunicodechar + etoc + helvetic + wasy + courier + ] + ) + ); +in +stdenv.mkDerivation (finalAttrs: { + pname = "rdc"; + version = "10.1"; + + outputs = [ + "out" + ] + ++ lib.optionals buildDocs [ + "doc" + ] + ++ lib.optionals buildTests [ + "test" + ]; + + src = fetchFromGitHub { + owner = "ROCm"; + repo = "rocm-systems"; + tag = "therock-${finalAttrs.version}"; + sparseCheckout = [ + "projects/rdc" + "shared" + ]; + hash = "sha256-tcov375DXHX1L/HhACevulzBOhq7Gp08j8SDEuVgPYU="; + }; + sourceRoot = "${finalAttrs.src.name}/projects/rdc"; + + patches = [ + # Find libcap through pkg-config (https://github.com/ROCm/rocm-systems/pull/2424, closed) + ./fix-libcap-pkgconfig.patch + ]; + + nativeBuildInputs = [ + cmake + pkg-config + protobuf + ] + ++ lib.optionals buildDocs [ + doxygen + graphviz + latex + ]; + + buildInputs = [ + amdsmi + rocm-smi + rocm-runtime + libcap + libdrm + grpc + openssl + ] + ++ lib.optionals buildTests [ + gtest + ]; + + cmakeFlags = [ + "-DCMAKE_VERBOSE_MAKEFILE=ON" + "-DRDC_INSTALL_PREFIX=${placeholder "out"}" + "-DBUILD_RVS=OFF" # TODO: Needs RVS package + "-DBUILD_ROCRTEST=ON" + "-DRSMI_INC_DIR=${rocm-smi}/include" + "-DRSMI_LIB_DIR=${rocm-smi}/lib" + "-DGRPC_ROOT=${grpc}" + # Manually define CMAKE_INSTALL_ + # See: https://github.com/NixOS/nixpkgs/pull/197838 + "-DCMAKE_INSTALL_BINDIR=bin" + "-DCMAKE_INSTALL_LIBDIR=lib" + "-DCMAKE_INSTALL_INCLUDEDIR=include" + "-DCMAKE_INSTALL_LIBEXECDIR=libexec" + "-DCMAKE_INSTALL_DOCDIR=doc" + ] + ++ lib.optionals buildTests [ + "-DBUILD_TESTS=ON" + ]; + + postPatch = '' + substituteInPlace CMakeLists.txt \ + --replace-fail "file(STRINGS /etc/os-release LINUX_DISTRO LIMIT_COUNT 1 REGEX \"NAME=\")" "set(LINUX_DISTRO \"NixOS\")" + ''; + + postInstall = '' + find $out/bin -executable -type f -exec \ + patchelf {} --shrink-rpath --allowed-rpath-prefixes "$NIX_STORE" \; + '' + + lib.optionalString buildTests '' + mkdir -p $test + mv $out/bin/rdctst_tests $test/bin + ''; + + passthru.updateScript = rocmUpdateScript { inherit finalAttrs; }; + + meta = { + description = "Simplifies administration and addresses infrastructure challenges in cluster and datacenter environments"; + homepage = "https://github.com/ROCm/rocm-systems/tree/develop/projects/rdc"; + license = lib.licenses.mit; + teams = [ lib.teams.rocm ]; + platforms = lib.platforms.linux; + }; +}) diff --git a/pkgs/development/rocm-modules/10/rdc/fix-libcap-pkgconfig.patch b/pkgs/development/rocm-modules/10/rdc/fix-libcap-pkgconfig.patch new file mode 100644 index 0000000000000..54e7303ce88f3 --- /dev/null +++ b/pkgs/development/rocm-modules/10/rdc/fix-libcap-pkgconfig.patch @@ -0,0 +1,57 @@ +From 299f2c770ff0abf07b8f394f154168d2aa78aa60 Mon Sep 17 00:00:00 2001 +From: Luna Nova +Date: Sun, 21 Dec 2025 08:23:01 -0800 +Subject: [PATCH] fix(rdc): use pkg-config to find libcap + +If libcap isn't available in system wide FHS include paths building would fail. +Fix this by using pkg-config to find libcap which provides both the library +to link and include dirs. +--- + CMakeLists.txt | 6 +++--- + rdc_libs/rdc/CMakeLists.txt | 2 +- + server/CMakeLists.txt | 2 +- + 3 files changed, 5 insertions(+), 5 deletions(-) + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 907c9fbff42..00e0a9dd87d 100755 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -192,9 +192,9 @@ mark_as_advanced( + CPACK_GENERATOR + ) + +-# check if libcap exists +-# needed for sys/capabilities.h +-find_library(LIB_CAP NAMES cap REQUIRED) ++# needed for sys/capability.h ++find_package(PkgConfig REQUIRED) ++pkg_check_modules(LIBCAP REQUIRED IMPORTED_TARGET libcap) + + if(BUILD_STANDALONE AND GRPC_ROOT STREQUAL GRPC_ROOT_DEFAULT) + message( +diff --git a/rdc_libs/rdc/CMakeLists.txt b/rdc_libs/rdc/CMakeLists.txt +index d4e368edca5..f9a3ac4c487 100644 +--- a/rdc_libs/rdc/CMakeLists.txt ++++ b/rdc_libs/rdc/CMakeLists.txt +@@ -71,7 +71,7 @@ set(RDC_LIB_INC_LIST + message("RDC_LIB_INC_LIST=${RDC_LIB_INC_LIST}") + + add_library(${RDC_LIB} SHARED ${RDC_LIB_SRC_LIST} ${RDC_LIB_INC_LIST}) +-target_link_libraries(${RDC_LIB} ${BOOTSTRAP_LIB} pthread amd_smi cap) ++target_link_libraries(${RDC_LIB} ${BOOTSTRAP_LIB} pthread amd_smi PkgConfig::LIBCAP) + target_include_directories( + ${RDC_LIB} + PRIVATE "${PROJECT_SOURCE_DIR}" "${PROJECT_SOURCE_DIR}/include" "${AMD_SMI_INCLUDE_DIR}" +diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt +index baf0d851c86..0a6e71dda1c 100755 +--- a/server/CMakeLists.txt ++++ b/server/CMakeLists.txt +@@ -97,7 +97,7 @@ target_link_libraries( + pthread + rt + gRPC::grpc++ +- cap ++ PkgConfig::LIBCAP + dl + amd_smi + rdc_bootstrap diff --git a/pkgs/development/rocm-modules/10/rocdbgapi/default.nix b/pkgs/development/rocm-modules/10/rocdbgapi/default.nix new file mode 100644 index 0000000000000..c0014ac9a3640 --- /dev/null +++ b/pkgs/development/rocm-modules/10/rocdbgapi/default.nix @@ -0,0 +1,115 @@ +{ + lib, + stdenv, + fetchFromGitHub, + rocmUpdateScript, + cmake, + rocm-cmake, + rocm-comgr, + rocm-runtime, + hwdata, + texliveSmall, + doxygen, + graphviz, + writableTmpDirAsHomeHook, + buildDocs ? true, +}: + +let + latex = lib.optionalAttrs buildDocs ( + texliveSmall.withPackages ( + ps: with ps; [ + changepage + latexmk + varwidth + multirow + hanging + adjustbox + collectbox + stackengine + enumitem + alphalph + wasysym + sectsty + tocloft + newunicodechar + etoc + helvetic + wasy + courier + tabularray + ltablex + ninecolors + xltabular + ] + ) + ); +in +stdenv.mkDerivation (finalAttrs: { + pname = "rocdbgapi"; + version = "10.1"; + + outputs = [ + "out" + ] + ++ lib.optionals buildDocs [ + "doc" + ]; + + buildFlags = lib.optionals buildDocs [ "doc" ]; + + src = fetchFromGitHub { + owner = "ROCm"; + # ROCdbgapi lives in the rocm-systems monorepo since ROCm 10 + repo = "rocm-systems"; + tag = "therock-${finalAttrs.version}"; + sparseCheckout = [ + "projects/rocdbgapi" + "shared" + ]; + hash = "sha256-vadmzQL3Sqtbtf7vMKCN6jabBuxnugPHDJd2yDhVDTM="; + }; + sourceRoot = "${finalAttrs.src.name}/projects/rocdbgapi"; + + nativeBuildInputs = [ + cmake + rocm-cmake + ] + ++ lib.optionals buildDocs [ + writableTmpDirAsHomeHook + latex + doxygen + graphviz + ]; + + buildInputs = [ + rocm-comgr + rocm-runtime + hwdata + ]; + + cmakeFlags = [ + "-DPCI_IDS_PATH=${hwdata}/share/hwdata" + # Manually define CMAKE_INSTALL_ + # See: https://github.com/NixOS/nixpkgs/pull/197838 + "-DCMAKE_INSTALL_BINDIR=bin" + "-DCMAKE_INSTALL_LIBDIR=lib" + "-DCMAKE_INSTALL_INCLUDEDIR=include" + ]; + + postInstall = lib.optionalString buildDocs '' + mkdir -p $doc/share/doc/amd-dbgapi/ + mv $out/share/html/amd-dbgapi $doc/share/doc/amd-dbgapi/html + rmdir $out/share/html + ''; + + passthru.updateScript = rocmUpdateScript { inherit finalAttrs; }; + + meta = { + description = "Debugger support for control of execution and inspection state of AMD's GPU architectures"; + homepage = "https://github.com/ROCm/ROCdbgapi"; + license = lib.licenses.mit; + teams = [ lib.teams.rocm ]; + platforms = lib.platforms.linux; + }; +}) diff --git a/pkgs/development/rocm-modules/10/rocgdb/default.nix b/pkgs/development/rocm-modules/10/rocgdb/default.nix new file mode 100644 index 0000000000000..4dbb291e5305f --- /dev/null +++ b/pkgs/development/rocm-modules/10/rocgdb/default.nix @@ -0,0 +1,114 @@ +{ + lib, + stdenv, + fetchFromGitHub, + rocmUpdateScript, + pkg-config, + texinfo, + bison, + flex, + glibc, + zlib, + zstd, + gmp, + mpfr, + ncurses, + expat, + rocdbgapi, + perl, + python3, + babeltrace, + sourceHighlight, +}: + +stdenv.mkDerivation (finalAttrs: { + pname = "rocgdb"; + version = "10.1"; + + src = fetchFromGitHub { + owner = "ROCm"; + repo = "ROCgdb"; + # ROCgdb switched to the two-component therock-X.Y tag scheme in 10.0. + tag = "therock-${finalAttrs.version}"; + hash = "sha256-BenTuntdO+NUgzty0en2b7uaIvugISNU4rqxRG/Ka1w="; + }; + + nativeBuildInputs = [ + pkg-config + texinfo # For makeinfo + bison + flex + perl # used in mkinstalldirs script during installPhase + python3 + ]; + + buildInputs = [ + zlib + zstd + gmp + mpfr + ncurses + expat + rocdbgapi + python3 + babeltrace + sourceHighlight + ]; + + configureFlags = [ + # Ensure we build the amdgpu target + "--enable-targets=${stdenv.targetPlatform.config},amdgcn-amd-amdhsa" + "--with-amd-dbgapi=yes" + + "--with-iconv-path=${glibc.bin}" + "--enable-tui" + "--with-babeltrace=${babeltrace}" + "--with-python=python3" + "--with-system-zlib" + "--with-system-zstd" + "--enable-64-bit-bfd" + "--with-gmp=${gmp.dev}" + "--with-mpfr=${mpfr.dev}" + "--with-expat=${expat}" + + # So the installed binary is called "rocgdb" instead on plain "gdb" + "--program-prefix=roc" + + # Disable building many components not used or incompatible with the amdgcn target + "--disable-sim" + "--disable-gdbserver" + "--disable-ld" + "--disable-gas" + "--disable-gdbserver" + "--disable-gdbtk" + "--disable-gprofng" + "--disable-shared" + ]; + + postPatch = '' + for file in *; do + if [ -f "$file" ]; then + patchShebangs "$file" + fi + done + ''; + + # The source directory for ROCgdb (based on upstream GDB) contains multiple project + # of GNU’s toolchain (binutils and onther), we only need to install the GDB part. + installPhase = '' + make install-gdb + ''; + + env.CFLAGS = "-Wno-switch -Wno-format-nonliteral -I${zstd.dev}/include -I${zlib.dev}/include -I${expat.dev}/include -I${ncurses.dev}/include"; + env.CXXFLAGS = finalAttrs.env.CFLAGS; + + passthru.updateScript = rocmUpdateScript { inherit finalAttrs; }; + + meta = { + description = "ROCm source-level debugger for Linux, based on GDB"; + homepage = "https://github.com/ROCm/ROCgdb"; + license = lib.licenses.gpl3Plus; + teams = [ lib.teams.rocm ]; + platforms = lib.platforms.linux; + }; +}) diff --git a/pkgs/development/rocm-modules/10/rocprof-trace-decoder/default.nix b/pkgs/development/rocm-modules/10/rocprof-trace-decoder/default.nix new file mode 100644 index 0000000000000..77eaa6dc1e31d --- /dev/null +++ b/pkgs/development/rocm-modules/10/rocprof-trace-decoder/default.nix @@ -0,0 +1,84 @@ +{ + lib, + stdenv, + fetchFromGitHub, + rocm-comgr, + rocm-runtime, + cmake, + python3, + nlohmann_json, + gtest, +}: + +stdenv.mkDerivation (finalAttrs: { + pname = "rocprof-trace-decoder"; + version = "0.2.2"; + + src = fetchFromGitHub { + owner = "ROCm"; + repo = "rocm-systems"; + # rocprof-trace-decoder keeps its own 0.x version, so the rocm-systems + # monorepo tag cannot be derived from finalAttrs.version like sibling packages. + rev = "therock-10.1"; + sparseCheckout = [ + "projects/rocprof-trace-decoder" + "shared" + ]; + hash = "sha256-0cbkljtw/gZJwXPUpJuLXpQlXfR/MQgEGbLDsRdzpcw="; + }; + + sourceRoot = "${finalAttrs.src.name}/projects/rocprof-trace-decoder"; + + strictDeps = true; + + nativeBuildInputs = [ cmake ]; + + buildInputs = [ + rocm-comgr + rocm-runtime + ]; + + cmakeFlags = [ + (lib.cmakeBool "BUILD_TESTS" finalAttrs.finalPackage.doCheck) + # The ATT integration tests need the pyelftools python module + # and bundled trace-data fixtures; keep the unit tests, skip integration. + (lib.cmakeBool "BUILD_INTEGRATION_TESTS" false) + ]; + + nativeCheckInputs = [ + python3 + ]; + + checkInputs = [ + nlohmann_json + gtest + ]; + + preCheck = '' + patchShebangs test + ''; + + checkPhase = + let + # Sanitize tests fail because the UBSan runtime (__ubsan_vptr_type_cache) is not available for + # LD_PRELOAD in the sandbox. + skipPattern = "_sanitize$"; + in + '' + runHook preCheck + + ctest --test-dir . --output-on-failure -E '${skipPattern}' + + runHook postCheck + ''; + + doCheck = true; + + meta = { + description = "Library for decoding ROCm thread trace data"; + homepage = "https://github.com/ROCm/rocm-systems/tree/develop/projects/rocprof-trace-decoder"; + license = lib.licenses.mit; + teams = [ lib.teams.rocm ]; + platforms = lib.platforms.linux; + }; +}) diff --git a/pkgs/development/rocm-modules/10/rocprofiler-sdk/default.nix b/pkgs/development/rocm-modules/10/rocprofiler-sdk/default.nix new file mode 100644 index 0000000000000..6da6fe8b92773 --- /dev/null +++ b/pkgs/development/rocm-modules/10/rocprofiler-sdk/default.nix @@ -0,0 +1,193 @@ +{ + lib, + stdenv, + fetchFromGitHub, + cmake, + ninja, + pkg-config, + clr, + rocm-cmake, + rocm-runtime, + rocprofiler-register, + rocprof-trace-decoder, + aqlprofile, + rocm-comgr, + rocmUpdateScript, + rccl, + python3, + python3Packages, + libdrm, + elfutils, + sqlite, + otf2, + zlib, + zstd, + xz, + numactl, + fmt, + gtest, + yaml-cpp, + elfio, + nlohmann_json, + makeWrapper, + gpuTargets ? (clr.localGpuTargets or clr.gpuTargets), + buildTests ? false, + buildSamples ? false, +}: + +# FIXME: devendor remaining git submodules: +# external/cereal -> https://github.com/jrmadsen/cereal +# external/gotcha -> https://github.com/jrmadsen/GOTCHA +# external/perfetto -> https://github.com/google/perfetto +# external/ptl -> https://github.com/jrmadsen/PTL + +# rocprofiler-sdk is the home of rocprofv3 +stdenv.mkDerivation (finalAttrs: { + pname = "rocprofiler-sdk"; + version = "10.1"; + + outputs = [ + "out" + "dev" + ]; + + src = fetchFromGitHub { + owner = "ROCm"; + repo = "rocm-systems"; + tag = "therock-${finalAttrs.version}"; + hash = "sha256-UmYHC+87XIW+Z4WVd7k0MQ9baTvFlq/SgCt9Foy1mPI="; + fetchSubmodules = true; + sparseCheckout = [ + "projects/rocprofiler-sdk" + ]; + }; + sourceRoot = "${finalAttrs.src.name}/projects/rocprofiler-sdk"; + + strictDeps = true; + + nativeBuildInputs = [ + cmake + ninja + pkg-config + clr + python3 + makeWrapper + ]; + + buildInputs = [ + clr + rocm-cmake + rocm-runtime + rocprofiler-register + aqlprofile + # rocprofiler-sdk-tool does find_package(rocprof-trace-decoder REQUIRED) at + # configure time for the ATT quick-scan feature. + rocprof-trace-decoder + rocm-comgr + libdrm + elfutils + sqlite + otf2 + zlib + zstd + xz + numactl + fmt + gtest + yaml-cpp + elfio + nlohmann_json + rccl + python3Packages.pybind11 + ]; + + patches = [ + ./ompt-explicit-tool.patch + ]; + + postPatch = '' + # NixOS' ROCm distribution does not support libomptarget yet + substituteInPlace samples/CMakeLists.txt \ + --replace-fail 'add_subdirectory(openmp_target)' '# add_subdirectory(openmp_target)' + substituteInPlace tests/CMakeLists.txt \ + --replace-fail 'add_subdirectory(openmp-tools)' '# add_subdirectory(openmp-tools)' + substituteInPlace tests/bin/CMakeLists.txt \ + --replace-fail 'add_subdirectory(openmp)' '# add_subdirectory(openmp)' + + # This requires building perfetto's trace-processor-shell via a weird external script + substituteInPlace tests/CMakeLists.txt \ + --replace-fail 'add_subdirectory(pytest-packages)' "" + + # The sources have bare system includes (`` in the output/sql + + # python-binding sources, `` in the vendored aqlprofile) + # that assume a distro-style /usr/include. nixpkgs keeps those headers in the + # packages' dev outputs, and the rocm-toolchain clang used here + # (-DCMAKE_CXX_COMPILER=clang++, unwrapped) ignores NIX_CFLAGS_COMPILE, so add + # them as a real CMake -isystem at the top of the tree. (A global + # -DCMAKE_CXX_FLAGS instead would clobber abseil's C++-standard detection.) + substituteInPlace CMakeLists.txt \ + --replace-fail 'add_subdirectory(external)' \ + 'include_directories(SYSTEM ${lib.getDev sqlite}/include ${lib.getDev libdrm}/include) + add_subdirectory(external)' + + patchShebangs source/libexec/rocprofiler-sdk/rocprofiler-sdk-launch-compiler/rocprofiler-sdk-launch-compiler.sh + ''; + + cmakeFlags = [ + (lib.cmakeBool "ROCPROFILER_BUILD_GHC_FS" false) + (lib.cmakeBool "ROCPROFILER_BUILD_FMT" false) + (lib.cmakeBool "ROCPROFILER_BUILD_GTEST" false) + (lib.cmakeBool "ROCPROFILER_BUILD_PYBIND11" false) + (lib.cmakeBool "ROCPROFILER_BUILD_YAML_CPP" false) + (lib.cmakeBool "ROCPROFILER_BUILD_ELFIO" false) + (lib.cmakeBool "ROCPROFILER_BUILD_OTF2" false) + (lib.cmakeBool "ROCPROFILER_BUILD_JSON" false) + (lib.cmakeBool "ROCPROFILER_BUILD_TESTS" buildTests) + (lib.cmakeBool "ROCPROFILER_BUILD_SAMPLES" buildSamples) + (lib.cmakeBool "ROCPROFILER_BUILD_BENCHMARK" false) + (lib.cmakeBool "ROCPROFILER_BUILD_WERROR" false) + # rocprofiler-sdk's CMake file doesn't add this dependency properly. + "-DCMAKE_HIP_FLAGS=-I${rocm-runtime}/include" + "-DCMAKE_INSTALL_BINDIR=bin" + "-DCMAKE_INSTALL_LIBDIR=lib" + "-DCMAKE_INSTALL_INCLUDEDIR=include" + # Required for rocprofiler-sdk-launch-compiler.sh + "-DCMAKE_INSTALL_LIBEXECDIR=libexec" + "--debug-find-pkg=amd_comgr" + "--trace-source=tests/pc_sampling/CMakeLists.txt" + "--trace-expand" + ] + ++ lib.optionals (buildTests || buildSamples) [ + # rocprofiler-sdk normally doesn't depend on which GPU is in the system, only when + # building tests or samples. + (lib.cmakeFeature "GPU_TARGETS" (lib.concatStringsSep ";" gpuTargets)) + ]; + + doCheck = false; # Requires GPU + + postFixup = '' + patchelf $out/lib/*.so \ + --add-rpath ${aqlprofile}/lib \ + --add-needed libhsa-amd-aqlprofile64.so + + wrapProgram $out/bin/rocprofv3 --add-flags "--att-library-path ${ + lib.makeLibraryPath [ rocprof-trace-decoder ] + }" + ''; + + postInstall = '' + mkdir -p $dev/lib $dev/share/rocprofiler-sdk + mv $out/lib/cmake $dev/lib/ + mv $out/share/rocprofiler-sdk/{samples,tests} $dev/share/rocprofiler-sdk/ + ''; + + passthru.updateScript = rocmUpdateScript { inherit finalAttrs; }; + + meta = { + description = "ROCm GPU performance analysis SDK"; + homepage = "https://github.com/ROCm/rocprofiler-sdk"; + license = lib.licenses.mit; + teams = [ lib.teams.rocm ]; + platforms = lib.platforms.linux; + }; +}) diff --git a/pkgs/development/rocm-modules/10/rocprofiler-sdk/ompt-explicit-tool.patch b/pkgs/development/rocm-modules/10/rocprofiler-sdk/ompt-explicit-tool.patch new file mode 100644 index 0000000000000..75f57e30e6e47 --- /dev/null +++ b/pkgs/development/rocm-modules/10/rocprofiler-sdk/ompt-explicit-tool.patch @@ -0,0 +1,76 @@ +Subject: [PATCH] Avoid unsolicited OMPT initialization + +OpenMP calls ompt_start_tool from its own initialization, which can run +inside a library constructor. Registering clients from there dlopens every +library that defines rocprofiler_configure; if one of them is still being +loaded (PyTorch's libtorch_cpu, for Kineto), glibc reruns pending +constructors and OpenBLAS reenters OpenMP initialization, which livelocks +on its init lock. Decline the tool unless one was requested through +ROCP_TOOL_LIBRARIES (as rocprofv3 does) or rocprofiler_force_configure. +Clients found by symbol still register when the HIP runtime starts; they +only miss OMPT callbacks. + +OpenMP queries ompt_start_tool only once, at startup, so attaching a +profiler to an already running process cannot enable OMPT tracing; the +other services are unaffected. + +--- a/source/lib/rocprofiler-sdk/ompt.cpp ++++ b/source/lib/rocprofiler-sdk/ompt.cpp +@@ -265,6 +265,9 @@ ompt_start_tool(unsigned int omp_version, const char* runtime_version) ROCPROFIL + ompt_start_tool_result_t* + ompt_start_tool(unsigned int omp_version, const char* runtime_version) + { ++ // Registering clients during OpenMP startup can reenter it (see above). ++ if(!::rocprofiler::registration::is_tool_requested()) return nullptr; ++ + ::rocprofiler::registration::init_logging(); + ::rocprofiler::registration::initialize(); + return rocprofiler_ompt_start_tool(omp_version, runtime_version); +--- a/source/lib/rocprofiler-sdk/registration.cpp ++++ b/source/lib/rocprofiler-sdk/registration.cpp +@@ -86,6 +86,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -1039,6 +1040,16 @@ get_client_offset() + return _v; + } + ++static std::atomic forced_configure_requested{false}; ++ ++bool ++is_tool_requested() ++{ ++ if(forced_configure_requested.load(std::memory_order_acquire)) return true; ++ const auto* tools = secure_getenv("ROCP_TOOL_LIBRARIES"); ++ return tools && *tools; ++} ++ + int + get_init_status() + { +@@ -1235,6 +1246,8 @@ rocprofiler_force_configure(rocprofiler_configure_func_t configure_func) + + if(rocprofiler::registration::get_fini_status() != 0) return ROCPROFILER_STATUS_ERROR_FINALIZED; + ++ rocprofiler::registration::forced_configure_requested.store(true, std::memory_order_release); ++ + ROCP_INFO << "forcing rocprofiler configuration"; + + auto* forced_configs = rocprofiler::registration::get_forced_configures(); +--- a/source/lib/rocprofiler-sdk/registration.hpp ++++ b/source/lib/rocprofiler-sdk/registration.hpp +@@ -49,6 +49,9 @@ namespace registration + void + init_logging(); + ++bool ++is_tool_requested(); ++ + // initialize the clients + void + initialize(); diff --git a/pkgs/development/rocm-modules/10/rocprofiler/default.nix b/pkgs/development/rocm-modules/10/rocprofiler/default.nix new file mode 100644 index 0000000000000..358a2467e7975 --- /dev/null +++ b/pkgs/development/rocm-modules/10/rocprofiler/default.nix @@ -0,0 +1,141 @@ +{ + lib, + stdenv, + fetchFromGitHub, + rocmUpdateScript, + symlinkJoin, + cmake, + clang, + clr, + aqlprofile, + rocm-core, + rocm-runtime, + rocm-device-libs, + roctracer, + rocdbgapi, + numactl, + libpciaccess, + libxml2, + llvm, + elfutils, + mpi, + gtest, + python3Packages, + gpuTargets ? clr.localGpuTargets or clr.gpuTargets, +}: + +let + rocmtoolkit-merged = symlinkJoin { + name = "rocmtoolkit-merged"; + + paths = [ + rocm-core + rocm-runtime + rocm-device-libs + roctracer + rocdbgapi + clr + ]; + + postBuild = '' + rm -rf $out/nix-support + ''; + }; +in +stdenv.mkDerivation (finalAttrs: { + pname = "rocprofiler"; + version = "10.1"; + + src = fetchFromGitHub { + owner = "ROCm"; + repo = "rocm-systems"; + tag = "therock-${finalAttrs.version}"; + sparseCheckout = [ + "projects/rocprofiler" + "shared" + ]; + fetchSubmodules = true; + hash = "sha256-vKs+2TpbxmHZNmQK0WDRRKohUTpu/Zkc1V1b8V+lExk="; + }; + sourceRoot = "${finalAttrs.src.name}/projects/rocprofiler"; + + nativeBuildInputs = [ + cmake + clang + clr + python3Packages.lxml + python3Packages.cppheaderparser + python3Packages.pyyaml + python3Packages.barectf + python3Packages.pandas + ]; + + buildInputs = [ + llvm.clang-unwrapped + llvm.llvm + numactl + libpciaccess + libxml2 + elfutils + mpi + gtest + aqlprofile + ]; + + propagatedBuildInputs = [ rocmtoolkit-merged ]; + + #HACK: rocprofiler's cmake doesn't add these deps properly + env.CXXFLAGS = "-I${libpciaccess}/include -I${numactl.dev}/include -I${rocmtoolkit-merged}/include -I${elfutils.dev}/include -w"; + + cmakeFlags = [ + "-DCMAKE_MODULE_PATH=${clr}/lib/cmake/hip" + "-DHIP_ROOT_DIR=${clr}" + "-DGPU_TARGETS=${lib.concatStringsSep ";" gpuTargets}" + # Manually define CMAKE_INSTALL_ + # See: https://github.com/NixOS/nixpkgs/pull/197838 + "-DCMAKE_INSTALL_BINDIR=bin" + "-DCMAKE_INSTALL_LIBDIR=lib" + "-DCMAKE_INSTALL_INCLUDEDIR=include" + ]; + + postPatch = '' + patchShebangs . + + substituteInPlace cmake_modules/rocprofiler_utils.cmake \ + --replace-fail 'function(ROCPROFILER_CHECKOUT_GIT_SUBMODULE)' 'function(ROCPROFILER_CHECKOUT_GIT_SUBMODULE) + return()' + + substituteInPlace CMakeLists.txt \ + --replace-fail 'set(ROCPROFILER_BUILD_TESTS ON)' "" + + substituteInPlace tests-v2/featuretests/profiler/CMakeLists.txt \ + --replace "--build-id=sha1" "--build-id=sha1 --rocm-path=${clr} --rocm-device-lib-path=${rocm-device-libs}/amdgcn/bitcode" + + substituteInPlace test/CMakeLists.txt \ + --replace "\''${ROCM_ROOT_DIR}/amdgcn/bitcode" "${rocm-device-libs}/amdgcn/bitcode" + ''; + + postInstall = '' + # Why do these have the executable bit set? + chmod -x $out/libexec/rocprofiler/counters/*.xml + # rocprof shell script wants to find it in the same bin dir, easiest to symlink in + ln -s ${clr}/bin/rocm_agent_enumerator $out/bin/rocm_agent_enumerator + ''; + + postFixup = '' + patchelf $out/lib/*.so \ + --add-rpath ${aqlprofile}/lib \ + --add-needed libhsa-amd-aqlprofile64.so + ''; + + passthru.updateScript = rocmUpdateScript { inherit finalAttrs; }; + passthru.rocmtoolkit-merged = rocmtoolkit-merged; + + meta = { + description = "Profiling with perf-counters and derived metrics"; + homepage = "https://github.com/ROCm/rocm-systems/tree/develop/projects/rocprofiler"; + license = lib.licenses.mit; # mitx11 + teams = [ lib.teams.rocm ]; + platforms = lib.platforms.linux; + }; +}) diff --git a/pkgs/development/rocm-modules/10/rocr-debug-agent/default.nix b/pkgs/development/rocm-modules/10/rocr-debug-agent/default.nix new file mode 100644 index 0000000000000..b134c42718a49 --- /dev/null +++ b/pkgs/development/rocm-modules/10/rocr-debug-agent/default.nix @@ -0,0 +1,61 @@ +{ + lib, + stdenv, + fetchFromGitHub, + rocmUpdateScript, + cmake, + python3, + clr, + rocdbgapi, + elfutils, +}: + +stdenv.mkDerivation (finalAttrs: { + pname = "rocr-debug-agent"; + version = "10.1"; + + src = fetchFromGitHub { + owner = "ROCm"; + # rocr_debug_agent lives in the rocm-systems monorepo since ROCm 10 + repo = "rocm-systems"; + tag = "therock-${finalAttrs.version}"; + sparseCheckout = [ + "projects/rocr-debug-agent" + "shared" + ]; + hash = "sha256-WpBG1ogF161EM3tgXnVGstyDBdCmTXQPUPnkZF1eF3A="; + }; + sourceRoot = "${finalAttrs.src.name}/projects/rocr-debug-agent"; + + nativeBuildInputs = [ + cmake + clr + python3 # TODO: check for scripts that need patchShebangs in output + ]; + + buildInputs = [ + rocdbgapi + elfutils + ]; + + cmakeFlags = [ + "-DCMAKE_MODULE_PATH=${clr}/lib/cmake/hip" + "-DHIP_ROOT_DIR=${clr}" + "-DHIP_PATH=${clr}" + ]; + + # Weird install target + postInstall = '' + rm -rf $out/src + ''; + + passthru.updateScript = rocmUpdateScript { inherit finalAttrs; }; + + meta = { + description = "Library that provides some debugging functionality for ROCr"; + homepage = "https://github.com/ROCm/rocr_debug_agent"; + license = lib.licenses.ncsa; + teams = [ lib.teams.rocm ]; + platforms = lib.platforms.linux; + }; +}) diff --git a/pkgs/development/rocm-modules/10/rocshmem/default.nix b/pkgs/development/rocm-modules/10/rocshmem/default.nix new file mode 100644 index 0000000000000..c18d6c77da492 --- /dev/null +++ b/pkgs/development/rocm-modules/10/rocshmem/default.nix @@ -0,0 +1,84 @@ +{ + lib, + stdenv, + fetchFromGitHub, + rocmUpdateScript, + cmake, + rocm-cmake, + rocm-core, + rocm-runtime, + clr, + mpi, + useMpi ? true, + useReverseOffload ? useMpi, + buildExamples ? false, + gpuTargets ? (clr.localGpuTargets or [ ]), +}: + +stdenv.mkDerivation (finalAttrs: { + pname = "rocshmem${clr.gpuArchSuffix}"; + version = "10.1"; + + src = fetchFromGitHub { + owner = "ROCm"; + # rocSHMEM lives in the rocm-systems monorepo since ROCm 10 + repo = "rocm-systems"; + tag = "therock-${finalAttrs.version}"; + sparseCheckout = [ + "projects/rocshmem" + "shared" + ]; + hash = "sha256-uVUHYyJbaE5vauN1mczLf7rYNxRSD93eJDIDbzbYzHs="; + }; + sourceRoot = "${finalAttrs.src.name}/projects/rocshmem"; + + nativeBuildInputs = [ + cmake + rocm-cmake + clr + ]; + + buildInputs = [ + rocm-runtime + ] + ++ lib.optionals useMpi [ + mpi + ]; + + # The device bitcode command does not see the HSA headers the host target gets. + postPatch = '' + substituteInPlace cmake/DeviceBitcode.cmake \ + --replace-fail \ + '-Xclang -mcode-object-version=none' \ + '-Xclang -mcode-object-version=none + -I${rocm-runtime}/include' + ''; + + cmakeFlags = [ + (lib.cmakeBool "CMAKE_POSITION_INDEPENDENT_CODE" true) + (lib.cmakeFeature "CMAKE_CXX_COMPILER" "hipcc") + (lib.cmakeFeature "ROCM_PATH" "${clr}") + (lib.cmakeFeature "ROCM_VERSION" rocm-core.ROCM_LIBPATCH_VERSION) + (lib.cmakeBool "USE_IPC" true) + (lib.cmakeBool "USE_RO" useReverseOffload) + (lib.cmakeBool "USE_SINGLE_NODE" (!useMpi)) + (lib.cmakeBool "USE_EXTERNAL_MPI" useMpi) + (lib.cmakeBool "BUILD_EXAMPLES" buildExamples) + (lib.cmakeFeature "CMAKE_INSTALL_BINDIR" "bin") + (lib.cmakeFeature "CMAKE_INSTALL_LIBDIR" "lib") + (lib.cmakeFeature "CMAKE_INSTALL_INCLUDEDIR" "include") + ] + ++ lib.optionals (gpuTargets != [ ]) [ + (lib.cmakeFeature "GPU_TARGETS" (lib.concatStringsSep ";" gpuTargets)) + ]; + + passthru.updateScript = rocmUpdateScript { inherit finalAttrs; }; + + meta = { + description = "The ROCm OpenSHMEM (rocSHMEM) runtime"; + homepage = "https://github.com/ROCm/rocSHMEM"; + license = lib.licenses.mit; + teams = [ lib.teams.rocm ]; + platforms = lib.platforms.linux; + }; +})