rocmPackages_10: add profiling and debugging tools, RCCL

pkgs/development/rocm-modules/10/aqlprofile/default.nix

@@ -0,0 +1,42 @@
+{
+  lib,
+  stdenv,
+  clr,
+  cmake,
+  fetchFromGitHub,
+  rocmUpdateScript,
+}:
+
+stdenv.mkDerivation (finalAttrs: {
+  pname = "aqlprofile";
+  version = "10.1";
+
+  src = fetchFromGitHub {
+    owner = "ROCm";
+    repo = "rocm-systems";
+    tag = "therock-${finalAttrs.version}";
+    sparseCheckout = [
+      "projects/aqlprofile"
+      "shared"
+    ];
+    hash = "sha256-7yeEJoyShSCwy1MTCeArYBabh5qTVm5uZL8N74rvxPA=";
+  };
+  sourceRoot = "${finalAttrs.src.name}/projects/aqlprofile";
+
+  env.CXXFLAGS = "-DROCP_LD_AQLPROFILE=1";
+
+  nativeBuildInputs = [
+    cmake
+    clr
+  ];
+
+  passthru.updateScript = rocmUpdateScript { inherit finalAttrs; };
+
+  meta = {
+    description = "AQLPROFILE library for AMD HSA runtime API extension support";
+    homepage = "https://github.com/ROCm/rocm-systems/tree/develop/projects/aqlprofile";
+    license = lib.licenses.mit;
+    teams = [ lib.teams.rocm ];
+    platforms = lib.platforms.linux;
+  };
+})

pkgs/development/rocm-modules/10/default.nix

@@ -63,6 +63,10 @@ let
         inherit (self) rocmUpdateScript;
       };
 
+      aqlprofile = self.callPackage ./aqlprofile { };
+
+      rdc = self.callPackage ./rdc { };
+
       rocm-docs-core = python3Packages.callPackage ../rocm-docs-core { };
 
       hip-common = self.callPackage ./hip-common { };
@@ -78,12 +82,33 @@ let
 
       # hsakmt was merged into rocm-runtime
       hsakmt = self.rocm-runtime;
+
+      rocprofiler = self.callPackage ./rocprofiler {
+        inherit (llvm) clang;
+      };
       rocprofiler-register = self.callPackage ./rocprofiler-register {
         inherit (llvm) clang;
       };
+      rocprofiler-sdk = self.callPackage ./rocprofiler-sdk { };
+
+      rocprof-compute-viewer = self.callPackage ../rocprof-compute-viewer { };
+
+      rocprof-trace-decoder = self.callPackage ./rocprof-trace-decoder { };
 
       roctracer = self.callPackage ./roctracer { };
 
+      rocgdb = self.callPackage ./rocgdb { };
+
+      rocdbgapi = self.callPackage ./rocdbgapi { };
+
+      rocr-debug-agent = self.callPackage ./rocr-debug-agent { };
+
+      mscclpp = self.callPackage ../mscclpp { };
+
+      rccl = self.callPackage ./rccl { };
+
+      rocshmem = self.callPackage ./rocshmem { };
+
       half = self.callPackage ./half { };
 
       # Even if config.rocmSupport is false we need rocmSupport true

pkgs/development/rocm-modules/10/rccl/default.nix

@@ -0,0 +1,153 @@
+{
+  lib,
+  stdenv,
+  fetchFromGitHub,
+  rocmUpdateScript,
+  cmake,
+  rocm-cmake,
+  amdsmi,
+  rocm-core,
+  pkg-config,
+  clr,
+  perl,
+  hipify,
+  python3,
+  fmt,
+  gtest,
+  chrpath,
+  roctracer,
+  rocprofiler,
+  rocprofiler-register,
+  autoPatchelfHook,
+  buildTests ? false,
+  gpuTargets ? (clr.localGpuTargets or [ ]),
+  # for passthru.tests
+  rccl,
+}:
+
+let
+  useAsan = buildTests;
+  useUbsan = buildTests;
+  san = lib.optionalString (useAsan || useUbsan) (
+    "-fno-gpu-sanitize -fsanitize=undefined "
+    + (lib.optionalString useAsan "-fsanitize=address -shared-libsan ")
+  );
+in
+# Note: we can't properly test or make use of multi-node collective ops
+# https://github.com/NixOS/nixpkgs/issues/366242 tracks kernel support
+# kfd_peerdirect support which is on out-of-tree amdkfd in ROCm/ROCK-Kernel-Driver
+# infiniband ib_peer_mem support isn't in the mainline kernel but is carried by some distros
+stdenv.mkDerivation (finalAttrs: {
+  pname = "rccl${clr.gpuArchSuffix}";
+  version = "10.1";
+
+  outputs = [
+    "out"
+  ]
+  ++ lib.optionals buildTests [
+    "test"
+  ];
+
+  patches = [
+    ./rccl-test-missing-iomanip.diff
+  ];
+
+  src = fetchFromGitHub {
+    owner = "ROCm";
+    repo = "rocm-systems";
+    tag = "therock-${finalAttrs.version}";
+    sparseCheckout = [
+      "projects/rccl"
+      "shared"
+    ];
+    hash = "sha256-OkX15/P1wQ9E8QfAOklXvxmVw700kvIyi6CwdorqNhA=";
+  };
+  sourceRoot = "${finalAttrs.src.name}/projects/rccl";
+
+  requiredSystemFeatures = [ "big-parallel" ]; # Very resource intensive LTO
+
+  nativeBuildInputs = [
+    cmake
+    rocm-cmake
+    clr
+    perl
+    hipify
+    python3
+    pkg-config
+    autoPatchelfHook # ASAN doesn't add rpath without this
+  ];
+
+  buildInputs = [
+    # Headers only; RCCL dlopens amd_smi at runtime.
+    amdsmi
+    fmt
+    gtest
+    roctracer
+    rocprofiler
+    rocprofiler-register
+  ]
+  ++ lib.optionals buildTests [
+    chrpath
+  ];
+
+  cmakeFlags = [
+    "-DHIP_CLANG_NUM_PARALLEL_JOBS=4"
+    "-DROCM_PATH=${clr}"
+    "-DHIP_COMPILER=${clr}/bin/amdclang++"
+    "-DCMAKE_CXX_COMPILER=${clr}/bin/amdclang++"
+    "-DROCM_PATCH_VERSION=${rocm-core.ROCM_LIBPATCH_VERSION}"
+    "-DROCM_VERSION=${rocm-core.ROCM_LIBPATCH_VERSION}"
+    "-DBUILD_BFD=OFF" # Can't get it to detect bfd.h
+    # Manually define CMAKE_INSTALL_<DIR>
+    # See: https://github.com/NixOS/nixpkgs/pull/197838
+    "-DCMAKE_INSTALL_BINDIR=bin"
+    "-DCMAKE_INSTALL_LIBDIR=lib"
+    "-DCMAKE_INSTALL_INCLUDEDIR=include"
+  ]
+  ++ lib.optionals (gpuTargets != [ ]) [
+    # AMD can't make up their minds and keep changing which one is used in different projects.
+    "-DAMDGPU_TARGETS=${lib.concatStringsSep ";" gpuTargets}"
+    "-DGPU_TARGETS=${lib.concatStringsSep ";" gpuTargets}"
+  ]
+  ++ lib.optionals buildTests [
+    "-DBUILD_TESTS=ON"
+  ];
+
+  # -O2 and -fno-strict-aliasing due to UB issues in RCCL :c
+  # Reported upstream
+  env.CFLAGS = "-I${clr}/include -I${roctracer}/include -O2 -fno-strict-aliasing ${san}-fno-omit-frame-pointer -momit-leaf-frame-pointer";
+  env.CXXFLAGS = "-I${clr}/include -I${roctracer}/include -O2 -fno-strict-aliasing ${san}-fno-omit-frame-pointer -momit-leaf-frame-pointer";
+  env.LDFLAGS = "${san}";
+  postPatch = ''
+    patchShebangs src tools
+  '';
+
+  postInstall =
+    lib.optionalString useAsan ''
+      patchelf --add-needed ${clr}/llvm/lib/linux/libclang_rt.asan-${stdenv.hostPlatform.parsed.cpu.name}.so $out/lib/librccl.so
+    ''
+    + lib.optionalString buildTests ''
+      mkdir -p $test/bin
+      mv $out/bin/* $test/bin
+      rmdir $out/bin
+    '';
+
+  passthru.updateScript = rocmUpdateScript { inherit finalAttrs; };
+
+  # This package with sanitizers + manual integration test binaries built
+  # must be ran manually
+  passthru.tests.rccl = rccl.override {
+    buildTests = true;
+  };
+
+  meta = {
+    description = "ROCm communication collectives library";
+    homepage = "https://github.com/ROCm/rccl";
+    license = with lib.licenses; [
+      bsd2
+      bsd3
+    ];
+    teams = [ lib.teams.rocm ];
+    platforms = lib.platforms.linux;
+  };
+})

pkgs/development/rocm-modules/10/rccl/rccl-test-missing-iomanip.diff

@@ -0,0 +1,10 @@
+--- a/test/common/TestBed.cpp
++++ b/test/common/TestBed.cpp
+@@ -4,6 +4,7 @@
+  * See LICENSE.txt for license information
+  ************************************************************************/
+ #include <unistd.h>
++#include <iomanip>
+ #include "TestBed.hpp"
+ #include <rccl/rccl.h>
+