From 458c8245d48bdb5beafaa0f75ab6e299594dffff Mon Sep 17 00:00:00 2001 From: Connor Baker Date: Mon, 18 Dec 2023 06:26:12 +0000 Subject: [PATCH 1/5] cuda-modules: wip refactor to allow user overrides; better separation of data and construction with modules --- .../cuda-modules/cuda/extension.nix | 102 ------- pkgs/development/cuda-modules/cuda/fixups.nix | 174 ++++++++++++ .../cuda-modules/cuda/meta-fixup.nix | 19 ++ .../cuda-modules/cuda/overrides.nix | 129 --------- pkgs/development/cuda-modules/cudnn/fixup.nix | 6 +- .../cuda-modules/cudnn/releases.nix | 262 ------------------ pkgs/development/cuda-modules/cudnn/shims.nix | 22 +- .../generic-builders/manifest.nix | 11 +- .../generic-builders/multiplex.nix | 71 ++--- .../cuda-modules/modules/cuda/default.nix | 49 +++- .../cuda/manifests/feature_11.4.4.json | 0 .../cuda/manifests/feature_11.5.2.json | 0 .../cuda/manifests/feature_11.6.2.json | 0 .../cuda/manifests/feature_11.7.1.json | 0 .../cuda/manifests/feature_11.8.0.json | 0 .../cuda/manifests/feature_12.0.1.json | 0 .../cuda/manifests/feature_12.1.1.json | 0 .../cuda/manifests/feature_12.2.2.json | 0 .../cuda/manifests/feature_12.3.0.json | 0 .../cuda/manifests/redistrib_11.4.4.json | 0 .../cuda/manifests/redistrib_11.5.2.json | 0 .../cuda/manifests/redistrib_11.6.2.json | 0 .../cuda/manifests/redistrib_11.7.1.json | 0 .../cuda/manifests/redistrib_11.8.0.json | 0 .../cuda/manifests/redistrib_12.0.1.json | 0 .../cuda/manifests/redistrib_12.1.1.json | 0 .../cuda/manifests/redistrib_12.2.2.json | 0 .../cuda/manifests/redistrib_12.3.0.json | 0 .../cuda-modules/modules/cudnn/default.nix | 1 + .../cuda-modules/modules/cudnn/releases.nix | 260 +++++++++++++++++ .../generic/manifests/feature/manifest.nix | 2 +- .../generic/manifests/redistrib/manifest.nix | 2 +- .../{extension.nix => default.nix} | 17 +- pkgs/top-level/cuda-packages.nix | 200 ++++++++++--- 34 files changed, 719 insertions(+), 608 deletions(-) delete mode 100644 pkgs/development/cuda-modules/cuda/extension.nix create mode 100644 pkgs/development/cuda-modules/cuda/fixups.nix create mode 100644 pkgs/development/cuda-modules/cuda/meta-fixup.nix delete mode 100644 pkgs/development/cuda-modules/cuda/overrides.nix delete mode 100644 pkgs/development/cuda-modules/cudnn/releases.nix rename pkgs/development/cuda-modules/{ => modules}/cuda/manifests/feature_11.4.4.json (100%) rename pkgs/development/cuda-modules/{ => modules}/cuda/manifests/feature_11.5.2.json (100%) rename pkgs/development/cuda-modules/{ => modules}/cuda/manifests/feature_11.6.2.json (100%) rename pkgs/development/cuda-modules/{ => modules}/cuda/manifests/feature_11.7.1.json (100%) rename pkgs/development/cuda-modules/{ => modules}/cuda/manifests/feature_11.8.0.json (100%) rename pkgs/development/cuda-modules/{ => modules}/cuda/manifests/feature_12.0.1.json (100%) rename pkgs/development/cuda-modules/{ => modules}/cuda/manifests/feature_12.1.1.json (100%) rename pkgs/development/cuda-modules/{ => modules}/cuda/manifests/feature_12.2.2.json (100%) rename pkgs/development/cuda-modules/{ => modules}/cuda/manifests/feature_12.3.0.json (100%) rename pkgs/development/cuda-modules/{ => modules}/cuda/manifests/redistrib_11.4.4.json (100%) rename pkgs/development/cuda-modules/{ => modules}/cuda/manifests/redistrib_11.5.2.json (100%) rename pkgs/development/cuda-modules/{ => modules}/cuda/manifests/redistrib_11.6.2.json (100%) rename pkgs/development/cuda-modules/{ => modules}/cuda/manifests/redistrib_11.7.1.json (100%) rename pkgs/development/cuda-modules/{ => modules}/cuda/manifests/redistrib_11.8.0.json (100%) rename pkgs/development/cuda-modules/{ => modules}/cuda/manifests/redistrib_12.0.1.json (100%) rename pkgs/development/cuda-modules/{ => modules}/cuda/manifests/redistrib_12.1.1.json (100%) rename pkgs/development/cuda-modules/{ => modules}/cuda/manifests/redistrib_12.2.2.json (100%) rename pkgs/development/cuda-modules/{ => modules}/cuda/manifests/redistrib_12.3.0.json (100%) create mode 100644 pkgs/development/cuda-modules/modules/cudnn/releases.nix rename pkgs/development/cuda-modules/setup-hooks/{extension.nix => default.nix} (86%) diff --git a/pkgs/development/cuda-modules/cuda/extension.nix b/pkgs/development/cuda-modules/cuda/extension.nix deleted file mode 100644 index dff79c1ee55f7..0000000000000 --- a/pkgs/development/cuda-modules/cuda/extension.nix +++ /dev/null @@ -1,102 +0,0 @@ -{cudaVersion, lib}: -let - inherit (lib) attrsets modules trivial; - redistName = "cuda"; - - # Manifest files for CUDA redistributables (aka redist). These can be found at - # https://developer.download.nvidia.com/compute/cuda/redist/ - # Maps a cuda version to the specific version of the manifest. - cudaVersionMap = { - "11.4" = "11.4.4"; - "11.5" = "11.5.2"; - "11.6" = "11.6.2"; - "11.7" = "11.7.1"; - "11.8" = "11.8.0"; - "12.0" = "12.0.1"; - "12.1" = "12.1.1"; - "12.2" = "12.2.2"; - "12.3" = "12.3.0"; - }; - - # Check if the current CUDA version is supported. - cudaVersionMappingExists = builtins.hasAttr cudaVersion cudaVersionMap; - - # fullCudaVersion : String - fullCudaVersion = cudaVersionMap.${cudaVersion}; - - evaluatedModules = modules.evalModules { - modules = [ - ../modules - # We need to nest the manifests in a config.cuda.manifests attribute so the - # module system can evaluate them. - { - cuda.manifests = { - redistrib = trivial.importJSON (./manifests + "/redistrib_${fullCudaVersion}.json"); - feature = trivial.importJSON (./manifests + "/feature_${fullCudaVersion}.json"); - }; - } - ]; - }; - - # Generally we prefer to do things involving getting attribute names with feature_manifest instead - # of redistrib_manifest because the feature manifest will have *only* the redist architecture - # names as the keys, whereas the redistrib manifest will also have things like version, name, license, - # and license_path. - featureManifest = evaluatedModules.config.cuda.manifests.feature; - redistribManifest = evaluatedModules.config.cuda.manifests.redistrib; - - # Builder function which builds a single redist package for a given platform. - # buildRedistPackage : callPackage -> PackageName -> Derivation - buildRedistPackage = - callPackage: pname: - let - redistribRelease = redistribManifest.${pname}; - featureRelease = featureManifest.${pname}; - drv = - (callPackage ../generic-builders/manifest.nix { - # We pass the whole release to the builder because it has logic to handle - # the case we're trying to build on an unsupported platform. - inherit - pname - redistName - redistribRelease - featureRelease - ; - }).overrideAttrs - ( - prevAttrs: { - # Add the package-specific license. - meta = prevAttrs.meta // { - license = - let - licensePath = - if redistribRelease.license_path != null then - redistribRelease.license_path - else - "${pname}/LICENSE.txt"; - url = "https://developer.download.nvidia.com/compute/cuda/redist/${licensePath}"; - in - lib.licenses.nvidiaCudaRedist // {inherit url;}; - }; - } - ); - in - drv; - - # Build all the redist packages given final and prev. - redistPackages = - final: _prev: - # Wrap the whole thing in an optionalAttrs so we can return an empty set if the CUDA version - # is not supported. - # NOTE: We cannot include the call to optionalAttrs *in* the pipe as we would strictly evaluate the - # attrNames before we check if the CUDA version is supported. - attrsets.optionalAttrs cudaVersionMappingExists ( - trivial.pipe featureManifest [ - # Get all the package names - builtins.attrNames - # Build the redist packages - (trivial.flip attrsets.genAttrs (buildRedistPackage final.callPackage)) - ] - ); -in -redistPackages diff --git a/pkgs/development/cuda-modules/cuda/fixups.nix b/pkgs/development/cuda-modules/cuda/fixups.nix new file mode 100644 index 0000000000000..0990be103aac3 --- /dev/null +++ b/pkgs/development/cuda-modules/cuda/fixups.nix @@ -0,0 +1,174 @@ +# Attribute set where each package name maps to a function which, when `callPackage` is called on it, +# returns a function to be provided to `overrideAttrs` to override the attributes of that package. +# NOTE: Unless a package is always available, do not take it by name in the callPackage arguments; +# instead, take cudaPackages and use the package you need within a guard (e.g., cudaVersionAtLeast). +{ + libcufile = + { + cudaVersionOlder, + lib, + libcublas, + numactl, + rdma-core, + }: + prevAttrs: + (prevAttrs: { + buildInputs = prevAttrs.buildInputs ++ [ + libcublas.lib + numactl + rdma-core + ]; + # Before 11.7 libcufile depends on itself for some reason. + env.autoPatchelfIgnoreMissingDeps = + prevAttrs.env.autoPatchelfIgnoreMissingDeps + + lib.optionalString (cudaVersionOlder "11.7") " libcufile.so.0"; + }); + + libcusolver = + { + cudaPackages, + cudaVersionAtLeast, + lib, + libcublas, + }: + prevAttrs: { + buildInputs = + prevAttrs.buildInputs + # Always depends on this + ++ [ libcublas.lib ] + # Dependency from 12.0 and on + ++ lib.optionals (cudaVersionAtLeast "12.0") [ cudaPackages.libnvjitlink.lib ] + # Dependency from 12.1 and on + ++ lib.optionals (cudaVersionAtLeast "12.1") [ cudaPackages.libcusparse.lib ]; + }; + + libcusparse = + { + cudaPackages, + cudaVersionAtLeast, + lib, + }: + prevAttrs: { + buildInputs = + prevAttrs.buildInputs + ++ lib.optionals (cudaVersionAtLeast "12.0") [ cudaPackages.libnvjitlink.lib ]; + }; + + cuda_compat = + { flags }: + prevAttrs: { + env.autoPatchelfIgnoreMissingDeps = + prevAttrs.env.autoPatchelfIgnoreMissingDeps + " libnvrm_gpu.so libnvrm_mem.so libnvdla_runtime.so"; + # `cuda_compat` only works on aarch64-linux, and only when building for Jetson devices. + brokenConditions = prevAttrs.brokenConditions // { + "Trying to use cuda_compat on aarch64-linux targeting non-Jetson devices" = !flags.isJetsonBuild; + }; + }; + + cuda_gdb = + { + cudaVersionAtLeast, + gmp, + lib, + }: + prevAttrs: { + buildInputs = + prevAttrs.buildInputs + # x86_64 only needs gmp from 12.0 and on + ++ lib.optionals (cudaVersionAtLeast "12.0") [ gmp ]; + }; + + cuda_nvcc = + { setupCudaHook }: + prevAttrs: { + propagatedBuildInputs = [ setupCudaHook ]; + + meta = (prevAttrs.meta or { }) // { + mainProgram = "nvcc"; + }; + }; + + cuda_nvprof = + { cuda_cupti }: prevAttrs: { buildInputs = prevAttrs.buildInputs ++ [ cuda_cupti.lib ]; }; + + cuda_demo_suite = + { + freeglut, + libcufft, + libcurand, + libGLU, + libglvnd, + mesa, + }: + prevAttrs: { + buildInputs = prevAttrs.buildInputs ++ [ + freeglut + libcufft.lib + libcurand.lib + libGLU + libglvnd + mesa + ]; + }; + + nsight_compute = + { + lib, + qt5, + qt6, + }: + prevAttrs: { + nativeBuildInputs = + prevAttrs.nativeBuildInputs + ++ lib.optionals (lib.versionOlder prevAttrs.version "2022.2.0") [ qt5.wrapQtAppsHook ] + ++ lib.optionals (lib.versionAtLeast prevAttrs.version "2022.2.0") [ qt6.wrapQtAppsHook ]; + buildInputs = + prevAttrs.buildInputs + ++ lib.optionals (lib.versionOlder prevAttrs.version "2022.2.0") [ qt5.qtwebview ] + ++ lib.optionals (lib.versionAtLeast prevAttrs.version "2022.2.0") [ qt6.qtwebview ]; + }; + + nsight_systems = + { + alsa-lib, + e2fsprogs, + lib, + nss, + numactl, + pulseaudio, + qt5, + qt6, + wayland, + xorg, + }: + prevAttrs: { + nativeBuildInputs = + prevAttrs.nativeBuildInputs + ++ lib.optionals (lib.versionOlder prevAttrs.version "2022.2.0") [ qt5.wrapQtAppsHook ] + ++ lib.optionals (lib.versionAtLeast prevAttrs.version "2022.2.0") [ qt6.wrapQtAppsHook ]; + buildInputs = + prevAttrs.buildInputs + ++ [ + alsa-lib + e2fsprogs + nss + numactl + pulseaudio + wayland + xorg.libXcursor + xorg.libXdamage + xorg.libXrandr + xorg.libXtst + ] + ++ lib.optionals (lib.versionOlder prevAttrs.version "2022.2.0") [ qt5.qtwebview ] + ++ lib.optionals (lib.versionAtLeast prevAttrs.version "2022.2.0") [ qt6.qtwebview ]; + }; + + nvidia_driver = + { }: + { + # No need to support this package as we have drivers already + # in linuxPackages. + meta.broken = true; + }; +} diff --git a/pkgs/development/cuda-modules/cuda/meta-fixup.nix b/pkgs/development/cuda-modules/cuda/meta-fixup.nix new file mode 100644 index 0000000000000..7cbe983b3bc45 --- /dev/null +++ b/pkgs/development/cuda-modules/cuda/meta-fixup.nix @@ -0,0 +1,19 @@ +# A function which when callPackage'd returns a function to be given to overrideAttrs. +{ + lib, + manifests, + nullableOr, + ... +}: +prevAttrs: +prevAttrs +// { + # Add the package-specific license. + meta = prevAttrs.meta // { + license = lib.licenses.nvidiaCudaRedist // { + url = + "https://developer.download.nvidia.com/compute/cuda/redist/" + + nullableOr "${prevAttrs.pname}/LICENSE.txt" manifests.redistrib.${prevAttrs.pname}.license_path; + }; + }; +} diff --git a/pkgs/development/cuda-modules/cuda/overrides.nix b/pkgs/development/cuda-modules/cuda/overrides.nix deleted file mode 100644 index fd32978bfb59c..0000000000000 --- a/pkgs/development/cuda-modules/cuda/overrides.nix +++ /dev/null @@ -1,129 +0,0 @@ -{cudaVersion, lib}: -let - inherit (lib) attrsets lists strings; - # cudaVersionOlder : Version -> Boolean - cudaVersionOlder = strings.versionOlder cudaVersion; - # cudaVersionAtLeast : Version -> Boolean - cudaVersionAtLeast = strings.versionAtLeast cudaVersion; - - addBuildInputs = - drv: buildInputs: - drv.overrideAttrs (prevAttrs: {buildInputs = prevAttrs.buildInputs ++ buildInputs;}); -in -# NOTE: Filter out attributes that are not present in the previous version of -# the package set. This is necessary to prevent the appearance of attributes -# like `cuda_nvcc` in `cudaPackages_10_0, which predates redistributables. -final: prev: -attrsets.filterAttrs (attr: _: (builtins.hasAttr attr prev)) { - libcufile = prev.libcufile.overrideAttrs ( - prevAttrs: { - buildInputs = prevAttrs.buildInputs ++ [ - final.libcublas.lib - final.pkgs.numactl - final.pkgs.rdma-core - ]; - # Before 11.7 libcufile depends on itself for some reason. - env.autoPatchelfIgnoreMissingDeps = - prevAttrs.env.autoPatchelfIgnoreMissingDeps - + strings.optionalString (cudaVersionOlder "11.7") " libcufile.so.0"; - } - ); - - libcusolver = addBuildInputs prev.libcusolver ( - # Always depends on this - [final.libcublas.lib] - # Dependency from 12.0 and on - ++ lists.optionals (cudaVersionAtLeast "12.0") [final.libnvjitlink.lib] - # Dependency from 12.1 and on - ++ lists.optionals (cudaVersionAtLeast "12.1") [final.libcusparse.lib] - ); - - libcusparse = addBuildInputs prev.libcusparse ( - lists.optionals (cudaVersionAtLeast "12.0") [final.libnvjitlink.lib] - ); - - cuda_compat = prev.cuda_compat.overrideAttrs ( - prevAttrs: { - env.autoPatchelfIgnoreMissingDeps = - prevAttrs.env.autoPatchelfIgnoreMissingDeps + " libnvrm_gpu.so libnvrm_mem.so libnvdla_runtime.so"; - # `cuda_compat` only works on aarch64-linux, and only when building for Jetson devices. - brokenConditions = prevAttrs.brokenConditions // { - "Trying to use cuda_compat on aarch64-linux targeting non-Jetson devices" = - !final.flags.isJetsonBuild; - }; - } - ); - - cuda_gdb = addBuildInputs prev.cuda_gdb ( - # x86_64 only needs gmp from 12.0 and on - lists.optionals (cudaVersionAtLeast "12.0") [final.pkgs.gmp] - ); - - cuda_nvcc = prev.cuda_nvcc.overrideAttrs ( - oldAttrs: { - propagatedBuildInputs = [final.setupCudaHook]; - - meta = (oldAttrs.meta or {}) // { - mainProgram = "nvcc"; - }; - } - ); - - cuda_nvprof = prev.cuda_nvprof.overrideAttrs ( - prevAttrs: {buildInputs = prevAttrs.buildInputs ++ [final.cuda_cupti.lib];} - ); - - cuda_demo_suite = addBuildInputs prev.cuda_demo_suite [ - final.pkgs.freeglut - final.pkgs.libGLU - final.pkgs.libglvnd - final.pkgs.mesa - final.libcufft.lib - final.libcurand.lib - ]; - - nsight_compute = prev.nsight_compute.overrideAttrs ( - prevAttrs: { - nativeBuildInputs = - prevAttrs.nativeBuildInputs - ++ ( - if (strings.versionOlder prev.nsight_compute.version "2022.2.0") then - [final.pkgs.qt5.wrapQtAppsHook] - else - [final.pkgs.qt6.wrapQtAppsHook] - ); - buildInputs = - prevAttrs.buildInputs - ++ ( - if (strings.versionOlder prev.nsight_compute.version "2022.2.0") then - [final.pkgs.qt5.qtwebview] - else - [final.pkgs.qt6.qtwebview] - ); - } - ); - - nsight_systems = prev.nsight_systems.overrideAttrs ( - prevAttrs: { - nativeBuildInputs = prevAttrs.nativeBuildInputs ++ [final.pkgs.qt5.wrapQtAppsHook]; - buildInputs = prevAttrs.buildInputs ++ [ - final.pkgs.alsa-lib - final.pkgs.e2fsprogs - final.pkgs.nss - final.pkgs.numactl - final.pkgs.pulseaudio - final.pkgs.wayland - final.pkgs.xorg.libXcursor - final.pkgs.xorg.libXdamage - final.pkgs.xorg.libXrandr - final.pkgs.xorg.libXtst - ]; - } - ); - - nvidia_driver = prev.nvidia_driver.overrideAttrs { - # No need to support this package as we have drivers already - # in linuxPackages. - meta.broken = true; - }; -} diff --git a/pkgs/development/cuda-modules/cudnn/fixup.nix b/pkgs/development/cuda-modules/cudnn/fixup.nix index 1fb5a6ad015e4..6059863960f02 100644 --- a/pkgs/development/cuda-modules/cudnn/fixup.nix +++ b/pkgs/development/cuda-modules/cudnn/fixup.nix @@ -1,7 +1,7 @@ { cudaVersion, fetchurl, - final, + cudaPackages, lib, package, patchelf, @@ -35,8 +35,8 @@ finalAttrs: prevAttrs: { buildInputs = prevAttrs.buildInputs ++ [zlib] - ++ lists.optionals finalAttrs.passthru.useCudatoolkitRunfile [final.cudatoolkit] - ++ lists.optionals (!finalAttrs.passthru.useCudatoolkitRunfile) [final.libcublas.lib]; + ++ lists.optionals finalAttrs.passthru.useCudatoolkitRunfile [cudaPackages.cudatoolkit] + ++ lists.optionals (!finalAttrs.passthru.useCudatoolkitRunfile) [cudaPackages.libcublas.lib]; # Tell autoPatchelf about runtime dependencies. # NOTE: Versions from CUDNN releases have four components. diff --git a/pkgs/development/cuda-modules/cudnn/releases.nix b/pkgs/development/cuda-modules/cudnn/releases.nix deleted file mode 100644 index fe1f1f8d91e94..0000000000000 --- a/pkgs/development/cuda-modules/cudnn/releases.nix +++ /dev/null @@ -1,262 +0,0 @@ -# NOTE: Check https://docs.nvidia.com/deeplearning/cudnn/archives/index.html for support matrices. -# Version policy is to keep the latest minor release for each major release. -{ - cudnn.releases = { - # jetson - linux-aarch64 = [ - { - version = "8.9.5.30"; - minCudaVersion = "12.0"; - maxCudaVersion = "12.2"; - url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-aarch64/cudnn-linux-aarch64-8.9.5.30_cuda12-archive.tar.xz"; - hash = "sha256-BJH3sC9VwiB362eL8xTB+RdSS9UHz1tlgjm/mKRyM6E="; - } - ]; - # powerpc - linux-ppc64le = []; - # server-grade arm - linux-sbsa = [ - { - version = "8.4.1.50"; - minCudaVersion = "11.0"; - maxCudaVersion = "11.7"; - url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-sbsa/cudnn-linux-sbsa-8.4.1.50_cuda11.6-archive.tar.xz"; - hash = "sha256-CxufrFt4l04v2qp0hD2xj2Ns6PPZmdYv8qYVuZePw2A="; - } - { - version = "8.5.0.96"; - minCudaVersion = "11.0"; - maxCudaVersion = "11.7"; - url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-sbsa/cudnn-linux-sbsa-8.5.0.96_cuda11-archive.tar.xz"; - hash = "sha256-hngKu+zUY05zY/rR0ACuI7eQWl+Dg73b9zMsaTR5Hd4="; - } - { - version = "8.6.0.163"; - minCudaVersion = "11.0"; - maxCudaVersion = "11.8"; - url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-sbsa/cudnn-linux-sbsa-8.6.0.163_cuda11-archive.tar.xz"; - hash = "sha256-oCAieNPL1POtw/eBa/9gcWIcsEKwkDaYtHesrIkorAY="; - } - { - version = "8.7.0.84"; - minCudaVersion = "11.0"; - maxCudaVersion = "11.8"; - url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-sbsa/cudnn-linux-sbsa-8.7.0.84_cuda11-archive.tar.xz"; - hash = "sha256-z5Z/eNv2wHUkPMg6oYdZ43DbN1SqFbEqChTov2ejqdQ="; - } - { - version = "8.8.1.3"; - minCudaVersion = "11.0"; - maxCudaVersion = "11.8"; - url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-sbsa/cudnn-linux-sbsa-8.8.1.3_cuda11-archive.tar.xz"; - hash = "sha256-OzWq+aQkmIbZONmWSYyFoZzem3RldoXyJy7GVT6GM1k="; - } - { - version = "8.8.1.3"; - minCudaVersion = "12.0"; - maxCudaVersion = "12.0"; - url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-sbsa/cudnn-linux-sbsa-8.8.1.3_cuda12-archive.tar.xz"; - hash = "sha256-njl3qhudBuuGC1gqyJM2MGdaAkMCnCWb/sW7VpmGfSA="; - } - { - version = "8.9.7.29"; - minCudaVersion = "11.0"; - maxCudaVersion = "11.8"; - url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-sbsa/cudnn-linux-sbsa-8.9.7.29_cuda11-archive.tar.xz"; - hash = "sha256-kcN8+0WPVBQZ6YUQ8TqvWXXAIyxhPhi3djhUkAdO6hc="; - } - { - version = "8.9.7.29"; - minCudaVersion = "12.0"; - maxCudaVersion = "12.2"; - url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-sbsa/cudnn-linux-sbsa-8.9.7.29_cuda12-archive.tar.xz"; - hash = "sha256-6Yt8gAEHheXVygHuTOm1sMjHNYfqb4ZIvjTT+NHUe9E="; - } - ]; - # x86_64 - linux-x86_64 = [ - { - version = "7.4.2.24"; - minCudaVersion = "10.0"; - maxCudaVersion = "10.0"; - url = "https://developer.download.nvidia.com/compute/redist/cudnn/v7.4.2/cudnn-10.0-linux-x64-v7.4.2.24.tgz"; - hash = "sha256-Lt/IagK1DRfojEeJVaMy5qHoF05+U6NFi06lH68C2qM="; - } - { - version = "7.6.5.32"; - minCudaVersion = "10.0"; - maxCudaVersion = "10.0"; - url = "https://developer.download.nvidia.com/compute/redist/cudnn/v7.6.5/cudnn-10.0-linux-x64-v7.6.5.32.tgz"; - hash = "sha256-KDVeOV8LK5OsLIO2E2CzW6bNA3fkTni+GXtrYbS0kro="; - } - { - version = "7.6.5.32"; - minCudaVersion = "10.1"; - maxCudaVersion = "10.1"; - url = "https://developer.download.nvidia.com/compute/redist/cudnn/v7.6.5/cudnn-10.1-linux-x64-v7.6.5.32.tgz"; - hash = "sha256-fq7IA5osMKsLx1jTA1iHZ2k972v0myJIWiwAvy4TbLM="; - } - { - version = "7.6.5.32"; - minCudaVersion = "10.2"; - maxCudaVersion = "10.2"; - url = "https://developer.download.nvidia.com/compute/redist/cudnn/v7.6.5/cudnn-10.2-linux-x64-v7.6.5.32.tgz"; - hash = "sha256-YAJn8squ0v1Y6yFLpmnY6jXzlqfRm5SCLms2+fcIjCA='"; - } - { - version = "8.0.5.39"; - minCudaVersion = "10.1"; - maxCudaVersion = "10.1"; - url = "https://developer.download.nvidia.com/compute/redist/cudnn/v8.0.5/cudnn-10.1-linux-x64-v8.0.5.39.tgz"; - hash = "sha256-kJCElSmIlrM6qVBjo0cfk8NmJ9esAcF9w211xl7qSgA="; - } - { - version = "8.0.5.39"; - minCudaVersion = "10.2"; - maxCudaVersion = "10.2"; - url = "https://developer.download.nvidia.com/compute/redist/cudnn/v8.0.5/cudnn-10.2-linux-x64-v8.0.5.39.tgz"; - hash = "sha256-IfhMBcZ78eyFnnfDjM1b8VSWT6HDCPRJlZvkw1bjgvM="; - } - { - version = "8.0.5.39"; - minCudaVersion = "11.0"; - maxCudaVersion = "11.0"; - url = "https://developer.download.nvidia.com/compute/redist/cudnn/v8.0.5/cudnn-11.0-linux-x64-v8.0.5.39.tgz"; - hash = "sha256-ThbueJXetKixwZS4ErpJWG730mkCBRQB03F1EYmKm3M="; - } - { - version = "8.0.5.39"; - minCudaVersion = "11.1"; - maxCudaVersion = "11.1"; - url = "https://developer.download.nvidia.com/compute/redist/cudnn/v8.0.5/cudnn-11.1-linux-x64-v8.0.5.39.tgz"; - hash = "sha256-HQRr+nk5navMb2yxUHkYdUQ5RC6gyp4Pvs3URvmwDM4="; - } - { - version = "8.1.1.33"; - minCudaVersion = "10.2"; - maxCudaVersion = "10.2"; - url = "https://developer.download.nvidia.com/compute/redist/cudnn/v8.1.1/cudnn-10.2-linux-x64-v8.1.1.33.tgz"; - hash = "sha256-Kkp7mabpv6aQ6xm7QeSVU/KnpJGls6v8rpAOFmxbbr0="; - } - { - version = "8.1.1.33"; - minCudaVersion = "11.0"; - maxCudaVersion = "11.2"; - url = "https://developer.download.nvidia.com/compute/redist/cudnn/v8.1.1/cudnn-11.2-linux-x64-v8.1.1.33.tgz"; - hash = "sha256-mKh4TpKGLyABjSDCgbMNSgzZUfk2lPZDPM9K6cUCumo="; - } - { - version = "8.2.4.15"; - minCudaVersion = "10.2"; - maxCudaVersion = "10.2"; - url = "https://developer.download.nvidia.com/compute/redist/cudnn/v8.2.4/cudnn-10.2-linux-x64-v8.2.4.15.tgz"; - hash = "sha256-0jyUoxFaHHcRamwSfZF1+/WfcjNkN08mo0aZB18yIvE="; - } - { - version = "8.2.4.15"; - minCudaVersion = "11.0"; - maxCudaVersion = "11.4"; - url = "https://developer.download.nvidia.com/compute/redist/cudnn/v8.2.4/cudnn-11.4-linux-x64-v8.2.4.15.tgz"; - hash = "sha256-Dl0t+JC5ln76ZhnaQhMQ2XMjVlp58FoajLm3Fluq0Nc="; - } - { - version = "8.3.3.40"; - minCudaVersion = "10.2"; - maxCudaVersion = "10.2"; - url = "https://developer.download.nvidia.com/compute/redist/cudnn/v8.3.3/local_installers/10.2/cudnn-linux-x86_64-8.3.3.40_cuda10.2-archive.tar.xz"; - hash = "sha256-2FVPKzLmKV1fyPOsJeaPlAWLAYyAHaucFD42gS+JJqs="; - } - { - version = "8.3.3.40"; - minCudaVersion = "11.0"; - maxCudaVersion = "11.6"; - url = "https://developer.download.nvidia.com/compute/redist/cudnn/v8.3.3/local_installers/11.5/cudnn-linux-x86_64-8.3.3.40_cuda11.5-archive.tar.xz"; - hash = "sha256-6r6Wx1zwPqT1N5iU2RTx+K4UzqsSGYnoSwg22Sf7dzE="; - } - { - version = "8.4.1.50"; - minCudaVersion = "10.2"; - maxCudaVersion = "10.2"; - url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-8.4.1.50_cuda10.2-archive.tar.xz"; - hash = "sha256-I88qMmU6lIiLVmaPuX7TTbisgTav839mssxUo3lQNjg="; - } - { - version = "8.4.1.50"; - minCudaVersion = "11.0"; - maxCudaVersion = "11.7"; - url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-8.4.1.50_cuda11.6-archive.tar.xz"; - hash = "sha256-7JbSN22B/KQr3T1MPXBambKaBlurV/kgVhx2PinGfQE="; - } - { - version = "8.5.0.96"; - minCudaVersion = "10.2"; - maxCudaVersion = "10.2"; - url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-8.5.0.96_cuda10-archive.tar.xz"; - hash = "sha256-1mzhbbzR40WKkHnQLtJHhg0vYgf7G8a0OBcCwIOkJjM="; - } - { - version = "8.5.0.96"; - minCudaVersion = "11.0"; - maxCudaVersion = "11.7"; - url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-8.5.0.96_cuda11-archive.tar.xz"; - hash = "sha256-VFSm/ZTwCHKMqumtrZk8ToXvNjAuJrzkO+p9RYpee20="; - } - { - version = "8.6.0.163"; - minCudaVersion = "10.2"; - maxCudaVersion = "10.2"; - url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-8.6.0.163_cuda10-archive.tar.xz"; - hash = "sha256-t4sr/GrFqqdxu2VhaJQk5K1Xm/0lU4chXG8hVL09R9k="; - } - { - version = "8.6.0.163"; - minCudaVersion = "11.0"; - maxCudaVersion = "11.8"; - url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-8.6.0.163_cuda11-archive.tar.xz"; - hash = "sha256-u8OW30cpTGV+3AnGAGdNYIyxv8gLgtz0VHBgwhcRFZ4="; - } - { - version = "8.7.0.84"; - minCudaVersion = "10.2"; - maxCudaVersion = "10.2"; - url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-8.7.0.84_cuda10-archive.tar.xz"; - hash = "sha256-bZhaqc8+GbPV2FQvvbbufd8VnEJgvfkICc2N3/gitRg="; - } - { - version = "8.7.0.84"; - minCudaVersion = "11.0"; - maxCudaVersion = "11.8"; - url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-8.7.0.84_cuda11-archive.tar.xz"; - hash = "sha256-l2xMunIzyXrnQAavq1Fyl2MAukD1slCiH4z3H1nJ920="; - } - { - version = "8.8.1.3"; - minCudaVersion = "11.0"; - maxCudaVersion = "11.8"; - url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-8.8.1.3_cuda11-archive.tar.xz"; - hash = "sha256-r3WEyuDMVSS1kT7wjCm6YVQRPGDrCjegWQqRtRWoqPk="; - } - { - version = "8.8.1.3"; - minCudaVersion = "12.0"; - maxCudaVersion = "12.0"; - url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-8.8.1.3_cuda12-archive.tar.xz"; - hash = "sha256-edd6dpx+cXWrx7XC7VxJQUjAYYqGQThyLIh/lcYjd3w="; - } - { - version = "8.9.7.29"; - minCudaVersion = "11.0"; - maxCudaVersion = "11.8"; - url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-8.9.7.29_cuda11-archive.tar.xz"; - hash = "sha256-o+JQkCjOzaARfOWg9CEGNG6C6G05D0u5R1r8l2x3QC4="; - } - { - version = "8.9.7.29"; - minCudaVersion = "12.0"; - maxCudaVersion = "12.2"; - url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-8.9.7.29_cuda12-archive.tar.xz"; - hash = "sha256-R1MzYlx+QqevPKCy91BqEG4wyTsaoAgc2cE++24h47s="; - } - ]; - }; -} diff --git a/pkgs/development/cuda-modules/cudnn/shims.nix b/pkgs/development/cuda-modules/cudnn/shims.nix index e9eca8ef7c8b9..8cfc6847665a3 100644 --- a/pkgs/development/cuda-modules/cudnn/shims.nix +++ b/pkgs/development/cuda-modules/cudnn/shims.nix @@ -1,12 +1,20 @@ -# Shims to mimic the shape of ../modules/generic/manifests/{feature,redistrib}/release.nix -{package, redistArch}: +# Shims to mimic the shape of ../modules/generic/manifests/{feature,redistrib}/manifest.nix { - featureRelease.${redistArch}.outputs = { - lib = true; - static = true; - dev = true; + lib, + package, + # redistArch :: Optional String + # String is null if the given architecture is unsupported. + redistArch, +}: +{ + feature.cudnn = lib.optionalAttrs (redistArch != null) { + ${redistArch}.outputs = { + lib = true; + static = true; + dev = true; + }; }; - redistribRelease = { + redistrib.cudnn = { name = "NVIDIA CUDA Deep Neural Network library (cuDNN)"; inherit (package) version; }; diff --git a/pkgs/development/cuda-modules/generic-builders/manifest.nix b/pkgs/development/cuda-modules/generic-builders/manifest.nix index 71c914c8c8f24..33fe02ec51c05 100644 --- a/pkgs/development/cuda-modules/generic-builders/manifest.nix +++ b/pkgs/development/cuda-modules/generic-builders/manifest.nix @@ -19,13 +19,14 @@ # Also known as the Redistributable Name. # redistName : String, redistName, + # See ./modules/generic/manifests + # Expected to have the following attributes: + # - feature + # - redistrib + manifests, # If libPath is non-null, it must be a subdirectory of `lib`. # The contents of `libPath` will be moved to the root of `lib`. libPath ? null, - # See ./modules/generic/manifests/redistrib/release.nix - redistribRelease, - # See ./modules/generic/manifests/feature/release.nix - featureRelease, }: let inherit (lib) @@ -39,6 +40,8 @@ let sourceTypes ; + redistribRelease = manifests.redistrib.${pname}; + featureRelease = manifests.feature.${pname}; # Get the redist architectures for which package provides distributables. # These are used by meta.platforms. supportedRedistArchs = builtins.attrNames featureRelease; diff --git a/pkgs/development/cuda-modules/generic-builders/multiplex.nix b/pkgs/development/cuda-modules/generic-builders/multiplex.nix index b8053094bcc82..c33da70329361 100644 --- a/pkgs/development/cuda-modules/generic-builders/multiplex.nix +++ b/pkgs/development/cuda-modules/generic-builders/multiplex.nix @@ -1,5 +1,6 @@ { # callPackage-provided arguments + callPackage, lib, cudaVersion, flags, @@ -8,9 +9,7 @@ mkVersionedPackageName, # pname :: String pname, - # releasesModule :: Path - # A path to a module which provides a `releases` attribute - releasesModule, + evaluatedModules, # shims :: Path # A path to a module which provides a `shims` attribute # The redistribRelease is only used in ./manifest.nix for the package version @@ -48,13 +47,6 @@ let strings ; - evaluatedModules = modules.evalModules { - modules = [ - ../modules - releasesModule - ]; - }; - # NOTE: Important types: # - Releases: ../modules/${pname}/releases/releases.nix # - Package: ../modules/${pname}/releases/package.nix @@ -66,7 +58,7 @@ let # Compute versioned attribute name to be used in this package set # Patch version changes should not break the build, so we only use major and minor # computeName :: Package -> String - computeName = {version, ...}: mkVersionedPackageName pname version; + computeName = { version, ... }: mkVersionedPackageName pname version; # Check whether a package supports our CUDA version # isSupported :: Package -> Bool @@ -80,7 +72,8 @@ let # All the supported packages we can build for our platform. # supportedPackages :: List (AttrSet Packages) - supportedPackages = builtins.filter isSupported (allReleases.${redistArch} or []); + supportedPackages = + if redistArch == null then [ ] else builtins.filter isSupported (allReleases.${redistArch} or [ ]); # newestToOldestSupportedPackage :: List (AttrSet Packages) newestToOldestSupportedPackage = lists.reverseList supportedPackages; @@ -90,42 +83,28 @@ let # A function which takes the `final` overlay and the `package` being built and returns # a function to be consumed via `overrideAttrs`. overrideAttrsFixupFn = - final: package: - final.callPackage fixupFn { - inherit - final - cudaVersion - mkVersionedPackageName - package - ; - }; + package: callPackage fixupFn { inherit cudaVersion mkVersionedPackageName package; }; - extension = - final: _: + # Builds our package into derivation and wraps it in a nameValuePair, where the name is the versioned name + # of the package. + buildPackage = + package: let - # Builds our package into derivation and wraps it in a nameValuePair, where the name is the versioned name - # of the package. - buildPackage = - package: - let - shims = final.callPackage shimsFn {inherit package redistArch;}; - name = computeName package; - drv = final.callPackage ./manifest.nix { - inherit pname; - redistName = pname; - inherit (shims) redistribRelease featureRelease; - }; - fixedDrv = drv.overrideAttrs (overrideAttrsFixupFn final package); - in - attrsets.nameValuePair name fixedDrv; - - # versionedDerivations :: AttrSet Derivation - versionedDerivations = builtins.listToAttrs (lists.map buildPackage newestToOldestSupportedPackage); - - defaultDerivation = attrsets.optionalAttrs (versionedDerivations != {}) { - ${pname} = versionedDerivations.${nameOfNewest}; + manifests = callPackage shimsFn { inherit package redistArch; }; + name = computeName package; + drv = callPackage ./manifest.nix { + inherit pname manifests; + redistName = pname; }; + fixedDrv = drv.overrideAttrs (overrideAttrsFixupFn package); in - versionedDerivations // defaultDerivation; + attrsets.nameValuePair name fixedDrv; + + # versionedDerivations :: AttrSet Derivation + versionedDerivations = builtins.listToAttrs (lists.map buildPackage newestToOldestSupportedPackage); + + defaultDerivation = attrsets.optionalAttrs (versionedDerivations != { }) { + ${pname} = versionedDerivations.${nameOfNewest}; + }; in -extension +versionedDerivations // defaultDerivation diff --git a/pkgs/development/cuda-modules/modules/cuda/default.nix b/pkgs/development/cuda-modules/modules/cuda/default.nix index 4ea35d0482265..6fd52c8f23523 100644 --- a/pkgs/development/cuda-modules/modules/cuda/default.nix +++ b/pkgs/development/cuda-modules/modules/cuda/default.nix @@ -1 +1,48 @@ -{options, ...}: {options.cuda.manifests = options.generic.manifests;} +{ + config, + lib, + options, + ... +}: +let + inherit (lib) filesystem trivial versions; + + # Read all the files in the manifests directory + # manifestFiles :: [ Path ] + cudaVersionMajor = versions.major config.cudaVersion; + cudaVersionMinor = versions.minor config.cudaVersion; + filenameRegexFn = builtins.match ".+(feature|redistrib)_${cudaVersionMajor}\\.${cudaVersionMinor}\\.[[:digit:]]+\\.json$"; + manifests = + builtins.foldl' + ( + manifestAttrs: filename: + let + matches = filenameRegexFn (builtins.toString filename); + in + if matches == null then + # No matches! + manifestAttrs + else + let + fileKind = + trivial.throwIfNot (builtins.length matches == 1) + "Unexpected error while processing manifest files: multiple regex matches in filename ${builtins.toString filename}" + (builtins.head matches); + newAttr = + trivial.throwIf (builtins.hasAttr fileKind manifestAttrs) + "Unexpected error while processing manifest files: duplicate file kind ${builtins.toString fileKind}" + { "${fileKind}" = trivial.importJSON filename; }; + in + manifestAttrs // newAttr + ) + { } + (filesystem.listFilesRecursive ./manifests); +in +{ + options.cuda.manifests = options.generic.manifests; + config.cuda.manifests = + # It's okay if neither feature nor redistrib manifests are found, but if one is found, the other must be found as well + trivial.throwIf (builtins.hasAttr "feature" manifests != builtins.hasAttr "redistrib" manifests) + "Both feature and redistrib manifests must be present, or neither" + manifests; +} diff --git a/pkgs/development/cuda-modules/cuda/manifests/feature_11.4.4.json b/pkgs/development/cuda-modules/modules/cuda/manifests/feature_11.4.4.json similarity index 100% rename from pkgs/development/cuda-modules/cuda/manifests/feature_11.4.4.json rename to pkgs/development/cuda-modules/modules/cuda/manifests/feature_11.4.4.json diff --git a/pkgs/development/cuda-modules/cuda/manifests/feature_11.5.2.json b/pkgs/development/cuda-modules/modules/cuda/manifests/feature_11.5.2.json similarity index 100% rename from pkgs/development/cuda-modules/cuda/manifests/feature_11.5.2.json rename to pkgs/development/cuda-modules/modules/cuda/manifests/feature_11.5.2.json diff --git a/pkgs/development/cuda-modules/cuda/manifests/feature_11.6.2.json b/pkgs/development/cuda-modules/modules/cuda/manifests/feature_11.6.2.json similarity index 100% rename from pkgs/development/cuda-modules/cuda/manifests/feature_11.6.2.json rename to pkgs/development/cuda-modules/modules/cuda/manifests/feature_11.6.2.json diff --git a/pkgs/development/cuda-modules/cuda/manifests/feature_11.7.1.json b/pkgs/development/cuda-modules/modules/cuda/manifests/feature_11.7.1.json similarity index 100% rename from pkgs/development/cuda-modules/cuda/manifests/feature_11.7.1.json rename to pkgs/development/cuda-modules/modules/cuda/manifests/feature_11.7.1.json diff --git a/pkgs/development/cuda-modules/cuda/manifests/feature_11.8.0.json b/pkgs/development/cuda-modules/modules/cuda/manifests/feature_11.8.0.json similarity index 100% rename from pkgs/development/cuda-modules/cuda/manifests/feature_11.8.0.json rename to pkgs/development/cuda-modules/modules/cuda/manifests/feature_11.8.0.json diff --git a/pkgs/development/cuda-modules/cuda/manifests/feature_12.0.1.json b/pkgs/development/cuda-modules/modules/cuda/manifests/feature_12.0.1.json similarity index 100% rename from pkgs/development/cuda-modules/cuda/manifests/feature_12.0.1.json rename to pkgs/development/cuda-modules/modules/cuda/manifests/feature_12.0.1.json diff --git a/pkgs/development/cuda-modules/cuda/manifests/feature_12.1.1.json b/pkgs/development/cuda-modules/modules/cuda/manifests/feature_12.1.1.json similarity index 100% rename from pkgs/development/cuda-modules/cuda/manifests/feature_12.1.1.json rename to pkgs/development/cuda-modules/modules/cuda/manifests/feature_12.1.1.json diff --git a/pkgs/development/cuda-modules/cuda/manifests/feature_12.2.2.json b/pkgs/development/cuda-modules/modules/cuda/manifests/feature_12.2.2.json similarity index 100% rename from pkgs/development/cuda-modules/cuda/manifests/feature_12.2.2.json rename to pkgs/development/cuda-modules/modules/cuda/manifests/feature_12.2.2.json diff --git a/pkgs/development/cuda-modules/cuda/manifests/feature_12.3.0.json b/pkgs/development/cuda-modules/modules/cuda/manifests/feature_12.3.0.json similarity index 100% rename from pkgs/development/cuda-modules/cuda/manifests/feature_12.3.0.json rename to pkgs/development/cuda-modules/modules/cuda/manifests/feature_12.3.0.json diff --git a/pkgs/development/cuda-modules/cuda/manifests/redistrib_11.4.4.json b/pkgs/development/cuda-modules/modules/cuda/manifests/redistrib_11.4.4.json similarity index 100% rename from pkgs/development/cuda-modules/cuda/manifests/redistrib_11.4.4.json rename to pkgs/development/cuda-modules/modules/cuda/manifests/redistrib_11.4.4.json diff --git a/pkgs/development/cuda-modules/cuda/manifests/redistrib_11.5.2.json b/pkgs/development/cuda-modules/modules/cuda/manifests/redistrib_11.5.2.json similarity index 100% rename from pkgs/development/cuda-modules/cuda/manifests/redistrib_11.5.2.json rename to pkgs/development/cuda-modules/modules/cuda/manifests/redistrib_11.5.2.json diff --git a/pkgs/development/cuda-modules/cuda/manifests/redistrib_11.6.2.json b/pkgs/development/cuda-modules/modules/cuda/manifests/redistrib_11.6.2.json similarity index 100% rename from pkgs/development/cuda-modules/cuda/manifests/redistrib_11.6.2.json rename to pkgs/development/cuda-modules/modules/cuda/manifests/redistrib_11.6.2.json diff --git a/pkgs/development/cuda-modules/cuda/manifests/redistrib_11.7.1.json b/pkgs/development/cuda-modules/modules/cuda/manifests/redistrib_11.7.1.json similarity index 100% rename from pkgs/development/cuda-modules/cuda/manifests/redistrib_11.7.1.json rename to pkgs/development/cuda-modules/modules/cuda/manifests/redistrib_11.7.1.json diff --git a/pkgs/development/cuda-modules/cuda/manifests/redistrib_11.8.0.json b/pkgs/development/cuda-modules/modules/cuda/manifests/redistrib_11.8.0.json similarity index 100% rename from pkgs/development/cuda-modules/cuda/manifests/redistrib_11.8.0.json rename to pkgs/development/cuda-modules/modules/cuda/manifests/redistrib_11.8.0.json diff --git a/pkgs/development/cuda-modules/cuda/manifests/redistrib_12.0.1.json b/pkgs/development/cuda-modules/modules/cuda/manifests/redistrib_12.0.1.json similarity index 100% rename from pkgs/development/cuda-modules/cuda/manifests/redistrib_12.0.1.json rename to pkgs/development/cuda-modules/modules/cuda/manifests/redistrib_12.0.1.json diff --git a/pkgs/development/cuda-modules/cuda/manifests/redistrib_12.1.1.json b/pkgs/development/cuda-modules/modules/cuda/manifests/redistrib_12.1.1.json similarity index 100% rename from pkgs/development/cuda-modules/cuda/manifests/redistrib_12.1.1.json rename to pkgs/development/cuda-modules/modules/cuda/manifests/redistrib_12.1.1.json diff --git a/pkgs/development/cuda-modules/cuda/manifests/redistrib_12.2.2.json b/pkgs/development/cuda-modules/modules/cuda/manifests/redistrib_12.2.2.json similarity index 100% rename from pkgs/development/cuda-modules/cuda/manifests/redistrib_12.2.2.json rename to pkgs/development/cuda-modules/modules/cuda/manifests/redistrib_12.2.2.json diff --git a/pkgs/development/cuda-modules/cuda/manifests/redistrib_12.3.0.json b/pkgs/development/cuda-modules/modules/cuda/manifests/redistrib_12.3.0.json similarity index 100% rename from pkgs/development/cuda-modules/cuda/manifests/redistrib_12.3.0.json rename to pkgs/development/cuda-modules/modules/cuda/manifests/redistrib_12.3.0.json diff --git a/pkgs/development/cuda-modules/modules/cudnn/default.nix b/pkgs/development/cuda-modules/modules/cudnn/default.nix index dd52cbaa24b4d..e04fda735b9bb 100644 --- a/pkgs/development/cuda-modules/modules/cudnn/default.nix +++ b/pkgs/development/cuda-modules/modules/cudnn/default.nix @@ -1,6 +1,7 @@ {options, ...}: { options.cudnn.releases = options.generic.releases; + config.cudnn.releases = builtins.import ./releases.nix; # TODO(@connorbaker): Figure out how to add additional options to the # to the generic release. # { diff --git a/pkgs/development/cuda-modules/modules/cudnn/releases.nix b/pkgs/development/cuda-modules/modules/cudnn/releases.nix new file mode 100644 index 0000000000000..4ef3d2486ea1d --- /dev/null +++ b/pkgs/development/cuda-modules/modules/cudnn/releases.nix @@ -0,0 +1,260 @@ +# NOTE: Check https://docs.nvidia.com/deeplearning/cudnn/archives/index.html for support matrices. +# Version policy is to keep the latest minor release for each major release. +{ + # jetson + linux-aarch64 = [ + { + version = "8.9.5.30"; + minCudaVersion = "12.0"; + maxCudaVersion = "12.2"; + url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-aarch64/cudnn-linux-aarch64-8.9.5.30_cuda12-archive.tar.xz"; + hash = "sha256-BJH3sC9VwiB362eL8xTB+RdSS9UHz1tlgjm/mKRyM6E="; + } + ]; + # powerpc + linux-ppc64le = [ ]; + # server-grade arm + linux-sbsa = [ + { + version = "8.4.1.50"; + minCudaVersion = "11.0"; + maxCudaVersion = "11.7"; + url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-sbsa/cudnn-linux-sbsa-8.4.1.50_cuda11.6-archive.tar.xz"; + hash = "sha256-CxufrFt4l04v2qp0hD2xj2Ns6PPZmdYv8qYVuZePw2A="; + } + { + version = "8.5.0.96"; + minCudaVersion = "11.0"; + maxCudaVersion = "11.7"; + url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-sbsa/cudnn-linux-sbsa-8.5.0.96_cuda11-archive.tar.xz"; + hash = "sha256-hngKu+zUY05zY/rR0ACuI7eQWl+Dg73b9zMsaTR5Hd4="; + } + { + version = "8.6.0.163"; + minCudaVersion = "11.0"; + maxCudaVersion = "11.8"; + url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-sbsa/cudnn-linux-sbsa-8.6.0.163_cuda11-archive.tar.xz"; + hash = "sha256-oCAieNPL1POtw/eBa/9gcWIcsEKwkDaYtHesrIkorAY="; + } + { + version = "8.7.0.84"; + minCudaVersion = "11.0"; + maxCudaVersion = "11.8"; + url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-sbsa/cudnn-linux-sbsa-8.7.0.84_cuda11-archive.tar.xz"; + hash = "sha256-z5Z/eNv2wHUkPMg6oYdZ43DbN1SqFbEqChTov2ejqdQ="; + } + { + version = "8.8.1.3"; + minCudaVersion = "11.0"; + maxCudaVersion = "11.8"; + url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-sbsa/cudnn-linux-sbsa-8.8.1.3_cuda11-archive.tar.xz"; + hash = "sha256-OzWq+aQkmIbZONmWSYyFoZzem3RldoXyJy7GVT6GM1k="; + } + { + version = "8.8.1.3"; + minCudaVersion = "12.0"; + maxCudaVersion = "12.0"; + url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-sbsa/cudnn-linux-sbsa-8.8.1.3_cuda12-archive.tar.xz"; + hash = "sha256-njl3qhudBuuGC1gqyJM2MGdaAkMCnCWb/sW7VpmGfSA="; + } + { + version = "8.9.7.29"; + minCudaVersion = "11.0"; + maxCudaVersion = "11.8"; + url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-sbsa/cudnn-linux-sbsa-8.9.7.29_cuda11-archive.tar.xz"; + hash = "sha256-kcN8+0WPVBQZ6YUQ8TqvWXXAIyxhPhi3djhUkAdO6hc="; + } + { + version = "8.9.7.29"; + minCudaVersion = "12.0"; + maxCudaVersion = "12.2"; + url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-sbsa/cudnn-linux-sbsa-8.9.7.29_cuda12-archive.tar.xz"; + hash = "sha256-6Yt8gAEHheXVygHuTOm1sMjHNYfqb4ZIvjTT+NHUe9E="; + } + ]; + # x86_64 + linux-x86_64 = [ + { + version = "7.4.2.24"; + minCudaVersion = "10.0"; + maxCudaVersion = "10.0"; + url = "https://developer.download.nvidia.com/compute/redist/cudnn/v7.4.2/cudnn-10.0-linux-x64-v7.4.2.24.tgz"; + hash = "sha256-Lt/IagK1DRfojEeJVaMy5qHoF05+U6NFi06lH68C2qM="; + } + { + version = "7.6.5.32"; + minCudaVersion = "10.0"; + maxCudaVersion = "10.0"; + url = "https://developer.download.nvidia.com/compute/redist/cudnn/v7.6.5/cudnn-10.0-linux-x64-v7.6.5.32.tgz"; + hash = "sha256-KDVeOV8LK5OsLIO2E2CzW6bNA3fkTni+GXtrYbS0kro="; + } + { + version = "7.6.5.32"; + minCudaVersion = "10.1"; + maxCudaVersion = "10.1"; + url = "https://developer.download.nvidia.com/compute/redist/cudnn/v7.6.5/cudnn-10.1-linux-x64-v7.6.5.32.tgz"; + hash = "sha256-fq7IA5osMKsLx1jTA1iHZ2k972v0myJIWiwAvy4TbLM="; + } + { + version = "7.6.5.32"; + minCudaVersion = "10.2"; + maxCudaVersion = "10.2"; + url = "https://developer.download.nvidia.com/compute/redist/cudnn/v7.6.5/cudnn-10.2-linux-x64-v7.6.5.32.tgz"; + hash = "sha256-YAJn8squ0v1Y6yFLpmnY6jXzlqfRm5SCLms2+fcIjCA='"; + } + { + version = "8.0.5.39"; + minCudaVersion = "10.1"; + maxCudaVersion = "10.1"; + url = "https://developer.download.nvidia.com/compute/redist/cudnn/v8.0.5/cudnn-10.1-linux-x64-v8.0.5.39.tgz"; + hash = "sha256-kJCElSmIlrM6qVBjo0cfk8NmJ9esAcF9w211xl7qSgA="; + } + { + version = "8.0.5.39"; + minCudaVersion = "10.2"; + maxCudaVersion = "10.2"; + url = "https://developer.download.nvidia.com/compute/redist/cudnn/v8.0.5/cudnn-10.2-linux-x64-v8.0.5.39.tgz"; + hash = "sha256-IfhMBcZ78eyFnnfDjM1b8VSWT6HDCPRJlZvkw1bjgvM="; + } + { + version = "8.0.5.39"; + minCudaVersion = "11.0"; + maxCudaVersion = "11.0"; + url = "https://developer.download.nvidia.com/compute/redist/cudnn/v8.0.5/cudnn-11.0-linux-x64-v8.0.5.39.tgz"; + hash = "sha256-ThbueJXetKixwZS4ErpJWG730mkCBRQB03F1EYmKm3M="; + } + { + version = "8.0.5.39"; + minCudaVersion = "11.1"; + maxCudaVersion = "11.1"; + url = "https://developer.download.nvidia.com/compute/redist/cudnn/v8.0.5/cudnn-11.1-linux-x64-v8.0.5.39.tgz"; + hash = "sha256-HQRr+nk5navMb2yxUHkYdUQ5RC6gyp4Pvs3URvmwDM4="; + } + { + version = "8.1.1.33"; + minCudaVersion = "10.2"; + maxCudaVersion = "10.2"; + url = "https://developer.download.nvidia.com/compute/redist/cudnn/v8.1.1/cudnn-10.2-linux-x64-v8.1.1.33.tgz"; + hash = "sha256-Kkp7mabpv6aQ6xm7QeSVU/KnpJGls6v8rpAOFmxbbr0="; + } + { + version = "8.1.1.33"; + minCudaVersion = "11.0"; + maxCudaVersion = "11.2"; + url = "https://developer.download.nvidia.com/compute/redist/cudnn/v8.1.1/cudnn-11.2-linux-x64-v8.1.1.33.tgz"; + hash = "sha256-mKh4TpKGLyABjSDCgbMNSgzZUfk2lPZDPM9K6cUCumo="; + } + { + version = "8.2.4.15"; + minCudaVersion = "10.2"; + maxCudaVersion = "10.2"; + url = "https://developer.download.nvidia.com/compute/redist/cudnn/v8.2.4/cudnn-10.2-linux-x64-v8.2.4.15.tgz"; + hash = "sha256-0jyUoxFaHHcRamwSfZF1+/WfcjNkN08mo0aZB18yIvE="; + } + { + version = "8.2.4.15"; + minCudaVersion = "11.0"; + maxCudaVersion = "11.4"; + url = "https://developer.download.nvidia.com/compute/redist/cudnn/v8.2.4/cudnn-11.4-linux-x64-v8.2.4.15.tgz"; + hash = "sha256-Dl0t+JC5ln76ZhnaQhMQ2XMjVlp58FoajLm3Fluq0Nc="; + } + { + version = "8.3.3.40"; + minCudaVersion = "10.2"; + maxCudaVersion = "10.2"; + url = "https://developer.download.nvidia.com/compute/redist/cudnn/v8.3.3/local_installers/10.2/cudnn-linux-x86_64-8.3.3.40_cuda10.2-archive.tar.xz"; + hash = "sha256-2FVPKzLmKV1fyPOsJeaPlAWLAYyAHaucFD42gS+JJqs="; + } + { + version = "8.3.3.40"; + minCudaVersion = "11.0"; + maxCudaVersion = "11.6"; + url = "https://developer.download.nvidia.com/compute/redist/cudnn/v8.3.3/local_installers/11.5/cudnn-linux-x86_64-8.3.3.40_cuda11.5-archive.tar.xz"; + hash = "sha256-6r6Wx1zwPqT1N5iU2RTx+K4UzqsSGYnoSwg22Sf7dzE="; + } + { + version = "8.4.1.50"; + minCudaVersion = "10.2"; + maxCudaVersion = "10.2"; + url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-8.4.1.50_cuda10.2-archive.tar.xz"; + hash = "sha256-I88qMmU6lIiLVmaPuX7TTbisgTav839mssxUo3lQNjg="; + } + { + version = "8.4.1.50"; + minCudaVersion = "11.0"; + maxCudaVersion = "11.7"; + url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-8.4.1.50_cuda11.6-archive.tar.xz"; + hash = "sha256-7JbSN22B/KQr3T1MPXBambKaBlurV/kgVhx2PinGfQE="; + } + { + version = "8.5.0.96"; + minCudaVersion = "10.2"; + maxCudaVersion = "10.2"; + url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-8.5.0.96_cuda10-archive.tar.xz"; + hash = "sha256-1mzhbbzR40WKkHnQLtJHhg0vYgf7G8a0OBcCwIOkJjM="; + } + { + version = "8.5.0.96"; + minCudaVersion = "11.0"; + maxCudaVersion = "11.7"; + url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-8.5.0.96_cuda11-archive.tar.xz"; + hash = "sha256-VFSm/ZTwCHKMqumtrZk8ToXvNjAuJrzkO+p9RYpee20="; + } + { + version = "8.6.0.163"; + minCudaVersion = "10.2"; + maxCudaVersion = "10.2"; + url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-8.6.0.163_cuda10-archive.tar.xz"; + hash = "sha256-t4sr/GrFqqdxu2VhaJQk5K1Xm/0lU4chXG8hVL09R9k="; + } + { + version = "8.6.0.163"; + minCudaVersion = "11.0"; + maxCudaVersion = "11.8"; + url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-8.6.0.163_cuda11-archive.tar.xz"; + hash = "sha256-u8OW30cpTGV+3AnGAGdNYIyxv8gLgtz0VHBgwhcRFZ4="; + } + { + version = "8.7.0.84"; + minCudaVersion = "10.2"; + maxCudaVersion = "10.2"; + url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-8.7.0.84_cuda10-archive.tar.xz"; + hash = "sha256-bZhaqc8+GbPV2FQvvbbufd8VnEJgvfkICc2N3/gitRg="; + } + { + version = "8.7.0.84"; + minCudaVersion = "11.0"; + maxCudaVersion = "11.8"; + url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-8.7.0.84_cuda11-archive.tar.xz"; + hash = "sha256-l2xMunIzyXrnQAavq1Fyl2MAukD1slCiH4z3H1nJ920="; + } + { + version = "8.8.1.3"; + minCudaVersion = "11.0"; + maxCudaVersion = "11.8"; + url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-8.8.1.3_cuda11-archive.tar.xz"; + hash = "sha256-r3WEyuDMVSS1kT7wjCm6YVQRPGDrCjegWQqRtRWoqPk="; + } + { + version = "8.8.1.3"; + minCudaVersion = "12.0"; + maxCudaVersion = "12.0"; + url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-8.8.1.3_cuda12-archive.tar.xz"; + hash = "sha256-edd6dpx+cXWrx7XC7VxJQUjAYYqGQThyLIh/lcYjd3w="; + } + { + version = "8.9.7.29"; + minCudaVersion = "11.0"; + maxCudaVersion = "11.8"; + url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-8.9.7.29_cuda11-archive.tar.xz"; + hash = "sha256-o+JQkCjOzaARfOWg9CEGNG6C6G05D0u5R1r8l2x3QC4="; + } + { + version = "8.9.7.29"; + minCudaVersion = "12.0"; + maxCudaVersion = "12.2"; + url = "https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-8.9.7.29_cuda12-archive.tar.xz"; + hash = "sha256-R1MzYlx+QqevPKCy91BqEG4wyTsaoAgc2cE++24h47s="; + } + ]; +} diff --git a/pkgs/development/cuda-modules/modules/generic/manifests/feature/manifest.nix b/pkgs/development/cuda-modules/modules/generic/manifests/feature/manifest.nix index 29ca678e0e5a5..d8b8e90af147c 100644 --- a/pkgs/development/cuda-modules/modules/generic/manifests/feature/manifest.nix +++ b/pkgs/development/cuda-modules/modules/generic/manifests/feature/manifest.nix @@ -5,6 +5,6 @@ let in options.mkOption { description = "A feature manifest is an attribute set which includes a mapping from package name to release"; - example = trivial.importJSON ../../../../cuda/manifests/feature_11.5.2.json; + example = trivial.importJSON ../../../cuda/manifests/feature_11.5.2.json; type = types.attrsOf Release.type; } diff --git a/pkgs/development/cuda-modules/modules/generic/manifests/redistrib/manifest.nix b/pkgs/development/cuda-modules/modules/generic/manifests/redistrib/manifest.nix index 0cfa40241fdc0..fc9fbf7d603ba 100644 --- a/pkgs/development/cuda-modules/modules/generic/manifests/redistrib/manifest.nix +++ b/pkgs/development/cuda-modules/modules/generic/manifests/redistrib/manifest.nix @@ -5,7 +5,7 @@ let in options.mkOption { description = "A redistributable manifest is an attribute set which includes a mapping from package name to release"; - example = trivial.importJSON ../../../../cuda/manifests/redistrib_11.5.2.json; + example = trivial.importJSON ../../../cuda/manifests/redistrib_11.5.2.json; type = types.submodule { # Allow any attribute name as these will be the package names freeformType = types.attrsOf Release.type; diff --git a/pkgs/development/cuda-modules/setup-hooks/extension.nix b/pkgs/development/cuda-modules/setup-hooks/default.nix similarity index 86% rename from pkgs/development/cuda-modules/setup-hooks/extension.nix rename to pkgs/development/cuda-modules/setup-hooks/default.nix index 930730ce6c06b..b6db0729ce451 100644 --- a/pkgs/development/cuda-modules/setup-hooks/extension.nix +++ b/pkgs/development/cuda-modules/setup-hooks/default.nix @@ -1,8 +1,9 @@ -final: _: { +{callPackage}: +{ # Internal hook, used by cudatoolkit and cuda redist packages # to accommodate automatic CUDAToolkit_ROOT construction markForCudatoolkitRootHook = - final.callPackage + callPackage ( {makeSetupHook}: makeSetupHook {name = "mark-for-cudatoolkit-root-hook";} ./mark-for-cudatoolkit-root-hook.sh @@ -11,7 +12,7 @@ final: _: { # Currently propagated by cuda_nvcc or cudatoolkit, rather than used directly setupCudaHook = - (final.callPackage + (callPackage ( {makeSetupHook, backendStdenv}: makeSetupHook @@ -33,7 +34,7 @@ final: _: { ); autoAddOpenGLRunpathHook = - final.callPackage + callPackage ( {addOpenGLRunpath, makeSetupHook}: makeSetupHook @@ -51,17 +52,17 @@ final: _: { # it doesn't have any effect) and thus appear first. Meaning this hook must be # executed last. autoAddCudaCompatRunpathHook = - final.callPackage + callPackage ( - {makeSetupHook, cuda_compat}: + {makeSetupHook, cuda_compat, flags}: makeSetupHook { name = "auto-add-cuda-compat-runpath-hook"; substitutions = { # Hotfix Ofborg evaluation - libcudaPath = if final.flags.isJetsonBuild then "${cuda_compat}/compat" else null; + libcudaPath = if flags.isJetsonBuild then "${cuda_compat}/compat" else null; }; - meta.broken = !final.flags.isJetsonBuild; + meta.broken = !flags.isJetsonBuild; } ./auto-add-cuda-compat-runpath.sh ) diff --git a/pkgs/top-level/cuda-packages.nix b/pkgs/top-level/cuda-packages.nix index d474cf852e55f..25b80051028c2 100644 --- a/pkgs/top-level/cuda-packages.nix +++ b/pkgs/top-level/cuda-packages.nix @@ -24,6 +24,7 @@ callPackage, cudaVersion, lib, + hostPlatform, newScope, pkgs, __attrsFailEvaluation ? true, @@ -31,23 +32,75 @@ let inherit (lib) attrsets - customisation fixedPoints + modules + options strings + trivial + types versions ; + # Utility function + # nullableOr : Optional a -> a -> a + nullableOr = nullable: default: if nullable != null then nullable else default; + + evaluatedModules = modules.evalModules { + modules = [ + { + options.cudaVersion = options.mkOption { + description = "The version of CUDA to use."; + type = types.str; + }; + config = { + inherit cudaVersion; + }; + } + ../development/cuda-modules/modules + ]; + }; + + genericManifestBuilderFn = + { + callPackage, + pname, + redistName, + manifests, + libPath ? null, + overrideAttrsFns ? [ ], + }: + let + drv = callPackage ../development/cuda-modules/generic-builders/manifest.nix { + inherit + pname + redistName + manifests + libPath + ; + }; + in + builtins.foldl' (drv: overrideAttrsFn: drv.overrideAttrs overrideAttrsFn) drv overrideAttrsFns; + # Backbone gpus = builtins.import ../development/cuda-modules/gpus.nix; nvccCompatibilities = builtins.import ../development/cuda-modules/nvcc-compatibilities.nix; - flags = callPackage ../development/cuda-modules/flags.nix {inherit cudaVersion gpus;}; + flags = callPackage ../development/cuda-modules/flags.nix { inherit cudaVersion gpus; }; passthruFunction = - final: ( + final: { inherit cudaVersion lib pkgs; - inherit gpus nvccCompatibilities flags; + inherit + gpus + nvccCompatibilities + flags + evaluatedModules + ; cudaMajorVersion = versions.major cudaVersion; cudaMajorMinorVersion = versions.majorMinor cudaVersion; + # cudaVersionOlder : Version -> Boolean + cudaVersionOlder = strings.versionOlder cudaVersion; + # cudaVersionAtLeast : Version -> Boolean + cudaVersionAtLeast = strings.versionAtLeast cudaVersion; # Maintain a reference to the final cudaPackages. # Without this, if we use `final.callPackage` and a package accepts `cudaPackages` as an argument, @@ -66,57 +119,116 @@ let # when linked with other C++ libraries. # E.g. for cudaPackages_11_8 we use gcc11 with gcc12's libstdc++ # Cf. https://github.com/NixOS/nixpkgs/pull/218265 for context - backendStdenv = final.callPackage ../development/cuda-modules/backend-stdenv.nix {}; + backendStdenv = final.callPackage ../development/cuda-modules/backend-stdenv.nix { }; # Loose packages - cudatoolkit = final.callPackage ../development/cuda-modules/cudatoolkit {}; - # SaxPy is only available after 11.4 because it requires redistributable versions of CUDA libraries. - saxpy = attrsets.optionalAttrs (strings.versionAtLeast cudaVersion "11.4") ( - final.callPackage ../development/cuda-modules/saxpy {} - ); - } - # NCCL is not supported on Jetson, because it does not use NVLink or PCI-e for inter-GPU communication. - # https://forums.developer.nvidia.com/t/can-jetson-orin-support-nccl/232845/9 - // attrsets.optionalAttrs (!flags.isJetsonBuild) { - nccl = final.callPackage ../development/cuda-modules/nccl {}; - nccl-tests = final.callPackage ../development/cuda-modules/nccl-tests {}; + cudatoolkit = final.callPackage ../development/cuda-modules/cudatoolkit { }; + saxpy = final.callPackage ../development/cuda-modules/saxpy { }; + nccl = final.callPackage ../development/cuda-modules/nccl { }; + nccl-tests = final.callPackage ../development/cuda-modules/nccl-tests { }; } + # Setup hooks + // builtins.import ../development/cuda-modules/setup-hooks { inherit (final) callPackage; } + # Redistributable packages + // ( + let + redistName = "cuda"; + inherit (evaluatedModules.config.${redistName}) manifests; + + # NOTE: The attribute values need to be callPackage'd before being used. + fixupFns = builtins.import ../development/cuda-modules/cuda/fixups.nix; + # NOTE: fixupFn is good to go as-is. + metaFixupFn = builtins.import ../development/cuda-modules/cuda/meta-fixup.nix { + inherit lib manifests nullableOr; + }; + + # Builder function which builds a single redist package for a given platform. + # buildRedistPackage : PackageName -> Derivation + buildRedistPackage = + pname: + genericManifestBuilderFn { + inherit (final) callPackage; + inherit manifests pname redistName; + overrideAttrsFns = [ + metaFixupFn + ] ++ lib.optionals (builtins.hasAttr pname fixupFns) [ (final.callPackage fixupFns.${pname} { }) ]; + }; + + redistPackages = trivial.pipe manifests.feature [ + # Get all the package names + builtins.attrNames + # Build the redist packages + (trivial.flip attrsets.genAttrs buildRedistPackage) + ]; + in + redistPackages + ) + # CUDNN + // (builtins.import ../development/cuda-modules/generic-builders/multiplex.nix { + inherit (final) callPackage; + inherit + cudaVersion + flags + mkVersionedPackageName + hostPlatform + lib + evaluatedModules + ; + pname = "cudnn"; + shimsFn = ../development/cuda-modules/cudnn/shims.nix; + fixupFn = ../development/cuda-modules/cudnn/fixup.nix; + }) ); mkVersionedPackageName = name: version: strings.concatStringsSep "_" [ name - (strings.replaceStrings ["."] ["_"] (versions.majorMinor version)) + (strings.replaceStrings [ "." ] [ "_" ] (versions.majorMinor version)) ]; - composedExtension = fixedPoints.composeManyExtensions [ - (import ../development/cuda-modules/setup-hooks/extension.nix) - (callPackage ../development/cuda-modules/cuda/extension.nix {inherit cudaVersion;}) - (callPackage ../development/cuda-modules/cuda/overrides.nix {inherit cudaVersion;}) - (callPackage ../development/cuda-modules/generic-builders/multiplex.nix { - inherit cudaVersion flags mkVersionedPackageName; - pname = "cudnn"; - releasesModule = ../development/cuda-modules/cudnn/releases.nix; - shimsFn = ../development/cuda-modules/cudnn/shims.nix; - fixupFn = ../development/cuda-modules/cudnn/fixup.nix; - }) - (callPackage ../development/cuda-modules/cutensor/extension.nix { - inherit cudaVersion flags mkVersionedPackageName; - }) - (callPackage ../development/cuda-modules/generic-builders/multiplex.nix { - inherit cudaVersion flags mkVersionedPackageName; - pname = "tensorrt"; - releasesModule = ../development/cuda-modules/tensorrt/releases.nix; - shimsFn = ../development/cuda-modules/tensorrt/shims.nix; - fixupFn = ../development/cuda-modules/tensorrt/fixup.nix; - }) - (callPackage ../test/cuda/cuda-samples/extension.nix {inherit cudaVersion;}) - (callPackage ../test/cuda/cuda-library-samples/extension.nix {}) - ]; + composedExtension = + fixedPoints.composeManyExtensions + [ + # (callPackage ../development/cuda-modules/cutensor/extension.nix { + # inherit cudaVersion flags mkVersionedPackageName; + # }) + # (callPackage ../development/cuda-modules/generic-builders/multiplex.nix { + # inherit cudaVersion flags mkVersionedPackageName; + # pname = "tensorrt"; + # releasesModule = ../development/cuda-modules/tensorrt/releases.nix; + # shimsFn = ../development/cuda-modules/tensorrt/shims.nix; + # fixupFn = ../development/cuda-modules/tensorrt/fixup.nix; + # }) + # (callPackage ../test/cuda/cuda-samples/extension.nix { inherit cudaVersion; }) + # (callPackage ../test/cuda/cuda-library-samples/extension.nix { }) + ]; + + # cudaPackages = customisation.makeScope newScope ( + # fixedPoints.extends composedExtension passthruFunction + # ); - cudaPackages = customisation.makeScope newScope ( - fixedPoints.extends composedExtension passthruFunction - ); + cudaPackages = pkgs.makeScopeWithSplicing' { + otherSplices = pkgs.generateSplicesForMkScope "cudaPackages"; + f = fixedPoints.extends composedExtension passthruFunction; + # f = final: { + # # Recursive reference to the initial cudaPackages' cudaVersion. + # inherit (config.cudaPackages) cudaVersion; + # gpus = builtins.import ../gpus.nix; + # nvccCompatibilities = builtins.import ../nvcc-compatibilities.nix; + # cudaMajorVersion = versions.major final.cudaVersion; + # cudaMajorMinorVersion = versions.majorMinor final.cudaVersion; + # # flags = final.callPackage ../flags.nix { }; + # # Exposed as cudaPackages.backendStdenv. + # # This is what nvcc uses as a backend, + # # and it has to be an officially supported one (e.g. gcc11 for cuda11). + # # + # # It, however, propagates current stdenv's libstdc++ to avoid "GLIBCXX_* not found errors" + # # when linked with other C++ libraries. + # # E.g. for cudaPackages_11_8 we use gcc11 with gcc12's libstdc++ + # # Cf. https://github.com/NixOS/nixpkgs/pull/218265 for context + # backendStdenv = final.callPackage ../backend-stdenv.nix { }; + # }; + }; in cudaPackages // { inherit __attrsFailEvaluation; } From 1a71265b7a788247f66b3647fd66a98ea633f749 Mon Sep 17 00:00:00 2001 From: Connor Baker Date: Tue, 19 Dec 2023 08:46:32 +0000 Subject: [PATCH 2/5] cuda-modules: wip refactor for cross-compilation --- .../cuda-modules/backend-stdenv.nix | 4 +- .../cuda-modules/cutensor/extension.nix | 164 ------- pkgs/development/cuda-modules/flags.nix | 37 +- .../generic-builders/multiplex.nix | 41 +- pkgs/development/cuda-modules/gpus.nix | 204 -------- .../cuda-modules/modules/cuda/default.nix | 112 +++-- .../meta-fixup.nix => modules/cuda/fixup.nix} | 13 +- .../{ => modules}/cuda/fixups.nix | 0 .../cuda-modules/modules/cudnn/default.nix | 21 +- .../{ => modules}/cudnn/fixup.nix | 0 .../{ => modules}/cudnn/shims.nix | 0 .../cuda-modules/modules/cutensor/default.nix | 58 ++- .../cuda-modules/modules/cutensor/fixup.nix | 30 ++ .../cutensor/manifests/feature_1.3.3.json | 0 .../cutensor/manifests/feature_1.4.0.json | 0 .../cutensor/manifests/feature_1.5.0.json | 0 .../cutensor/manifests/feature_1.6.2.json | 0 .../cutensor/manifests/feature_1.7.0.json | 0 .../cutensor/manifests/redistrib_1.3.3.json | 0 .../cutensor/manifests/redistrib_1.4.0.json | 0 .../cutensor/manifests/redistrib_1.5.0.json | 0 .../cutensor/manifests/redistrib_1.6.2.json | 0 .../cutensor/manifests/redistrib_1.7.0.json | 0 .../cuda-modules/modules/default.nix | 29 +- .../modules/generic/manifests/default.nix | 104 ++++- .../modules/generic/types/default.nix | 34 +- .../development/cuda-modules/modules/gpus.nix | 236 ++++++++++ .../modules/nvcc-compatibilities.nix | 149 ++++++ .../cuda-modules/modules/tensorrt/default.nix | 22 +- .../{ => modules}/tensorrt/fixup.nix | 34 +- .../modules/tensorrt/releases.nix | 128 +++++ .../cuda-modules/modules/tensorrt/shims.nix | 24 + .../cuda-modules/modules/versions.nix | 31 ++ .../cuda-modules/nvcc-compatibilities.nix | 124 ----- .../cuda-modules/setup-hooks/default.nix | 88 ++-- .../cuda-modules/tensorrt/releases.nix | 130 ------ .../cuda-modules/tensorrt/shims.nix | 16 - pkgs/top-level/all-packages.nix | 42 +- pkgs/top-level/cuda-packages.nix | 440 +++++++++++------- 39 files changed, 1281 insertions(+), 1034 deletions(-) delete mode 100644 pkgs/development/cuda-modules/cutensor/extension.nix delete mode 100644 pkgs/development/cuda-modules/gpus.nix rename pkgs/development/cuda-modules/{cuda/meta-fixup.nix => modules/cuda/fixup.nix} (55%) rename pkgs/development/cuda-modules/{ => modules}/cuda/fixups.nix (100%) rename pkgs/development/cuda-modules/{ => modules}/cudnn/fixup.nix (100%) rename pkgs/development/cuda-modules/{ => modules}/cudnn/shims.nix (100%) create mode 100644 pkgs/development/cuda-modules/modules/cutensor/fixup.nix rename pkgs/development/cuda-modules/{ => modules}/cutensor/manifests/feature_1.3.3.json (100%) rename pkgs/development/cuda-modules/{ => modules}/cutensor/manifests/feature_1.4.0.json (100%) rename pkgs/development/cuda-modules/{ => modules}/cutensor/manifests/feature_1.5.0.json (100%) rename pkgs/development/cuda-modules/{ => modules}/cutensor/manifests/feature_1.6.2.json (100%) rename pkgs/development/cuda-modules/{ => modules}/cutensor/manifests/feature_1.7.0.json (100%) rename pkgs/development/cuda-modules/{ => modules}/cutensor/manifests/redistrib_1.3.3.json (100%) rename pkgs/development/cuda-modules/{ => modules}/cutensor/manifests/redistrib_1.4.0.json (100%) rename pkgs/development/cuda-modules/{ => modules}/cutensor/manifests/redistrib_1.5.0.json (100%) rename pkgs/development/cuda-modules/{ => modules}/cutensor/manifests/redistrib_1.6.2.json (100%) rename pkgs/development/cuda-modules/{ => modules}/cutensor/manifests/redistrib_1.7.0.json (100%) create mode 100644 pkgs/development/cuda-modules/modules/gpus.nix create mode 100644 pkgs/development/cuda-modules/modules/nvcc-compatibilities.nix rename pkgs/development/cuda-modules/{ => modules}/tensorrt/fixup.nix (82%) create mode 100644 pkgs/development/cuda-modules/modules/tensorrt/releases.nix create mode 100644 pkgs/development/cuda-modules/modules/tensorrt/shims.nix create mode 100644 pkgs/development/cuda-modules/modules/versions.nix delete mode 100644 pkgs/development/cuda-modules/nvcc-compatibilities.nix delete mode 100644 pkgs/development/cuda-modules/tensorrt/releases.nix delete mode 100644 pkgs/development/cuda-modules/tensorrt/shims.nix diff --git a/pkgs/development/cuda-modules/backend-stdenv.nix b/pkgs/development/cuda-modules/backend-stdenv.nix index 10fedd1e6f271..2ad3b92286c83 100644 --- a/pkgs/development/cuda-modules/backend-stdenv.nix +++ b/pkgs/development/cuda-modules/backend-stdenv.nix @@ -1,6 +1,6 @@ { + config, lib, - nvccCompatibilities, cudaVersion, buildPackages, overrideCC, @@ -8,7 +8,7 @@ wrapCCWith, }: let - gccMajorVersion = nvccCompatibilities.${cudaVersion}.gccMaxMajorVersion; + gccMajorVersion = config.nvccCompatibilities.${cudaVersion}.gccMaxMajorVersion; # We use buildPackages (= pkgsBuildHost) because we look for a gcc that # runs on our build platform, and that produces executables for the host # platform (= platform on which we deploy and run the downstream packages). diff --git a/pkgs/development/cuda-modules/cutensor/extension.nix b/pkgs/development/cuda-modules/cutensor/extension.nix deleted file mode 100644 index b762fd22ede88..0000000000000 --- a/pkgs/development/cuda-modules/cutensor/extension.nix +++ /dev/null @@ -1,164 +0,0 @@ -# Support matrix can be found at -# https://docs.nvidia.com/deeplearning/cudnn/archives/cudnn-880/support-matrix/index.html -# -# TODO(@connorbaker): -# This is a very similar strategy to CUDA/CUDNN: -# -# - Get all versions supported by the current release of CUDA -# - Build all of them -# - Make the newest the default -# -# Unique twists: -# -# - Instead of providing different releases for each version of CUDA, CuTensor has multiple subdirectories in `lib` -# -- one for each version of CUDA. -{ - cudaVersion, - flags, - hostPlatform, - lib, - mkVersionedPackageName, -}: -let - inherit (lib) - attrsets - lists - modules - versions - strings - trivial - ; - - redistName = "cutensor"; - pname = "libcutensor"; - - cutensorVersions = [ - "1.3.3" - "1.4.0" - "1.5.0" - "1.6.2" - "1.7.0" - ]; - - # Manifests :: { redistrib, feature } - - # Each release of cutensor gets mapped to an evaluated module for that release. - # From there, we can get the min/max CUDA versions supported by that release. - # listOfManifests :: List Manifests - listOfManifests = - let - configEvaluator = - fullCutensorVersion: - modules.evalModules { - modules = [ - ../modules - # We need to nest the manifests in a config.cutensor.manifests attribute so the - # module system can evaluate them. - { - cutensor.manifests = { - redistrib = trivial.importJSON (./manifests + "/redistrib_${fullCutensorVersion}.json"); - feature = trivial.importJSON (./manifests + "/feature_${fullCutensorVersion}.json"); - }; - } - ]; - }; - # Un-nest the manifests attribute set. - releaseGrabber = evaluatedModules: evaluatedModules.config.cutensor.manifests; - in - lists.map - (trivial.flip trivial.pipe [ - configEvaluator - releaseGrabber - ]) - cutensorVersions; - - # Our cudaVersion tells us which version of CUDA we're building against. - # The subdirectories in lib/ tell us which versions of CUDA are supported. - # Typically the names will look like this: - # - # - 10.2 - # - 11 - # - 11.0 - # - 12 - - # libPath :: String - libPath = - let - cudaMajorMinor = versions.majorMinor cudaVersion; - cudaMajor = versions.major cudaVersion; - in - if cudaMajorMinor == "10.2" then cudaMajorMinor else cudaMajor; - - # A release is supported if it has a libPath that matches our CUDA version for our platform. - # LibPath are not constant across the same release -- one platform may support fewer - # CUDA versions than another. - redistArch = flags.getRedistArch hostPlatform.system; - # platformIsSupported :: Manifests -> Boolean - platformIsSupported = - {feature, ...}: - (attrsets.attrByPath - [ - pname - redistArch - ] - null - feature - ) != null; - - # TODO(@connorbaker): With an auxilliary file keeping track of the CUDA versions each release supports, - # we could filter out releases that don't support our CUDA version. - # However, we don't have that currently, so we make a best-effort to try to build TensorRT with whatever - # libPath corresponds to our CUDA version. - # supportedManifests :: List Manifests - supportedManifests = builtins.filter platformIsSupported listOfManifests; - - # Compute versioned attribute name to be used in this package set - # Patch version changes should not break the build, so we only use major and minor - # computeName :: RedistribRelease -> String - computeName = {version, ...}: mkVersionedPackageName redistName version; -in -final: _: -let - # buildCutensorPackage :: Manifests -> AttrSet Derivation - buildCutensorPackage = - {redistrib, feature}: - let - drv = final.callPackage ../generic-builders/manifest.nix { - inherit pname redistName libPath; - redistribRelease = redistrib.${pname}; - featureRelease = feature.${pname}; - }; - fixedDrv = drv.overrideAttrs ( - prevAttrs: { - buildInputs = - prevAttrs.buildInputs - ++ lists.optionals (strings.versionOlder cudaVersion "11.4") [final.cudatoolkit] - ++ lists.optionals (strings.versionAtLeast cudaVersion "11.4") ( - [final.libcublas.lib] - # For some reason, the 1.4.x release of cuTENSOR requires the cudart library. - ++ lists.optionals (strings.hasPrefix "1.4" redistrib.${pname}.version) [final.cuda_cudart.lib] - ); - meta = prevAttrs.meta // { - description = "cuTENSOR: A High-Performance CUDA Library For Tensor Primitives"; - homepage = "https://developer.nvidia.com/cutensor"; - maintainers = prevAttrs.meta.maintainers ++ [lib.maintainers.obsidian-systems-maintenance]; - license = lib.licenses.unfreeRedistributable // { - shortName = "cuTENSOR EULA"; - name = "cuTENSOR SUPPLEMENT TO SOFTWARE LICENSE AGREEMENT FOR NVIDIA SOFTWARE DEVELOPMENT KITS"; - url = "https://docs.nvidia.com/cuda/cutensor/license.html"; - }; - }; - } - ); - in - attrsets.nameValuePair (computeName redistrib.${pname}) fixedDrv; - - extension = - let - nameOfNewest = computeName (lists.last supportedManifests).redistrib.${pname}; - drvs = builtins.listToAttrs (lists.map buildCutensorPackage supportedManifests); - containsDefault = attrsets.optionalAttrs (drvs != {}) {cutensor = drvs.${nameOfNewest};}; - in - drvs // containsDefault; -in -extension diff --git a/pkgs/development/cuda-modules/flags.nix b/pkgs/development/cuda-modules/flags.nix index 139952bc9dfd9..3e90c8ac2b86c 100644 --- a/pkgs/development/cuda-modules/flags.nix +++ b/pkgs/development/cuda-modules/flags.nix @@ -1,15 +1,9 @@ -# Type aliases -# Gpu :: AttrSet -# - See the documentation in ./gpus.nix. { - config, - cudaCapabilities ? (config.cudaCapabilities or []), - cudaForwardCompat ? (config.cudaForwardCompat or true), + pkgs, lib, + config, cudaVersion, hostPlatform, - # gpus :: List Gpu - gpus, }: let inherit (lib) @@ -20,23 +14,6 @@ let trivial ; - # Flags are determined based on your CUDA toolkit by default. You may benefit - # from improved performance, reduced file size, or greater hardware support by - # passing a configuration based on your specific GPU environment. - # - # cudaCapabilities :: List Capability - # List of hardware generations to build. - # E.g. [ "8.0" ] - # Currently, the last item is considered the optional forward-compatibility arch, - # but this may change in the future. - # - # cudaForwardCompat :: Bool - # Whether to include the forward compatibility gencode (+PTX) - # to support future GPU generations. - # E.g. true - # - # Please see the accompanying documentation or https://github.com/NixOS/nixpkgs/pull/205351 - # isSupported :: Gpu -> Bool isSupported = gpu: @@ -61,7 +38,7 @@ let # supportedGpus :: List Gpu # GPUs which are supported by the provided CUDA version. - supportedGpus = builtins.filter isSupported gpus; + supportedGpus = builtins.filter isSupported config.gpus; # defaultGpus :: List Gpu # GPUs which are supported by the provided CUDA version and we want to build for by default. @@ -105,7 +82,7 @@ let # they must be in the user-specified cudaCapabilities. # NOTE: We don't need to worry about mixes of Jetson and non-Jetson devices here -- there's # sanity-checking for all that in below. - jetsonTargets = lists.intersectLists jetsonComputeCapabilities cudaCapabilities; + jetsonTargets = lists.intersectLists jetsonComputeCapabilities config.cudaCapabilities; # dropDot :: String -> String dropDot = ver: builtins.replaceStrings ["."] [""] ver; @@ -236,7 +213,7 @@ let Exactly one of the following must be true: - All CUDA capabilities belong to Jetson devices and hostPlatform is aarch64. - No CUDA capabilities belong to Jetson devices. - See ${./gpus.nix} for a list of architectures supported by this version of Nixpkgs. + See cudaPackages.config.gpus for a list of architectures supported by this version of Nixpkgs. '' jetsonBuildSufficientCondition && jetsonBuildNecessaryCondition; @@ -385,6 +362,6 @@ asserts.assertMsg ; } // formatCapabilities { - cudaCapabilities = if cudaCapabilities == [] then defaultCapabilities else cudaCapabilities; - enableForwardCompat = cudaForwardCompat; + cudaCapabilities = if config.cudaCapabilities == [] then defaultCapabilities else config.cudaCapabilities; + enableForwardCompat = config.cudaForwardCompat; } diff --git a/pkgs/development/cuda-modules/generic-builders/multiplex.nix b/pkgs/development/cuda-modules/generic-builders/multiplex.nix index c33da70329361..d931a3cfaa3a8 100644 --- a/pkgs/development/cuda-modules/generic-builders/multiplex.nix +++ b/pkgs/development/cuda-modules/generic-builders/multiplex.nix @@ -9,35 +9,7 @@ mkVersionedPackageName, # pname :: String pname, - evaluatedModules, - # shims :: Path - # A path to a module which provides a `shims` attribute - # The redistribRelease is only used in ./manifest.nix for the package version - # and the package description (which NVIDIA's manifest calls the "name"). - # It's also used for fetching the source, but we override that since we can't - # re-use that portion of the functionality (different URLs, etc.). - # The featureRelease is used to populate meta.platforms (by way of looking at the attribute names) - # and to determine the outputs of the package. - # shimFn :: {package, redistArch} -> AttrSet - shimsFn ? ({package, redistArch}: throw "shimsFn must be provided"), - # fixupFn :: Path - # A path (or nix expression) to be evaluated with callPackage and then - # provided to the package's overrideAttrs function. - # It must accept at least the following arguments: - # - final - # - cudaVersion - # - mkVersionedPackageName - # - package - fixupFn ? ( - { - final, - cudaVersion, - mkVersionedPackageName, - package, - ... - }: - throw "fixupFn must be provided" - ), + config, }: let inherit (lib) @@ -46,15 +18,12 @@ let modules strings ; + cfg = config.${pname}; # NOTE: Important types: # - Releases: ../modules/${pname}/releases/releases.nix # - Package: ../modules/${pname}/releases/package.nix - # All releases across all platforms - # See ../modules/${pname}/releases/releases.nix - allReleases = evaluatedModules.config.${pname}.releases; - # Compute versioned attribute name to be used in this package set # Patch version changes should not break the build, so we only use major and minor # computeName :: Package -> String @@ -73,7 +42,7 @@ let # All the supported packages we can build for our platform. # supportedPackages :: List (AttrSet Packages) supportedPackages = - if redistArch == null then [ ] else builtins.filter isSupported (allReleases.${redistArch} or [ ]); + if redistArch == null then [ ] else builtins.filter isSupported (cfg.releases.${redistArch} or [ ]); # newestToOldestSupportedPackage :: List (AttrSet Packages) newestToOldestSupportedPackage = lists.reverseList supportedPackages; @@ -83,14 +52,14 @@ let # A function which takes the `final` overlay and the `package` being built and returns # a function to be consumed via `overrideAttrs`. overrideAttrsFixupFn = - package: callPackage fixupFn { inherit cudaVersion mkVersionedPackageName package; }; + package: callPackage cfg.fixupFnPath { inherit cudaVersion mkVersionedPackageName package; }; # Builds our package into derivation and wraps it in a nameValuePair, where the name is the versioned name # of the package. buildPackage = package: let - manifests = callPackage shimsFn { inherit package redistArch; }; + manifests = callPackage cfg.shimsFnPath { inherit package redistArch; }; name = computeName package; drv = callPackage ./manifest.nix { inherit pname manifests; diff --git a/pkgs/development/cuda-modules/gpus.nix b/pkgs/development/cuda-modules/gpus.nix deleted file mode 100644 index cf6e0a1eaf360..0000000000000 --- a/pkgs/development/cuda-modules/gpus.nix +++ /dev/null @@ -1,204 +0,0 @@ -# Type aliases -# -# Gpu = { -# archName: String -# - The name of the microarchitecture. -# computeCapability: String -# - The compute capability of the GPU. -# isJetson: Boolean -# - Whether a GPU is part of NVIDIA's line of Jetson embedded computers. This field is -# notable because it tells us what architecture to build for (as Jetson devices are -# aarch64). -# More on Jetson devices here: -# https://www.nvidia.com/en-us/autonomous-machines/embedded-systems/ -# NOTE: These architectures are only built upon request. -# minCudaVersion: String -# - The minimum (inclusive) CUDA version that supports this GPU. -# dontDefaultAfter: null | String -# - The CUDA version after which to exclude this GPU from the list of default capabilities -# we build. null means we always include this GPU in the default capabilities if it is -# supported. -# maxCudaVersion: null | String -# - The maximum (exclusive) CUDA version that supports this GPU. null means there is no -# maximum. -# } -# -# Many thanks to Arnon Shimoni for maintaining a list of these architectures and capabilities. -# Without your work, this would have been much more difficult. -# https://arnon.dk/matching-sm-architectures-arch-and-gencode-for-various-nvidia-cards/ -[ - { - # GeForce 700, GT-730 - archName = "Kepler"; - computeCapability = "3.0"; - isJetson = false; - minCudaVersion = "10.0"; - dontDefaultAfter = "10.2"; - maxCudaVersion = "10.2"; - } - { - archName = "Kepler"; - computeCapability = "3.2"; - isJetson = false; - minCudaVersion = "10.0"; - dontDefaultAfter = "10.2"; - maxCudaVersion = "10.2"; - } - { - # Tesla K40 - archName = "Kepler"; - computeCapability = "3.5"; - isJetson = false; - minCudaVersion = "10.0"; - dontDefaultAfter = "11.0"; - maxCudaVersion = "11.8"; - } - { - # Tesla K80 - archName = "Kepler"; - computeCapability = "3.7"; - isJetson = false; - minCudaVersion = "10.0"; - dontDefaultAfter = "11.0"; - maxCudaVersion = "11.8"; - } - { - # Tesla/Quadro M series - archName = "Maxwell"; - computeCapability = "5.0"; - isJetson = false; - minCudaVersion = "10.0"; - dontDefaultAfter = "11.0"; - maxCudaVersion = null; - } - { - # Quadro M6000 , GeForce 900, GTX-970, GTX-980, GTX Titan X - archName = "Maxwell"; - computeCapability = "5.2"; - isJetson = false; - minCudaVersion = "10.0"; - dontDefaultAfter = "11.0"; - maxCudaVersion = null; - } - { - # Tegra (Jetson) TX1 / Tegra X1, Drive CX, Drive PX, Jetson Nano - archName = "Maxwell"; - computeCapability = "5.3"; - isJetson = true; - minCudaVersion = "10.0"; - dontDefaultAfter = null; - maxCudaVersion = null; - } - { - # Quadro GP100, Tesla P100, DGX-1 (Generic Pascal) - archName = "Pascal"; - computeCapability = "6.0"; - isJetson = false; - minCudaVersion = "10.0"; - dontDefaultAfter = null; - maxCudaVersion = null; - } - { - # GTX 1080, GTX 1070, GTX 1060, GTX 1050, GTX 1030 (GP108), GT 1010 (GP108) Titan Xp, Tesla - # P40, Tesla P4, Discrete GPU on the NVIDIA Drive PX2 - archName = "Pascal"; - computeCapability = "6.1"; - isJetson = false; - minCudaVersion = "10.0"; - dontDefaultAfter = null; - maxCudaVersion = null; - } - { - # Integrated GPU on the NVIDIA Drive PX2, Tegra (Jetson) TX2 - archName = "Pascal"; - computeCapability = "6.2"; - isJetson = true; - minCudaVersion = "10.0"; - dontDefaultAfter = null; - maxCudaVersion = null; - } - { - # DGX-1 with Volta, Tesla V100, GTX 1180 (GV104), Titan V, Quadro GV100 - archName = "Volta"; - computeCapability = "7.0"; - isJetson = false; - minCudaVersion = "10.0"; - dontDefaultAfter = null; - maxCudaVersion = null; - } - { - # Jetson AGX Xavier, Drive AGX Pegasus, Xavier NX - archName = "Volta"; - computeCapability = "7.2"; - isJetson = true; - minCudaVersion = "10.0"; - dontDefaultAfter = null; - maxCudaVersion = null; - } - { - # GTX/RTX Turing – GTX 1660 Ti, RTX 2060, RTX 2070, RTX 2080, Titan RTX, Quadro RTX 4000, - # Quadro RTX 5000, Quadro RTX 6000, Quadro RTX 8000, Quadro T1000/T2000, Tesla T4 - archName = "Turing"; - computeCapability = "7.5"; - isJetson = false; - minCudaVersion = "10.0"; - dontDefaultAfter = null; - maxCudaVersion = null; - } - { - # NVIDIA A100 (the name “Tesla” has been dropped – GA100), NVIDIA DGX-A100 - archName = "Ampere"; - computeCapability = "8.0"; - isJetson = false; - minCudaVersion = "11.2"; - dontDefaultAfter = null; - maxCudaVersion = null; - } - { - # Tesla GA10x cards, RTX Ampere – RTX 3080, GA102 – RTX 3090, RTX A2000, A3000, RTX A4000, - # A5000, A6000, NVIDIA A40, GA106 – RTX 3060, GA104 – RTX 3070, GA107 – RTX 3050, RTX A10, RTX - # A16, RTX A40, A2 Tensor Core GPU - archName = "Ampere"; - computeCapability = "8.6"; - isJetson = false; - minCudaVersion = "11.2"; - dontDefaultAfter = null; - maxCudaVersion = null; - } - { - # Jetson AGX Orin and Drive AGX Orin only - archName = "Ampere"; - computeCapability = "8.7"; - isJetson = true; - minCudaVersion = "11.5"; - dontDefaultAfter = null; - maxCudaVersion = null; - } - { - # NVIDIA GeForce RTX 4090, RTX 4080, RTX 6000, Tesla L40 - archName = "Ada"; - computeCapability = "8.9"; - isJetson = false; - minCudaVersion = "11.8"; - dontDefaultAfter = null; - maxCudaVersion = null; - } - { - # NVIDIA H100 (GH100) - archName = "Hopper"; - computeCapability = "9.0"; - isJetson = false; - minCudaVersion = "11.8"; - dontDefaultAfter = null; - maxCudaVersion = null; - } - { - # NVIDIA H100 (GH100) (Thor) - archName = "Hopper"; - computeCapability = "9.0a"; - isJetson = false; - minCudaVersion = "12.0"; - dontDefaultAfter = null; - maxCudaVersion = null; - } -] diff --git a/pkgs/development/cuda-modules/modules/cuda/default.nix b/pkgs/development/cuda-modules/modules/cuda/default.nix index 6fd52c8f23523..fdaee6ffa3e96 100644 --- a/pkgs/development/cuda-modules/modules/cuda/default.nix +++ b/pkgs/development/cuda-modules/modules/cuda/default.nix @@ -1,48 +1,72 @@ -{ - config, - lib, - options, - ... -}: +{ config, lib, ... }: let - inherit (lib) filesystem trivial versions; - - # Read all the files in the manifests directory - # manifestFiles :: [ Path ] - cudaVersionMajor = versions.major config.cudaVersion; - cudaVersionMinor = versions.minor config.cudaVersion; - filenameRegexFn = builtins.match ".+(feature|redistrib)_${cudaVersionMajor}\\.${cudaVersionMinor}\\.[[:digit:]]+\\.json$"; - manifests = - builtins.foldl' - ( - manifestAttrs: filename: - let - matches = filenameRegexFn (builtins.toString filename); - in - if matches == null then - # No matches! - manifestAttrs - else - let - fileKind = - trivial.throwIfNot (builtins.length matches == 1) - "Unexpected error while processing manifest files: multiple regex matches in filename ${builtins.toString filename}" - (builtins.head matches); - newAttr = - trivial.throwIf (builtins.hasAttr fileKind manifestAttrs) - "Unexpected error while processing manifest files: duplicate file kind ${builtins.toString fileKind}" - { "${fileKind}" = trivial.importJSON filename; }; - in - manifestAttrs // newAttr - ) - { } - (filesystem.listFilesRecursive ./manifests); + inherit (lib) + filesystem + options + types + ; + cfg = config.generic.manifests; in { - options.cuda.manifests = options.generic.manifests; - config.cuda.manifests = - # It's okay if neither feature nor redistrib manifests are found, but if one is found, the other must be found as well - trivial.throwIf (builtins.hasAttr "feature" manifests != builtins.hasAttr "redistrib" manifests) - "Both feature and redistrib manifests must be present, or neither" - manifests; + options.cuda = { + # TODO(@connorbaker): Figure out how to move these general options into `generic.manifests.options` + # and re-use them here, ideally overriding only the default values to get the effects we want. + manifestPathParser = options.mkOption { + description = "Function to parse a manifest path into a manifest"; + default = cfg.utils.manifestPathToManifestMeta; + type = types.functionTo cfg.types.manifestMeta; + }; + manifestPaths = options.mkOption { + description = "List of paths to CUDA redistributable manifests"; + default = filesystem.listFilesRecursive ./manifests; + type = types.listOf types.path; + }; + manifestMetas = options.mkOption { + description = "List of meta information about CUDA redistributable manifests"; + default = builtins.map cfg.utils.manifestPathToManifestMeta config.cuda.manifestPaths; + type = types.listOf cfg.types.manifestMeta; + }; + manifestMetasToManifests = options.mkOption { + description = "Function to convert a list of manifest metas to a list of manifests"; + default = cfg.utils.manifestMetasToManifests; + type = types.functionTo cfg.types.manifests; + }; + manifests = options.mkOption { + description = "Mapping of manifest version (major and minor) to feature and redistributable manifests"; + default = cfg.utils.manifestMetasToManifests config.cuda.manifestMetas; + type = cfg.types.manifests; + }; + fixupFns = options.mkOption { + description = '' + Functions to pass to a derivation's overrideAttrs function to perform fixup after being callPackage'd + ''; + default = { }; + type = types.submodule { + options = { + generalFixupFn = options.mkOption { + description = '' + A general fixup applied to all redistributables's. Note that it requires `callPackage` to provide a + `manifests` argument. + NOTE: The value must be inspectable by `callPackage`. It seems that when functions are exposed via module + configurations, they do not preserve functionArgs, and so callPackage will fail because it cannot supply + arguments by default. + ''; + default = ./fixup.nix; + type = types.path; + }; + indexedFixupFn = options.mkOption { + description = '' + Attribute set of functions where each key is the `pname` of a redistributable and each value is a + function to fixup the derivation's attributes after being callPackage'd + NOTE: The value must be inspectable by `callPackage`. It seems that when functions are exposed via module + configurations, they do not preserve functionArgs, and so callPackage will fail because it cannot supply + arguments by default. + ''; + default = ./fixups.nix; + type = types.path; + }; + }; + }; + }; + }; } diff --git a/pkgs/development/cuda-modules/cuda/meta-fixup.nix b/pkgs/development/cuda-modules/modules/cuda/fixup.nix similarity index 55% rename from pkgs/development/cuda-modules/cuda/meta-fixup.nix rename to pkgs/development/cuda-modules/modules/cuda/fixup.nix index 7cbe983b3bc45..004433b89cfc3 100644 --- a/pkgs/development/cuda-modules/cuda/meta-fixup.nix +++ b/pkgs/development/cuda-modules/modules/cuda/fixup.nix @@ -1,10 +1,5 @@ # A function which when callPackage'd returns a function to be given to overrideAttrs. -{ - lib, - manifests, - nullableOr, - ... -}: +{ lib, manifests, ... }: prevAttrs: prevAttrs // { @@ -12,8 +7,12 @@ prevAttrs meta = prevAttrs.meta // { license = lib.licenses.nvidiaCudaRedist // { url = + let + default = "${prevAttrs.pname}/LICENSE.txt"; + nullableDesired = manifests.redistrib.${prevAttrs.pname}.license_path; + in "https://developer.download.nvidia.com/compute/cuda/redist/" - + nullableOr "${prevAttrs.pname}/LICENSE.txt" manifests.redistrib.${prevAttrs.pname}.license_path; + + (if nullableDesired != null then nullableDesired else default); }; }; } diff --git a/pkgs/development/cuda-modules/cuda/fixups.nix b/pkgs/development/cuda-modules/modules/cuda/fixups.nix similarity index 100% rename from pkgs/development/cuda-modules/cuda/fixups.nix rename to pkgs/development/cuda-modules/modules/cuda/fixups.nix diff --git a/pkgs/development/cuda-modules/modules/cudnn/default.nix b/pkgs/development/cuda-modules/modules/cudnn/default.nix index e04fda735b9bb..8c5c56c802084 100644 --- a/pkgs/development/cuda-modules/modules/cudnn/default.nix +++ b/pkgs/development/cuda-modules/modules/cudnn/default.nix @@ -1,7 +1,24 @@ -{options, ...}: +{lib, options, ...}: +let + inherit (lib) types; + inherit (lib.options) mkOption; +in { - options.cudnn.releases = options.generic.releases; + options.cudnn = { + inherit (options.generic) releases; + shimsFnPath = mkOption { + description = "The path to a function to transform releases into manifest files"; + default = ./shims.nix; + type = types.path; + }; + fixupFnPath = mkOption { + description = "The path to a function to transform releases into manifest files"; + default = ./fixup.nix; + type = types.path; + }; + }; config.cudnn.releases = builtins.import ./releases.nix; + # TODO(@connorbaker): Figure out how to add additional options to the # to the generic release. # { diff --git a/pkgs/development/cuda-modules/cudnn/fixup.nix b/pkgs/development/cuda-modules/modules/cudnn/fixup.nix similarity index 100% rename from pkgs/development/cuda-modules/cudnn/fixup.nix rename to pkgs/development/cuda-modules/modules/cudnn/fixup.nix diff --git a/pkgs/development/cuda-modules/cudnn/shims.nix b/pkgs/development/cuda-modules/modules/cudnn/shims.nix similarity index 100% rename from pkgs/development/cuda-modules/cudnn/shims.nix rename to pkgs/development/cuda-modules/modules/cudnn/shims.nix diff --git a/pkgs/development/cuda-modules/modules/cutensor/default.nix b/pkgs/development/cuda-modules/modules/cutensor/default.nix index 8ec2189fee4cc..5b997472323be 100644 --- a/pkgs/development/cuda-modules/modules/cutensor/default.nix +++ b/pkgs/development/cuda-modules/modules/cutensor/default.nix @@ -1 +1,57 @@ -{options, ...}: {options.cutensor.manifests = options.generic.manifests;} +{ config, lib, ... }: +let + inherit (lib) filesystem options types; + cfg = config.generic.manifests; +in +{ + options.cutensor = { + # TODO(@connorbaker): Figure out how to move these general options into `generic.manifests.options` + # and re-use them here, ideally overriding only the default values to get the effects we want. + manifestPathParser = options.mkOption { + description = "Function to parse a manifest path into a manifest"; + default = cfg.utils.manifestPathToManifestMeta; + type = types.functionTo cfg.types.manifestMeta; + }; + manifestPaths = options.mkOption { + description = "List of paths to CUDA redistributable manifests"; + default = filesystem.listFilesRecursive ./manifests; + type = types.listOf types.path; + }; + manifestMetas = options.mkOption { + description = "List of meta information about CUDA redistributable manifests"; + default = builtins.map cfg.utils.manifestPathToManifestMeta config.cutensor.manifestPaths; + type = types.listOf cfg.types.manifestMeta; + }; + manifestMetasToManifests = options.mkOption { + description = "Function to convert a list of manifest metas to a list of manifests"; + default = cfg.utils.manifestMetasToManifests; + type = types.functionTo cfg.types.manifests; + }; + manifests = options.mkOption { + description = "Mapping of manifest version (major and minor) to feature and redistributable manifests"; + default = cfg.utils.manifestMetasToManifests config.cutensor.manifestMetas; + type = cfg.types.manifests; + }; + fixupFns = options.mkOption { + description = '' + Functions to pass to a derivation's overrideAttrs function to perform fixup after being callPackage'd + ''; + default = { }; + type = types.submodule { + options = { + generalFixupFn = options.mkOption { + description = '' + A general fixup applied to all redistributables's. + + NOTE: The value must be inspectable by `callPackage`. It seems that when functions are exposed via module + configurations, they do not preserve functionArgs, and so callPackage will fail because it cannot supply + arguments by default. + ''; + default = ./fixup.nix; + type = types.path; + }; + }; + }; + }; + }; +} diff --git a/pkgs/development/cuda-modules/modules/cutensor/fixup.nix b/pkgs/development/cuda-modules/modules/cutensor/fixup.nix new file mode 100644 index 0000000000000..ae57a016e520c --- /dev/null +++ b/pkgs/development/cuda-modules/modules/cutensor/fixup.nix @@ -0,0 +1,30 @@ +# A function which when callPackage'd returns a function to be given to overrideAttrs. +{ + cudaVersionOlder, + cudaVersionAtLeast, + cudaPackages, + lib, +}: +let + inherit (lib) lists strings; +in +prevAttrs: { + buildInputs = + prevAttrs.buildInputs + ++ lists.optionals (cudaVersionOlder "11.4") [ cudaPackages.cudatoolkit ] + ++ lists.optionals (cudaVersionAtLeast "11.4") ( + [ cudaPackages.libcublas.lib ] + # For some reason, the 1.4.x release of cuTENSOR requires the cudart library. + ++ lists.optionals (strings.hasPrefix "1.4" prevAttrs.version) [ cudaPackages.cuda_cudart.lib ] + ); + meta = prevAttrs.meta // { + description = "cuTENSOR: A High-Performance CUDA Library For Tensor Primitives"; + homepage = "https://developer.nvidia.com/cutensor"; + maintainers = prevAttrs.meta.maintainers ++ [ lib.maintainers.obsidian-systems-maintenance ]; + license = lib.licenses.unfreeRedistributable // { + shortName = "cuTENSOR EULA"; + name = "cuTENSOR SUPPLEMENT TO SOFTWARE LICENSE AGREEMENT FOR NVIDIA SOFTWARE DEVELOPMENT KITS"; + url = "https://docs.nvidia.com/cuda/cutensor/license.html"; + }; + }; +} diff --git a/pkgs/development/cuda-modules/cutensor/manifests/feature_1.3.3.json b/pkgs/development/cuda-modules/modules/cutensor/manifests/feature_1.3.3.json similarity index 100% rename from pkgs/development/cuda-modules/cutensor/manifests/feature_1.3.3.json rename to pkgs/development/cuda-modules/modules/cutensor/manifests/feature_1.3.3.json diff --git a/pkgs/development/cuda-modules/cutensor/manifests/feature_1.4.0.json b/pkgs/development/cuda-modules/modules/cutensor/manifests/feature_1.4.0.json similarity index 100% rename from pkgs/development/cuda-modules/cutensor/manifests/feature_1.4.0.json rename to pkgs/development/cuda-modules/modules/cutensor/manifests/feature_1.4.0.json diff --git a/pkgs/development/cuda-modules/cutensor/manifests/feature_1.5.0.json b/pkgs/development/cuda-modules/modules/cutensor/manifests/feature_1.5.0.json similarity index 100% rename from pkgs/development/cuda-modules/cutensor/manifests/feature_1.5.0.json rename to pkgs/development/cuda-modules/modules/cutensor/manifests/feature_1.5.0.json diff --git a/pkgs/development/cuda-modules/cutensor/manifests/feature_1.6.2.json b/pkgs/development/cuda-modules/modules/cutensor/manifests/feature_1.6.2.json similarity index 100% rename from pkgs/development/cuda-modules/cutensor/manifests/feature_1.6.2.json rename to pkgs/development/cuda-modules/modules/cutensor/manifests/feature_1.6.2.json diff --git a/pkgs/development/cuda-modules/cutensor/manifests/feature_1.7.0.json b/pkgs/development/cuda-modules/modules/cutensor/manifests/feature_1.7.0.json similarity index 100% rename from pkgs/development/cuda-modules/cutensor/manifests/feature_1.7.0.json rename to pkgs/development/cuda-modules/modules/cutensor/manifests/feature_1.7.0.json diff --git a/pkgs/development/cuda-modules/cutensor/manifests/redistrib_1.3.3.json b/pkgs/development/cuda-modules/modules/cutensor/manifests/redistrib_1.3.3.json similarity index 100% rename from pkgs/development/cuda-modules/cutensor/manifests/redistrib_1.3.3.json rename to pkgs/development/cuda-modules/modules/cutensor/manifests/redistrib_1.3.3.json diff --git a/pkgs/development/cuda-modules/cutensor/manifests/redistrib_1.4.0.json b/pkgs/development/cuda-modules/modules/cutensor/manifests/redistrib_1.4.0.json similarity index 100% rename from pkgs/development/cuda-modules/cutensor/manifests/redistrib_1.4.0.json rename to pkgs/development/cuda-modules/modules/cutensor/manifests/redistrib_1.4.0.json diff --git a/pkgs/development/cuda-modules/cutensor/manifests/redistrib_1.5.0.json b/pkgs/development/cuda-modules/modules/cutensor/manifests/redistrib_1.5.0.json similarity index 100% rename from pkgs/development/cuda-modules/cutensor/manifests/redistrib_1.5.0.json rename to pkgs/development/cuda-modules/modules/cutensor/manifests/redistrib_1.5.0.json diff --git a/pkgs/development/cuda-modules/cutensor/manifests/redistrib_1.6.2.json b/pkgs/development/cuda-modules/modules/cutensor/manifests/redistrib_1.6.2.json similarity index 100% rename from pkgs/development/cuda-modules/cutensor/manifests/redistrib_1.6.2.json rename to pkgs/development/cuda-modules/modules/cutensor/manifests/redistrib_1.6.2.json diff --git a/pkgs/development/cuda-modules/cutensor/manifests/redistrib_1.7.0.json b/pkgs/development/cuda-modules/modules/cutensor/manifests/redistrib_1.7.0.json similarity index 100% rename from pkgs/development/cuda-modules/cutensor/manifests/redistrib_1.7.0.json rename to pkgs/development/cuda-modules/modules/cutensor/manifests/redistrib_1.7.0.json diff --git a/pkgs/development/cuda-modules/modules/default.nix b/pkgs/development/cuda-modules/modules/default.nix index ccccd871479e1..70f9e4696921c 100644 --- a/pkgs/development/cuda-modules/modules/default.nix +++ b/pkgs/development/cuda-modules/modules/default.nix @@ -1,10 +1,37 @@ +{ config, lib, ... }: +let + inherit (lib) options types; +in { imports = [ ./generic + ./gpus.nix + ./nvcc-compatibilities.nix + ./versions.nix # Always after generic ./cuda ./cudnn - ./cutensor ./tensorrt + ./cutensor ]; + + # Flags are determined based on your CUDA toolkit by default. You may benefit + # from improved performance, reduced file size, or greater hardware support by + # passing a configuration based on your specific GPU environment. + # Please see the accompanying documentation or https://github.com/NixOS/nixpkgs/pull/205351 + + options = { + cudaSupport = options.mkOption { + description = "Build packages with CUDA support"; + type = types.bool; + }; + cudaCapabilities = options.mkOption { + description = "CUDA capabilities (hardware generations) to build for"; + type = types.listOf config.generic.types.cudaCapability; + }; + cudaForwardCompat = options.mkOption { + description = "Build with forward compatibility gencode (+PTX) to support future GPU generations"; + type = types.bool; + }; + }; } diff --git a/pkgs/development/cuda-modules/modules/generic/manifests/default.nix b/pkgs/development/cuda-modules/modules/generic/manifests/default.nix index 6c12919ff4000..895ae11f581da 100644 --- a/pkgs/development/cuda-modules/modules/generic/manifests/default.nix +++ b/pkgs/development/cuda-modules/modules/generic/manifests/default.nix @@ -1,7 +1,105 @@ -{lib, config, ...}: +{ lib, config, ... }: +let + inherit (lib) + attrsets + lists + options + trivial + types + versions + ; +in { options.generic.manifests = { - feature = import ./feature/manifest.nix {inherit lib config;}; - redistrib = import ./redistrib/manifest.nix {inherit lib;}; + utils = { + manifestPathToManifestMeta = options.mkOption { + type = types.functionTo config.generic.manifests.types.manifestMeta; + description = "A function that takes a path to a manifest file and returns a manifestMeta"; + default = + filename: + let + regex = "^.*(feature|redistrib)_([[:digit:]]+\\.[[:digit:]]+\\.[[:digit:]]+)\\.json$"; + components = builtins.match regex (builtins.baseNameOf filename); + in + trivial.throwIf (components == null || builtins.length components != 2) + "Unexpected error while processing manifest files: regex failed to match filename ${builtins.toString filename}" + { + path = filename; + kind = builtins.head components; + version = versions.majorMinor (lists.last components); + }; + }; + manifestMetasToManifests = options.mkOption { + type = types.functionTo config.generic.manifests.types.manifests; + description = "A function that takes a list of manifestMeta an attribute set of manifest version to manifests"; + default = + builtins.foldl' + ( + manifests: manifestMeta: + let + inherit (manifestMeta) version kind path; + nullableExistingEntry = + attrsets.attrByPath + [ + version + kind + ] + null + manifests; + + newEntry = + attrsets.setAttrByPath + [ + version + kind + ] + ( + trivial.throwIf (nullableExistingEntry != null) + "Unexpected error while processing manifest files: duplicate version ${version}" + (trivial.importJSON path) + ); + in + attrsets.recursiveUpdate manifests newEntry + ) + { }; + }; + }; + types = options.mkOption { + description = "A set of generic types."; + type = types.attrsOf types.optionType; + default = { + manifestMeta = types.submodule { + options = { + path = options.mkOption { + description = "Path to the manifest file"; + type = types.path; + }; + kind = options.mkOption { + description = "The kind of manifest file"; + type = types.enum [ + "feature" + "redistrib" + ]; + }; + version = options.mkOption { + description = "The manifest version (major.minor)"; + type = config.generic.types.majorMinorVersion; + }; + }; + }; + manifestPathParser = types.functionTo config.generic.manifests.types.manifestMeta; + manifestPaths = types.listOf types.path; + manifestMetas = types.listOf config.generic.manifests.types.manifestMeta; + manifestMetasToManifests = types.functionTo config.generic.manifests.types.manifests; + manifests = types.attrsOf ( + types.submodule { + options = { + feature = import ./feature/manifest.nix { inherit lib config; }; + redistrib = import ./redistrib/manifest.nix { inherit lib; }; + }; + } + ); + }; + }; }; } diff --git a/pkgs/development/cuda-modules/modules/generic/types/default.nix b/pkgs/development/cuda-modules/modules/generic/types/default.nix index 61d13b3cc8d2b..4454c8119ae99 100644 --- a/pkgs/development/cuda-modules/modules/generic/types/default.nix +++ b/pkgs/development/cuda-modules/modules/generic/types/default.nix @@ -1,24 +1,37 @@ -{lib, ...}: +{ lib, ... }: let - inherit (lib) options types; + inherit (lib) + options + types + ; in { options.generic.types = options.mkOption { - type = types.attrsOf types.optionType; - default = {}; description = "A set of generic types."; + type = types.attrsOf types.optionType; }; config.generic.types = { cudaArch = types.strMatching "^sm_[[:digit:]]+[a-z]?$" // { name = "cudaArch"; description = "A CUDA architecture name."; }; + + cudaCapability = types.strMatching "^[[:digit:]]+\\.[[:digit:]]+[a-z]?$" // { + name = "cudaCapability"; + description = "A CUDA capability version."; + }; + # https://github.com/ConnorBaker/cuda-redist-find-features/blob/c841980e146f8664bbcd0ba1399e486b7910617b/cuda_redist_find_features/types/_lib_so_name.py libSoName = types.strMatching ".*\\.so(\\.[[:digit:]]+)*$" // { name = "libSoName"; description = "The name of a shared object file."; }; + majorVersion = types.strMatching "^([[:digit:]]+)$" // { + name = "majorVersion"; + description = "A version number with a major component."; + }; + majorMinorVersion = types.strMatching "^([[:digit:]]+)\\.([[:digit:]]+)$" // { name = "majorMinorVersion"; description = "A version number with a major and minor component."; @@ -35,5 +48,18 @@ in name = "majorMinorPatchBuildVersion"; description = "A version number with a major, minor, patch, and build component."; }; + + fixupFn = + types.functionTo ( + types.oneOf [ + types.attrs + (types.functionTo types.attrs) + (types.functionTo (types.functionTo types.attrs)) + ] + ) + // { + name = "fixupFn"; + description = "A function that takes an attribute set provided by callPackage and returns a value suitable for overrideAttrs."; + }; }; } diff --git a/pkgs/development/cuda-modules/modules/gpus.nix b/pkgs/development/cuda-modules/modules/gpus.nix new file mode 100644 index 0000000000000..4dea5b8be4779 --- /dev/null +++ b/pkgs/development/cuda-modules/modules/gpus.nix @@ -0,0 +1,236 @@ +# Many thanks to Arnon Shimoni for maintaining a list of these architectures and capabilities. +# Without your work, this would have been much more difficult. +# https://arnon.dk/matching-sm-architectures-arch-and-gencode-for-various-nvidia-cards/ + +{ config, lib, ... }: +let + inherit (lib) modules options types; +in +{ + options.gpus = options.mkOption { + description = "Known GPUs"; + type = types.listOf ( + types.submodule { + options = { + archName = options.mkOption { + description = "Name of the microarchitecture"; + example = "Kepler"; + type = types.nonEmptyStr; + }; + computeCapability = options.mkOption { + description = "Compute capability of the GPU"; + example = "3.0"; + type = config.generic.types.cudaCapability; + }; + isJetson = options.mkOption { + description = '' + Whether a GPU is part of NVIDIA's line of Jetson embedded computers. + + This field is notable because it tells us what architecture to build for (as Jetson devices are + aarch64). More on Jetson devices here: + https://www.nvidia.com/en-us/autonomous-machines/embedded-systems/ + + NOTE: These architectures are only built upon request. + ''; + type = types.bool; + }; + minCudaVersion = options.mkOption { + description = "The minimum (inclusive) CUDA version that supports this GPU."; + example = "10.0"; + type = config.generic.types.majorMinorVersion; + }; + maxCudaVersion = options.mkOption { + description = "The maximum (exclusive) CUDA version that supports this GPU. null means there is no maximum."; + example = "10.2"; + type = types.nullOr config.generic.types.majorMinorVersion; + }; + dontDefaultAfter = options.mkOption { + description = '' + The CUDA version after which to exclude this GPU from the list of default capabilities + we build. null means we always include this GPU in the default capabilities if it is + supported. + ''; + example = "10.2"; + type = types.nullOr config.generic.types.majorMinorVersion; + }; + }; + } + ); + }; + config.gpus = modules.mkDefault [ + { + # GeForce 700, GT-730 + archName = "Kepler"; + computeCapability = "3.0"; + isJetson = false; + minCudaVersion = "10.0"; + dontDefaultAfter = "10.2"; + maxCudaVersion = "10.2"; + } + { + archName = "Kepler"; + computeCapability = "3.2"; + isJetson = false; + minCudaVersion = "10.0"; + dontDefaultAfter = "10.2"; + maxCudaVersion = "10.2"; + } + { + # Tesla K40 + archName = "Kepler"; + computeCapability = "3.5"; + isJetson = false; + minCudaVersion = "10.0"; + dontDefaultAfter = "11.0"; + maxCudaVersion = "11.8"; + } + { + # Tesla K80 + archName = "Kepler"; + computeCapability = "3.7"; + isJetson = false; + minCudaVersion = "10.0"; + dontDefaultAfter = "11.0"; + maxCudaVersion = "11.8"; + } + { + # Tesla/Quadro M series + archName = "Maxwell"; + computeCapability = "5.0"; + isJetson = false; + minCudaVersion = "10.0"; + dontDefaultAfter = "11.0"; + maxCudaVersion = null; + } + { + # Quadro M6000 , GeForce 900, GTX-970, GTX-980, GTX Titan X + archName = "Maxwell"; + computeCapability = "5.2"; + isJetson = false; + minCudaVersion = "10.0"; + dontDefaultAfter = "11.0"; + maxCudaVersion = null; + } + { + # Tegra (Jetson) TX1 / Tegra X1, Drive CX, Drive PX, Jetson Nano + archName = "Maxwell"; + computeCapability = "5.3"; + isJetson = true; + minCudaVersion = "10.0"; + dontDefaultAfter = null; + maxCudaVersion = null; + } + { + # Quadro GP100, Tesla P100, DGX-1 (Generic Pascal) + archName = "Pascal"; + computeCapability = "6.0"; + isJetson = false; + minCudaVersion = "10.0"; + dontDefaultAfter = null; + maxCudaVersion = null; + } + { + # GTX 1080, GTX 1070, GTX 1060, GTX 1050, GTX 1030 (GP108), GT 1010 (GP108) Titan Xp, Tesla + # P40, Tesla P4, Discrete GPU on the NVIDIA Drive PX2 + archName = "Pascal"; + computeCapability = "6.1"; + isJetson = false; + minCudaVersion = "10.0"; + dontDefaultAfter = null; + maxCudaVersion = null; + } + { + # Integrated GPU on the NVIDIA Drive PX2, Tegra (Jetson) TX2 + archName = "Pascal"; + computeCapability = "6.2"; + isJetson = true; + minCudaVersion = "10.0"; + dontDefaultAfter = null; + maxCudaVersion = null; + } + { + # DGX-1 with Volta, Tesla V100, GTX 1180 (GV104), Titan V, Quadro GV100 + archName = "Volta"; + computeCapability = "7.0"; + isJetson = false; + minCudaVersion = "10.0"; + dontDefaultAfter = null; + maxCudaVersion = null; + } + { + # Jetson AGX Xavier, Drive AGX Pegasus, Xavier NX + archName = "Volta"; + computeCapability = "7.2"; + isJetson = true; + minCudaVersion = "10.0"; + dontDefaultAfter = null; + maxCudaVersion = null; + } + { + # GTX/RTX Turing – GTX 1660 Ti, RTX 2060, RTX 2070, RTX 2080, Titan RTX, Quadro RTX 4000, + # Quadro RTX 5000, Quadro RTX 6000, Quadro RTX 8000, Quadro T1000/T2000, Tesla T4 + archName = "Turing"; + computeCapability = "7.5"; + isJetson = false; + minCudaVersion = "10.0"; + dontDefaultAfter = null; + maxCudaVersion = null; + } + { + # NVIDIA A100 (the name “Tesla” has been dropped – GA100), NVIDIA DGX-A100 + archName = "Ampere"; + computeCapability = "8.0"; + isJetson = false; + minCudaVersion = "11.2"; + dontDefaultAfter = null; + maxCudaVersion = null; + } + { + # Tesla GA10x cards, RTX Ampere – RTX 3080, GA102 – RTX 3090, RTX A2000, A3000, RTX A4000, + # A5000, A6000, NVIDIA A40, GA106 – RTX 3060, GA104 – RTX 3070, GA107 – RTX 3050, RTX A10, RTX + # A16, RTX A40, A2 Tensor Core GPU + archName = "Ampere"; + computeCapability = "8.6"; + isJetson = false; + minCudaVersion = "11.2"; + dontDefaultAfter = null; + maxCudaVersion = null; + } + { + # Jetson AGX Orin and Drive AGX Orin only + archName = "Ampere"; + computeCapability = "8.7"; + isJetson = true; + minCudaVersion = "11.5"; + dontDefaultAfter = null; + maxCudaVersion = null; + } + { + # NVIDIA GeForce RTX 4090, RTX 4080, RTX 6000, Tesla L40 + archName = "Ada"; + computeCapability = "8.9"; + isJetson = false; + minCudaVersion = "11.8"; + dontDefaultAfter = null; + maxCudaVersion = null; + } + { + # NVIDIA H100 (GH100) + archName = "Hopper"; + computeCapability = "9.0"; + isJetson = false; + minCudaVersion = "11.8"; + dontDefaultAfter = null; + maxCudaVersion = null; + } + { + # NVIDIA H100 (GH100) (Thor) + archName = "Hopper"; + computeCapability = "9.0a"; + isJetson = false; + minCudaVersion = "12.0"; + dontDefaultAfter = null; + maxCudaVersion = null; + } + ]; +} diff --git a/pkgs/development/cuda-modules/modules/nvcc-compatibilities.nix b/pkgs/development/cuda-modules/modules/nvcc-compatibilities.nix new file mode 100644 index 0000000000000..2b08f56e29b0c --- /dev/null +++ b/pkgs/development/cuda-modules/modules/nvcc-compatibilities.nix @@ -0,0 +1,149 @@ +# Taken from +# https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#host-compiler-support-policy +# +# NVCC performs a version check on the host compiler’s major version and so newer minor versions +# of the compilers listed below will be supported, but major versions falling outside the range +# will not be supported. +# +# NOTE: These constraints don't apply to Jetson, which uses something else. +# NOTE: NVIDIA can and will add support for newer compilers even during patch releases. +# E.g.: CUDA 12.2.1 maxxed out with support for Clang 15.0; 12.2.2 added support for Clang 16.0. +# NOTE: Because all platforms NVIDIA supports use GCC and Clang, we omit the architectures here. + +{ config, lib, ... }: +let + inherit (lib) modules options types; +in +{ + options.nvccCompatibilities = options.mkOption { + description = "Mapping of CUDA version to known NVCC compatibilities with GCC and Clang"; + type = types.attrsOf (types.submodule { + options = { + clangMinMajorVersion = options.mkOption { + description = "Minimum Clang major version to support this CUDA version"; + example = "6"; + type = config.generic.types.majorVersion; + }; + clangMaxMajorVersion = options.mkOption { + description = "Maximum Clang major version to support this CUDA version"; + example = "6"; + type = config.generic.types.majorVersion; + }; + gccMinMajorVersion = options.mkOption { + description = "Minimum GCC major version to support this CUDA version"; + example = "5"; + type = config.generic.types.majorVersion; + }; + gccMaxMajorVersion = options.mkOption { + description = "Maximum GCC major version to support this CUDA version"; + example = "7"; + type = config.generic.types.majorVersion; + }; + }; + }); + }; + config.nvccCompatibilities = + let + attrs = { + # Our baseline + # https://docs.nvidia.com/cuda/archive/10.0/cuda-toolkit-release-notes/index.html#cuda-compiler-new-features + "10.0" = { + clangMaxMajorVersion = "6"; + clangMinMajorVersion = "6"; + gccMaxMajorVersion = "7"; + gccMinMajorVersion = "5"; + }; + + # Added support for Clang 7 and GCC 8 + # https://docs.nvidia.com/cuda/archive/10.1/cuda-toolkit-release-notes/index.html#cuda-compiler-new-features + "10.1" = attrs."10.0" // { + clangMaxMajorVersion = "7"; + gccMaxMajorVersion = "8"; + }; + + # Added clang 8 + # https://docs.nvidia.com/cuda/archive/10.2/cuda-toolkit-release-notes/index.html#cuda-compiler-new-features + "10.2" = attrs."10.1" // { + clangMaxMajorVersion = "8"; + }; + + # Added support for Clang 9 and GCC 9 + # https://docs.nvidia.com/cuda/archive/11.0/cuda-toolkit-release-notes/index.html#cuda-compiler-new-features + "11.0" = { + clangMaxMajorVersion = "9"; + clangMinMajorVersion = "7"; + gccMaxMajorVersion = "9"; + gccMinMajorVersion = "6"; + }; + + # Added support for Clang 10 and GCC 10 + # https://docs.nvidia.com/cuda/archive/11.1.1/cuda-toolkit-release-notes/index.html#cuda-compiler-new-features + "11.1" = attrs."11.0" // { + clangMaxMajorVersion = "10"; + gccMaxMajorVersion = "10"; + }; + + # Added support for Clang 11 + # https://docs.nvidia.com/cuda/archive/11.2.2/cuda-installation-guide-linux/index.html#system-requirements + "11.2" = attrs."11.1" // { + clangMaxMajorVersion = "11"; + }; + + # No changes from 11.2 to 11.3 + "11.3" = attrs."11.2"; + + # Added support for Clang 12 and GCC 11 + # https://docs.nvidia.com/cuda/archive/11.4.4/cuda-toolkit-release-notes/index.html#cuda-general-new-features + "11.4" = attrs."11.3" // { + clangMaxMajorVersion = "12"; + # NOTE: There is a bug in the version of GLIBC that GCC 11 uses which causes it to fail to compile some CUDA + # code. As such, we skip it for this release, and do the bump in 11.6 (skipping 11.5). + # https://forums.developer.nvidia.com/t/cuda-11-5-samples-throw-multiple-error-attribute-malloc-does-not-take-arguments/192750/15 + # gccMaxMajorVersion = "11"; + }; + + # No changes from 11.4 to 11.5 + "11.5" = attrs."11.4"; + + # No changes from 11.5 to 11.6 + # However, as mentioned above, we add GCC 11 this release. + "11.6" = attrs."11.5" // { + gccMaxMajorVersion = "11"; + }; + + # Added support for Clang 13 + # https://docs.nvidia.com/cuda/archive/11.7.1/cuda-toolkit-release-notes/index.html#cuda-compiler-new-features + "11.7" = attrs."11.6" // { + clangMaxMajorVersion = "13"; + }; + + # Added support for Clang 14 + # https://docs.nvidia.com/cuda/archive/11.8.0/cuda-installation-guide-linux/index.html#system-requirements + "11.8" = attrs."11.7" // { + clangMaxMajorVersion = "14"; + }; + + # Added support for GCC 12 + # https://docs.nvidia.com/cuda/archive/12.0.1/cuda-installation-guide-linux/index.html#system-requirements + "12.0" = attrs."11.8" // { + gccMaxMajorVersion = "12"; + }; + + # Added support for Clang 15 + # https://docs.nvidia.com/cuda/archive/12.1.1/cuda-toolkit-release-notes/index.html#cuda-compilers-new-features + "12.1" = attrs."12.0" // { + clangMaxMajorVersion = "15"; + }; + + # Added support for Clang 16 + # https://docs.nvidia.com/cuda/archive/12.2.2/cuda-installation-guide-linux/index.html#host-compiler-support-policy + "12.2" = attrs."12.1" // { + clangMaxMajorVersion = "16"; + }; + + # No changes from 12.2 to 12.3 + "12.3" = attrs."12.2"; + }; + in + modules.mkDefault attrs; +} diff --git a/pkgs/development/cuda-modules/modules/tensorrt/default.nix b/pkgs/development/cuda-modules/modules/tensorrt/default.nix index e62942c679aa0..36d04d16ce503 100644 --- a/pkgs/development/cuda-modules/modules/tensorrt/default.nix +++ b/pkgs/development/cuda-modules/modules/tensorrt/default.nix @@ -1,6 +1,24 @@ -{options, ...}: +{ lib, options, ... }: +let + inherit (lib) types; + inherit (lib.options) mkOption; +in { - options.tensorrt.releases = options.generic.releases; + options.tensorrt = { + inherit (options.generic) releases; + shimsFnPath = mkOption { + description = "The path to a function to transform releases into manifest files"; + default = ./shims.nix; + type = types.path; + }; + fixupFnPath = mkOption { + description = "The path to a function to transform releases into manifest files"; + default = ./fixup.nix; + type = types.path; + }; + }; + config.tensorrt.releases = builtins.import ./releases.nix; + # TODO(@connorbaker): Figure out how to add additional options to the # to the generic release. # { diff --git a/pkgs/development/cuda-modules/tensorrt/fixup.nix b/pkgs/development/cuda-modules/modules/tensorrt/fixup.nix similarity index 82% rename from pkgs/development/cuda-modules/tensorrt/fixup.nix rename to pkgs/development/cuda-modules/modules/tensorrt/fixup.nix index d713189328ed7..16079d440ab6b 100644 --- a/pkgs/development/cuda-modules/tensorrt/fixup.nix +++ b/pkgs/development/cuda-modules/modules/tensorrt/fixup.nix @@ -1,7 +1,7 @@ { cudaVersion, - final, hostPlatform, + cudaPackages, lib, mkVersionedPackageName, package, @@ -11,11 +11,18 @@ }: let inherit (lib) + attrsets + lists maintainers meta strings versions ; + # targetArch :: Optional String + targetArch = attrsets.attrByPath [ hostPlatform.system ] null { + x86_64-linux = "x86_64-linux-gnu"; + aarch64-linux = "aarch64-linux-gnu"; + }; in finalAttrs: prevAttrs: { # Useful for inspecting why something went wrong. @@ -58,20 +65,11 @@ finalAttrs: prevAttrs: { # We need to look inside the extracted output to get the files we need. sourceRoot = "TensorRT-${finalAttrs.version}"; - buildInputs = prevAttrs.buildInputs ++ [finalAttrs.passthru.cudnn.lib]; + buildInputs = prevAttrs.buildInputs ++ [ finalAttrs.passthru.cudnn.lib ]; preInstall = - let - targetArch = - if hostPlatform.isx86_64 then - "x86_64-linux-gnu" - else if hostPlatform.isAarch64 then - "aarch64-linux-gnu" - else - throw "Unsupported architecture"; - in (prevAttrs.preInstall or "") - + '' + + strings.optionalString (targetArch != null) '' # Replace symlinks to bin and lib with the actual directories from targets. for dir in bin lib; do rm "$dir" @@ -101,13 +99,19 @@ finalAttrs: prevAttrs: { cudnn = let desiredName = mkVersionedPackageName "cudnn" package.cudnnVersion; - desiredIsAvailable = final ? desiredName; + desiredIsAvailable = cudaPackages ? desiredName; in - if package.cudnnVersion == null || !desiredIsAvailable then final.cudnn else final.${desiredName}; + if package.cudnnVersion == null || !desiredIsAvailable then + cudaPackages.cudnn + else + cudaPackages.${desiredName}; }; meta = prevAttrs.meta // { homepage = "https://developer.nvidia.com/tensorrt"; - maintainers = prevAttrs.meta.maintainers ++ [maintainers.aidalgol]; + maintainers = prevAttrs.meta.maintainers ++ [ maintainers.aidalgol ]; + badPlatforms = + (prevAttrs.meta.badPlatforms or [ ]) + ++ lists.optionals (targetArch == null) [ hostPlatform.system ]; }; } diff --git a/pkgs/development/cuda-modules/modules/tensorrt/releases.nix b/pkgs/development/cuda-modules/modules/tensorrt/releases.nix new file mode 100644 index 0000000000000..ffb16aa7048c7 --- /dev/null +++ b/pkgs/development/cuda-modules/modules/tensorrt/releases.nix @@ -0,0 +1,128 @@ +# NOTE: Check https://developer.nvidia.com/nvidia-tensorrt-8x-download. +# Version policy is to keep the latest minor release for each major release. +{ + # jetson + linux-aarch64 = [ ]; + # powerpc + linux-ppc64le = [ ]; + # server-grade arm + linux-sbsa = [ + { + version = "8.2.5.1"; + minCudaVersion = "11.4"; + maxCudaVersion = "11.4"; + cudnnVersion = "8.2"; + filename = "TensorRT-8.2.5.1.Ubuntu-20.04.aarch64-gnu.cuda-11.4.cudnn8.2.tar.gz"; + hash = "sha256-oWfQ3lq2aoMPv65THeotnMilTzP+QWqKeToLU8eO+qo="; + } + { + version = "8.4.3.1"; + minCudaVersion = "11.6"; + maxCudaVersion = "11.6"; + cudnnVersion = "8.4"; + filename = "TensorRT-8.4.3.1.Ubuntu-20.04.aarch64-gnu.cuda-11.6.cudnn8.4.tar.gz"; + hash = "sha256-9tLlrB8cKYFvN2xF0Pol5CZs06iuuI5mq+6jpzD8wWI="; + } + { + version = "8.5.3.1"; + minCudaVersion = "11.8"; + maxCudaVersion = "11.8"; + cudnnVersion = "8.6"; + filename = "TensorRT-8.5.3.1.Ubuntu-20.04.aarch64-gnu.cuda-11.8.cudnn8.6.tar.gz"; + hash = "sha256-GW//mX0brvN/waHo9Wd07xerOEz3X/H/HAW2ZehYtTA="; + } + { + version = "8.6.1.6"; + minCudaVersion = "12.0"; + maxCudaVersion = "12.0"; + cudnnVersion = null; + filename = "TensorRT-8.6.1.6.Ubuntu-20.04.aarch64-gnu.cuda-12.0.tar.gz"; + hash = "sha256-Lc4+v/yBr17VlecCSFMLUDlXMTYV68MGExwnUjGme5E="; + } + ]; + # x86_64 + linux-x86_64 = [ + { + version = "8.0.3.4"; + minCudaVersion = "10.2"; + maxCudaVersion = "10.2"; + cudnnVersion = "8.2"; + filename = "TensorRT-8.0.3.4.Linux.x86_64-gnu.cuda-10.2.cudnn8.2.tar.gz"; + hash = "sha256-LxcXgwe1OCRfwDsEsNLIkeNsOcx3KuF5Sj+g2dY6WD0="; + } + { + version = "8.0.3.4"; + minCudaVersion = "11.0"; + maxCudaVersion = "11.3"; + cudnnVersion = "8.2"; + filename = "TensorRT-8.0.3.4.Linux.x86_64-gnu.cuda-11.3.cudnn8.2.tar.gz"; + hash = "sha256-MXdDUCT/SqWm26jB7QarEcwOG/O7cS36Y6Q0IvQTE/M="; + } + { + version = "8.2.5.1"; + minCudaVersion = "10.2"; + maxCudaVersion = "10.2"; + cudnnVersion = "8.2"; + filename = "TensorRT-8.2.5.1.Linux.x86_64-gnu.cuda-10.2.cudnn8.2.tar.gz"; + hash = "sha256-XV2Bf2LH8OM2GEMjV80MDweb1hSVF/wFUcaW3KP2m8Q="; + } + { + # The docs claim this supports through 11.5 despite the file name indicating 11.4. + version = "8.2.5.1"; + minCudaVersion = "11.0"; + maxCudaVersion = "11.5"; + cudnnVersion = "8.2"; + filename = "TensorRT-8.2.5.1.Linux.x86_64-gnu.cuda-11.4.cudnn8.2.tar.gz"; + hash = "sha256-LcNpYvDiT7AavqzK1MRlijo2qDN7jznigeS77US713E="; + } + { + version = "8.4.3.1"; + minCudaVersion = "10.2"; + maxCudaVersion = "10.2"; + cudnnVersion = "8.4"; + filename = "TensorRT-8.4.3.1.Linux.x86_64-gnu.cuda-10.2.cudnn8.4.tar.gz"; + hash = "sha256-2c3Zzt93FBWWQtrSIvpbzzS6BT9s0NzALzdwXGLOZEU="; + } + { + # The docs claim this supports through 11.7 despite the file name indicating 11.6. + version = "8.4.3.1"; + minCudaVersion = "11.0"; + maxCudaVersion = "11.7"; + cudnnVersion = "8.4"; + filename = "TensorRT-8.4.3.1.Linux.x86_64-gnu.cuda-11.6.cudnn8.4.tar.gz"; + hash = "sha256-jXwghcFjncxzh1BIwjWYqFJs4wiRNoduMdkCWOSeT2E="; + } + { + version = "8.5.3.1"; + minCudaVersion = "10.2"; + maxCudaVersion = "10.2"; + cudnnVersion = "8.6"; + filename = "TensorRT-8.5.3.1.Linux.x86_64-gnu.cuda-10.2.cudnn8.6.tar.gz"; + hash = "sha256-WCt6yfOmFbrjqdYCj6AE2+s2uFpISwk6urP+2I0BnGQ="; + } + { + version = "8.5.3.1"; + minCudaVersion = "11.0"; + maxCudaVersion = "11.8"; + cudnnVersion = "8.6"; + filename = "TensorRT-8.5.3.1.Linux.x86_64-gnu.cuda-11.8.cudnn8.6.tar.gz"; + hash = "sha256-BNeuOYvPTUAfGxI0DVsNrX6Z/FAB28+SE0ptuGu7YDY="; + } + { + version = "8.6.1.6"; + minCudaVersion = "11.0"; + maxCudaVersion = "11.8"; + cudnnVersion = "8.9"; + filename = "TensorRT-8.6.1.6.Linux.x86_64-gnu.cuda-11.8.tar.gz"; + hash = "sha256-Fb/mBT1F/uxF7McSOpEGB2sLQ/oENfJC2J3KB3gzd1k="; + } + { + version = "8.6.1.6"; + minCudaVersion = "12.0"; + maxCudaVersion = "12.1"; + cudnnVersion = "8.9"; + filename = "TensorRT-8.6.1.6.Linux.x86_64-gnu.cuda-12.0.tar.gz"; + hash = "sha256-D4FXpfxTKZQ7M4uJNZE3M1CvqQyoEjnNrddYDNHrolQ="; + } + ]; +} diff --git a/pkgs/development/cuda-modules/modules/tensorrt/shims.nix b/pkgs/development/cuda-modules/modules/tensorrt/shims.nix new file mode 100644 index 0000000000000..11420ed5c2e03 --- /dev/null +++ b/pkgs/development/cuda-modules/modules/tensorrt/shims.nix @@ -0,0 +1,24 @@ +# Shims to mimic the shape of ../modules/generic/manifests/{feature,redistrib}/manifest.nix +{ + lib, + package, + # redistArch :: Optional String + # String is null if the given architecture is unsupported. + redistArch, +}: +{ + feature.tensorrt = lib.optionalAttrs (redistArch != null) { + ${redistArch}.outputs = { + bin = true; + lib = true; + static = true; + dev = true; + sample = true; + python = true; + }; + }; + redistrib.tensorrt = { + name = "TensorRT: a high-performance deep learning interface"; + inherit (package) version; + }; +} diff --git a/pkgs/development/cuda-modules/modules/versions.nix b/pkgs/development/cuda-modules/modules/versions.nix new file mode 100644 index 0000000000000..7681eccb158db --- /dev/null +++ b/pkgs/development/cuda-modules/modules/versions.nix @@ -0,0 +1,31 @@ +{config, lib, ...}: +let + inherit (lib) options types; +in +{ + options.versions = options.mkOption { + description = "A list of CUDA versions to create package sets for"; + type = types.listOf (config.generic.types.majorMinorVersion); + default = [ + # CUDA 10 + "10.0" + "10.1" + "10.2" + # CUDA 11 + "11.0" + "11.1" + "11.2" + "11.3" + "11.4" + "11.5" + "11.6" + "11.7" + "11.8" + # CUDA 12 + "12.0" + "12.1" + "12.2" + "12.3" + ]; + }; +} \ No newline at end of file diff --git a/pkgs/development/cuda-modules/nvcc-compatibilities.nix b/pkgs/development/cuda-modules/nvcc-compatibilities.nix deleted file mode 100644 index 4af1b511a1d9d..0000000000000 --- a/pkgs/development/cuda-modules/nvcc-compatibilities.nix +++ /dev/null @@ -1,124 +0,0 @@ -# Taken from -# https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#host-compiler-support-policy -# -# NVCC performs a version check on the host compiler’s major version and so newer minor versions -# of the compilers listed below will be supported, but major versions falling outside the range -# will not be supported. -# -# NOTE: These constraints don't apply to Jetson, which uses something else. -# NOTE: NVIDIA can and will add support for newer compilers even during patch releases. -# E.g.: CUDA 12.2.1 maxxed out with support for Clang 15.0; 12.2.2 added support for Clang 16.0. -# NOTE: Because all platforms NVIDIA supports use GCC and Clang, we omit the architectures here. -# Type Aliases -# CudaVersion = String (two-part version number, e.g. "11.2") -# Platform = String (e.g. "x86_64-linux") -# CompilerCompatibilities = { -# clangMaxMajorVersion = String (e.g. "15") -# clangMinMajorVersion = String (e.g. "7") -# gccMaxMajorVersion = String (e.g. "11") -# gccMinMajorVersion = String (e.g. "6") -# } -let - # attrs :: AttrSet CudaVersion CompilerCompatibilities - attrs = { - # Our baseline - # https://docs.nvidia.com/cuda/archive/10.0/cuda-toolkit-release-notes/index.html#cuda-compiler-new-features - "10.0" = { - clangMaxMajorVersion = "6"; - clangMinMajorVersion = "6"; - gccMaxMajorVersion = "7"; - gccMinMajorVersion = "5"; - }; - - # Added support for Clang 7 and GCC 8 - # https://docs.nvidia.com/cuda/archive/10.1/cuda-toolkit-release-notes/index.html#cuda-compiler-new-features - "10.1" = attrs."10.0" // { - clangMaxMajorVersion = "7"; - gccMaxMajorVersion = "8"; - }; - - # Added clang 8 - # https://docs.nvidia.com/cuda/archive/10.2/cuda-toolkit-release-notes/index.html#cuda-compiler-new-features - "10.2" = attrs."10.1" // { - clangMaxMajorVersion = "8"; - }; - - # Added support for Clang 9 and GCC 9 - # https://docs.nvidia.com/cuda/archive/11.0/cuda-toolkit-release-notes/index.html#cuda-compiler-new-features - "11.0" = { - clangMaxMajorVersion = "9"; - clangMinMajorVersion = "7"; - gccMaxMajorVersion = "9"; - gccMinMajorVersion = "6"; - }; - - # Added support for Clang 10 and GCC 10 - # https://docs.nvidia.com/cuda/archive/11.1.1/cuda-toolkit-release-notes/index.html#cuda-compiler-new-features - "11.1" = attrs."11.0" // { - clangMaxMajorVersion = "10"; - gccMaxMajorVersion = "10"; - }; - - # Added support for Clang 11 - # https://docs.nvidia.com/cuda/archive/11.2.2/cuda-installation-guide-linux/index.html#system-requirements - "11.2" = attrs."11.1" // { - clangMaxMajorVersion = "11"; - }; - - # No changes from 11.2 to 11.3 - "11.3" = attrs."11.2"; - - # Added support for Clang 12 and GCC 11 - # https://docs.nvidia.com/cuda/archive/11.4.4/cuda-toolkit-release-notes/index.html#cuda-general-new-features - "11.4" = attrs."11.3" // { - clangMaxMajorVersion = "12"; - # NOTE: There is a bug in the version of GLIBC that GCC 11 uses which causes it to fail to compile some CUDA - # code. As such, we skip it for this release, and do the bump in 11.6 (skipping 11.5). - # https://forums.developer.nvidia.com/t/cuda-11-5-samples-throw-multiple-error-attribute-malloc-does-not-take-arguments/192750/15 - # gccMaxMajorVersion = "11"; - }; - - # No changes from 11.4 to 11.5 - "11.5" = attrs."11.4"; - - # No changes from 11.5 to 11.6 - # However, as mentioned above, we add GCC 11 this release. - "11.6" = attrs."11.5" // { - gccMaxMajorVersion = "11"; - }; - - # Added support for Clang 13 - # https://docs.nvidia.com/cuda/archive/11.7.1/cuda-toolkit-release-notes/index.html#cuda-compiler-new-features - "11.7" = attrs."11.6" // { - clangMaxMajorVersion = "13"; - }; - - # Added support for Clang 14 - # https://docs.nvidia.com/cuda/archive/11.8.0/cuda-installation-guide-linux/index.html#system-requirements - "11.8" = attrs."11.7" // { - clangMaxMajorVersion = "14"; - }; - - # Added support for GCC 12 - # https://docs.nvidia.com/cuda/archive/12.0.1/cuda-installation-guide-linux/index.html#system-requirements - "12.0" = attrs."11.8" // { - gccMaxMajorVersion = "12"; - }; - - # Added support for Clang 15 - # https://docs.nvidia.com/cuda/archive/12.1.1/cuda-toolkit-release-notes/index.html#cuda-compilers-new-features - "12.1" = attrs."12.0" // { - clangMaxMajorVersion = "15"; - }; - - # Added support for Clang 16 - # https://docs.nvidia.com/cuda/archive/12.2.2/cuda-installation-guide-linux/index.html#host-compiler-support-policy - "12.2" = attrs."12.1" // { - clangMaxMajorVersion = "16"; - }; - - # No changes from 12.2 to 12.3 - "12.3" = attrs."12.2"; - }; -in -attrs diff --git a/pkgs/development/cuda-modules/setup-hooks/default.nix b/pkgs/development/cuda-modules/setup-hooks/default.nix index b6db0729ce451..b40acad58bc80 100644 --- a/pkgs/development/cuda-modules/setup-hooks/default.nix +++ b/pkgs/development/cuda-modules/setup-hooks/default.nix @@ -1,50 +1,36 @@ -{callPackage}: { # Internal hook, used by cudatoolkit and cuda redist packages # to accommodate automatic CUDAToolkit_ROOT construction markForCudatoolkitRootHook = - callPackage - ( - {makeSetupHook}: - makeSetupHook {name = "mark-for-cudatoolkit-root-hook";} ./mark-for-cudatoolkit-root-hook.sh - ) - {}; + { makeSetupHook }: + makeSetupHook { name = "mark-for-cudatoolkit-root-hook"; } ./mark-for-cudatoolkit-root-hook.sh; # Currently propagated by cuda_nvcc or cudatoolkit, rather than used directly setupCudaHook = - (callPackage - ( - {makeSetupHook, backendStdenv}: - makeSetupHook - { - name = "setup-cuda-hook"; + { makeSetupHook, backendStdenv }: + makeSetupHook + { + name = "setup-cuda-hook"; - substitutions.setupCudaHook = placeholder "out"; + substitutions.setupCudaHook = placeholder "out"; - # Point NVCC at a compatible compiler - substitutions.ccRoot = "${backendStdenv.cc}"; + # Point NVCC at a compatible compiler + substitutions.ccRoot = "${backendStdenv.cc}"; - # Required in addition to ccRoot as otherwise bin/gcc is looked up - # when building CMakeCUDACompilerId.cu - substitutions.ccFullPath = "${backendStdenv.cc}/bin/${backendStdenv.cc.targetPrefix}c++"; - } - ./setup-cuda-hook.sh - ) - {} - ); + # Required in addition to ccRoot as otherwise bin/gcc is looked up + # when building CMakeCUDACompilerId.cu + substitutions.ccFullPath = "${backendStdenv.cc}/bin/${backendStdenv.cc.targetPrefix}c++"; + } + ./setup-cuda-hook.sh; autoAddOpenGLRunpathHook = - callPackage - ( - {addOpenGLRunpath, makeSetupHook}: - makeSetupHook - { - name = "auto-add-opengl-runpath-hook"; - propagatedBuildInputs = [addOpenGLRunpath]; - } - ./auto-add-opengl-runpath-hook.sh - ) - {}; + { addOpenGLRunpath, makeSetupHook }: + makeSetupHook + { + name = "auto-add-opengl-runpath-hook"; + propagatedBuildInputs = [ addOpenGLRunpath ]; + } + ./auto-add-opengl-runpath-hook.sh; # autoAddCudaCompatRunpathHook hook must be added AFTER `setupCudaHook`. Both # hooks prepend a path with `libcuda.so` to the `DT_RUNPATH` section of @@ -52,19 +38,21 @@ # it doesn't have any effect) and thus appear first. Meaning this hook must be # executed last. autoAddCudaCompatRunpathHook = - callPackage - ( - {makeSetupHook, cuda_compat, flags}: - makeSetupHook - { - name = "auto-add-cuda-compat-runpath-hook"; - substitutions = { - # Hotfix Ofborg evaluation - libcudaPath = if flags.isJetsonBuild then "${cuda_compat}/compat" else null; - }; - meta.broken = !flags.isJetsonBuild; - } - ./auto-add-cuda-compat-runpath.sh - ) - {}; + { + makeSetupHook, + cudaPackages, + flags, + hostPlatform, + lib, + }: + let + isOkay = flags.isJetsonBuild && cudaPackages ? cuda_compat; + in + makeSetupHook + { + name = "auto-add-cuda-compat-runpath-hook"; + substitutions.libcudaPath = if isOkay then "${cudaPackages.cuda_compat}/compat" else null; + meta.badPlatforms = if isOkay then [] else [ hostPlatform.system ]; + } + ./auto-add-cuda-compat-runpath.sh; } diff --git a/pkgs/development/cuda-modules/tensorrt/releases.nix b/pkgs/development/cuda-modules/tensorrt/releases.nix deleted file mode 100644 index d6a1f0487dd43..0000000000000 --- a/pkgs/development/cuda-modules/tensorrt/releases.nix +++ /dev/null @@ -1,130 +0,0 @@ -# NOTE: Check https://developer.nvidia.com/nvidia-tensorrt-8x-download. -# Version policy is to keep the latest minor release for each major release. -{ - tensorrt.releases = { - # jetson - linux-aarch64 = []; - # powerpc - linux-ppc64le = []; - # server-grade arm - linux-sbsa = [ - { - version = "8.2.5.1"; - minCudaVersion = "11.4"; - maxCudaVersion = "11.4"; - cudnnVersion = "8.2"; - filename = "TensorRT-8.2.5.1.Ubuntu-20.04.aarch64-gnu.cuda-11.4.cudnn8.2.tar.gz"; - hash = "sha256-oWfQ3lq2aoMPv65THeotnMilTzP+QWqKeToLU8eO+qo="; - } - { - version = "8.4.3.1"; - minCudaVersion = "11.6"; - maxCudaVersion = "11.6"; - cudnnVersion = "8.4"; - filename = "TensorRT-8.4.3.1.Ubuntu-20.04.aarch64-gnu.cuda-11.6.cudnn8.4.tar.gz"; - hash = "sha256-9tLlrB8cKYFvN2xF0Pol5CZs06iuuI5mq+6jpzD8wWI="; - } - { - version = "8.5.3.1"; - minCudaVersion = "11.8"; - maxCudaVersion = "11.8"; - cudnnVersion = "8.6"; - filename = "TensorRT-8.5.3.1.Ubuntu-20.04.aarch64-gnu.cuda-11.8.cudnn8.6.tar.gz"; - hash = "sha256-GW//mX0brvN/waHo9Wd07xerOEz3X/H/HAW2ZehYtTA="; - } - { - version = "8.6.1.6"; - minCudaVersion = "12.0"; - maxCudaVersion = "12.0"; - cudnnVersion = null; - filename = "TensorRT-8.6.1.6.Ubuntu-20.04.aarch64-gnu.cuda-12.0.tar.gz"; - hash = "sha256-Lc4+v/yBr17VlecCSFMLUDlXMTYV68MGExwnUjGme5E="; - } - ]; - # x86_64 - linux-x86_64 = [ - { - version = "8.0.3.4"; - minCudaVersion = "10.2"; - maxCudaVersion = "10.2"; - cudnnVersion = "8.2"; - filename = "TensorRT-8.0.3.4.Linux.x86_64-gnu.cuda-10.2.cudnn8.2.tar.gz"; - hash = "sha256-LxcXgwe1OCRfwDsEsNLIkeNsOcx3KuF5Sj+g2dY6WD0="; - } - { - version = "8.0.3.4"; - minCudaVersion = "11.0"; - maxCudaVersion = "11.3"; - cudnnVersion = "8.2"; - filename = "TensorRT-8.0.3.4.Linux.x86_64-gnu.cuda-11.3.cudnn8.2.tar.gz"; - hash = "sha256-MXdDUCT/SqWm26jB7QarEcwOG/O7cS36Y6Q0IvQTE/M="; - } - { - version = "8.2.5.1"; - minCudaVersion = "10.2"; - maxCudaVersion = "10.2"; - cudnnVersion = "8.2"; - filename = "TensorRT-8.2.5.1.Linux.x86_64-gnu.cuda-10.2.cudnn8.2.tar.gz"; - hash = "sha256-XV2Bf2LH8OM2GEMjV80MDweb1hSVF/wFUcaW3KP2m8Q="; - } - { - # The docs claim this supports through 11.5 despite the file name indicating 11.4. - version = "8.2.5.1"; - minCudaVersion = "11.0"; - maxCudaVersion = "11.5"; - cudnnVersion = "8.2"; - filename = "TensorRT-8.2.5.1.Linux.x86_64-gnu.cuda-11.4.cudnn8.2.tar.gz"; - hash = "sha256-LcNpYvDiT7AavqzK1MRlijo2qDN7jznigeS77US713E="; - } - { - version = "8.4.3.1"; - minCudaVersion = "10.2"; - maxCudaVersion = "10.2"; - cudnnVersion = "8.4"; - filename = "TensorRT-8.4.3.1.Linux.x86_64-gnu.cuda-10.2.cudnn8.4.tar.gz"; - hash = "sha256-2c3Zzt93FBWWQtrSIvpbzzS6BT9s0NzALzdwXGLOZEU="; - } - { - # The docs claim this supports through 11.7 despite the file name indicating 11.6. - version = "8.4.3.1"; - minCudaVersion = "11.0"; - maxCudaVersion = "11.7"; - cudnnVersion = "8.4"; - filename = "TensorRT-8.4.3.1.Linux.x86_64-gnu.cuda-11.6.cudnn8.4.tar.gz"; - hash = "sha256-jXwghcFjncxzh1BIwjWYqFJs4wiRNoduMdkCWOSeT2E="; - } - { - version = "8.5.3.1"; - minCudaVersion = "10.2"; - maxCudaVersion = "10.2"; - cudnnVersion = "8.6"; - filename = "TensorRT-8.5.3.1.Linux.x86_64-gnu.cuda-10.2.cudnn8.6.tar.gz"; - hash = "sha256-WCt6yfOmFbrjqdYCj6AE2+s2uFpISwk6urP+2I0BnGQ="; - } - { - version = "8.5.3.1"; - minCudaVersion = "11.0"; - maxCudaVersion = "11.8"; - cudnnVersion = "8.6"; - filename = "TensorRT-8.5.3.1.Linux.x86_64-gnu.cuda-11.8.cudnn8.6.tar.gz"; - hash = "sha256-BNeuOYvPTUAfGxI0DVsNrX6Z/FAB28+SE0ptuGu7YDY="; - } - { - version = "8.6.1.6"; - minCudaVersion = "11.0"; - maxCudaVersion = "11.8"; - cudnnVersion = "8.9"; - filename = "TensorRT-8.6.1.6.Linux.x86_64-gnu.cuda-11.8.tar.gz"; - hash = "sha256-Fb/mBT1F/uxF7McSOpEGB2sLQ/oENfJC2J3KB3gzd1k="; - } - { - version = "8.6.1.6"; - minCudaVersion = "12.0"; - maxCudaVersion = "12.1"; - cudnnVersion = "8.9"; - filename = "TensorRT-8.6.1.6.Linux.x86_64-gnu.cuda-12.0.tar.gz"; - hash = "sha256-D4FXpfxTKZQ7M4uJNZE3M1CvqQyoEjnNrddYDNHrolQ="; - } - ]; - }; -} diff --git a/pkgs/development/cuda-modules/tensorrt/shims.nix b/pkgs/development/cuda-modules/tensorrt/shims.nix deleted file mode 100644 index 8be3e7988bb34..0000000000000 --- a/pkgs/development/cuda-modules/tensorrt/shims.nix +++ /dev/null @@ -1,16 +0,0 @@ -# Shims to mimic the shape of ../modules/generic/manifests/{feature,redistrib}/release.nix -{package, redistArch}: -{ - featureRelease.${redistArch}.outputs = { - bin = true; - lib = true; - static = true; - dev = true; - sample = true; - python = true; - }; - redistribRelease = { - name = "TensorRT: a high-performance deep learning interface"; - inherit (package) version; - }; -} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 60b5802a02a55..7bb6251b371a7 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -7332,26 +7332,32 @@ with pkgs; snooze = callPackage ../tools/system/snooze { }; - cudaPackages_10_0 = callPackage ./cuda-packages.nix { cudaVersion = "10.0"; }; - cudaPackages_10_1 = callPackage ./cuda-packages.nix { cudaVersion = "10.1"; }; - cudaPackages_10_2 = callPackage ./cuda-packages.nix { cudaVersion = "10.2"; }; - cudaPackages_10 = recurseIntoAttrs cudaPackages_10_2; + inherit (callPackage ./cuda-packages.nix {}) + # CUDA Packages 10.x + cudaPackages_10_0 + cudaPackages_10_1 + cudaPackages_10_2 + + # CUDA Packages 11.x + cudaPackages_11_0 + cudaPackages_11_1 + cudaPackages_11_2 + cudaPackages_11_3 + cudaPackages_11_4 + cudaPackages_11_5 + cudaPackages_11_6 + cudaPackages_11_7 + cudaPackages_11_8 + + # CUDA Packages 12.x + cudaPackages_12_0 + cudaPackages_12_1 + cudaPackages_12_2 + cudaPackages_12_3 + ; - cudaPackages_11_0 = callPackage ./cuda-packages.nix { cudaVersion = "11.0"; }; - cudaPackages_11_1 = callPackage ./cuda-packages.nix { cudaVersion = "11.1"; }; - cudaPackages_11_2 = callPackage ./cuda-packages.nix { cudaVersion = "11.2"; }; - cudaPackages_11_3 = callPackage ./cuda-packages.nix { cudaVersion = "11.3"; }; - cudaPackages_11_4 = callPackage ./cuda-packages.nix { cudaVersion = "11.4"; }; - cudaPackages_11_5 = callPackage ./cuda-packages.nix { cudaVersion = "11.5"; }; - cudaPackages_11_6 = callPackage ./cuda-packages.nix { cudaVersion = "11.6"; }; - cudaPackages_11_7 = callPackage ./cuda-packages.nix { cudaVersion = "11.7"; }; - cudaPackages_11_8 = callPackage ./cuda-packages.nix { cudaVersion = "11.8"; }; + cudaPackages_10 = recurseIntoAttrs cudaPackages_10_2; cudaPackages_11 = recurseIntoAttrs cudaPackages_11_8; - - cudaPackages_12_0 = callPackage ./cuda-packages.nix { cudaVersion = "12.0"; }; - cudaPackages_12_1 = callPackage ./cuda-packages.nix { cudaVersion = "12.1"; }; - cudaPackages_12_2 = callPackage ./cuda-packages.nix { cudaVersion = "12.2"; }; - cudaPackages_12_3 = callPackage ./cuda-packages.nix { cudaVersion = "12.3"; }; cudaPackages_12 = recurseIntoAttrs cudaPackages_12_0; # Use the older cudaPackages for tensorflow and jax, as determined by cudnn diff --git a/pkgs/top-level/cuda-packages.nix b/pkgs/top-level/cuda-packages.nix index 25b80051028c2..7f85829e84cbf 100644 --- a/pkgs/top-level/cuda-packages.nix +++ b/pkgs/top-level/cuda-packages.nix @@ -21,214 +21,292 @@ # # I've (@connorbaker) attempted to do that, though I'm unsure of how this will interact with overrides. { - callPackage, - cudaVersion, lib, hostPlatform, - newScope, pkgs, + makeScopeWithSplicing', + generateSplicesForMkScope, __attrsFailEvaluation ? true, }: let inherit (lib) attrsets - fixedPoints + lists modules - options strings trivial - types versions ; - # Utility function - # nullableOr : Optional a -> a -> a - nullableOr = nullable: default: if nullable != null then nullable else default; - evaluatedModules = modules.evalModules { - modules = [ - { - options.cudaVersion = options.mkOption { - description = "The version of CUDA to use."; - type = types.str; - }; - config = { - inherit cudaVersion; - }; - } - ../development/cuda-modules/modules + inherit + (modules.evalModules { + modules = [ + { + inherit (pkgs.config) cudaSupport; + cudaCapabilities = pkgs.config.cudaCapabilities or [ ]; + cudaForwardCompat = pkgs.config.cudaForwardCompat or true; + } + ../development/cuda-modules/modules + ]; + }) + config + ; + + mkVersionedPackageName = + name: version: + strings.concatStringsSep "_" [ + name + (strings.replaceStrings [ "." ] [ "_" ] (versions.majorMinor version)) ]; - }; - genericManifestBuilderFn = - { - callPackage, - pname, - redistName, - manifests, - libPath ? null, - overrideAttrsFns ? [ ], - }: + makeCudaPackages = + cudaVersion: final: let - drv = callPackage ../development/cuda-modules/generic-builders/manifest.nix { - inherit - pname - redistName - manifests - libPath - ; - }; - in - builtins.foldl' (drv: overrideAttrsFn: drv.overrideAttrs overrideAttrsFn) drv overrideAttrsFns; + # NOTE: Use of `final.callPackage` for `callPackageOnAttrs` doesn't run the risk of infinite recursion, + # as the `final` argument is not used to compute the attribute names. + callPackageOnAttrs = attrsets.mapAttrs (_: value: final.callPackage value { }); - # Backbone - gpus = builtins.import ../development/cuda-modules/gpus.nix; - nvccCompatibilities = builtins.import ../development/cuda-modules/nvcc-compatibilities.nix; - flags = callPackage ../development/cuda-modules/flags.nix { inherit cudaVersion gpus; }; - passthruFunction = - ( - final: - { - inherit cudaVersion lib pkgs; + # Flags used to enable different features of cudaPackages -- we cannot use final.callPackage + # because we use `flags` to determine the presence of certain packages, which would cause + # infinite recursion. + flags = builtins.import ../development/cuda-modules/flags.nix { inherit - gpus - nvccCompatibilities - flags - evaluatedModules + pkgs + lib + config + cudaVersion + hostPlatform ; - cudaMajorVersion = versions.major cudaVersion; - cudaMajorMinorVersion = versions.majorMinor cudaVersion; - # cudaVersionOlder : Version -> Boolean - cudaVersionOlder = strings.versionOlder cudaVersion; - # cudaVersionAtLeast : Version -> Boolean - cudaVersionAtLeast = strings.versionAtLeast cudaVersion; + }; - # Maintain a reference to the final cudaPackages. - # Without this, if we use `final.callPackage` and a package accepts `cudaPackages` as an argument, - # it's provided with `cudaPackages` from the top-level scope, which is not what we want. We want to - # provide the `cudaPackages` from the final scope -- that is, the *current* scope. - cudaPackages = final; + # Fixed-length cudaVersion strings + cudaMajorVersion = versions.major cudaVersion; + cudaMajorMinorVersion = versions.majorMinor cudaVersion; - # TODO(@connorbaker): `cudaFlags` is an alias for `flags` which should be removed in the future. - cudaFlags = flags; + # cudaVersionOlder : Version -> Boolean + cudaVersionOlder = strings.versionOlder cudaVersion; + # cudaVersionAtLeast : Version -> Boolean + cudaVersionAtLeast = strings.versionAtLeast cudaVersion; - # Exposed as cudaPackages.backendStdenv. - # This is what nvcc uses as a backend, - # and it has to be an officially supported one (e.g. gcc11 for cuda11). - # - # It, however, propagates current stdenv's libstdc++ to avoid "GLIBCXX_* not found errors" - # when linked with other C++ libraries. - # E.g. for cudaPackages_11_8 we use gcc11 with gcc12's libstdc++ - # Cf. https://github.com/NixOS/nixpkgs/pull/218265 for context - backendStdenv = final.callPackage ../development/cuda-modules/backend-stdenv.nix { }; + genericManifestBuilderFn = + { + pname, + redistName, + manifests, + generalFixupFn ? { }, + indexedFixupFn ? { }, + libPath ? null, + }: + trivial.pipe ../development/cuda-modules/generic-builders/manifest.nix [ + ( + path: + final.callPackage path { + inherit + pname + redistName + manifests + libPath + ; + } + ) + (drv: drv.overrideAttrs generalFixupFn) + (drv: drv.overrideAttrs (indexedFixupFn.${pname} or { })) + ]; - # Loose packages - cudatoolkit = final.callPackage ../development/cuda-modules/cudatoolkit { }; - saxpy = final.callPackage ../development/cuda-modules/saxpy { }; - nccl = final.callPackage ../development/cuda-modules/nccl { }; - nccl-tests = final.callPackage ../development/cuda-modules/nccl-tests { }; - } - # Setup hooks - // builtins.import ../development/cuda-modules/setup-hooks { inherit (final) callPackage; } - # Redistributable packages - // ( - let - redistName = "cuda"; - inherit (evaluatedModules.config.${redistName}) manifests; + genericMultiplexBuilderFn = + pname: + (builtins.import ../development/cuda-modules/generic-builders/multiplex.nix { + inherit (final) callPackage; + inherit + cudaVersion + mkVersionedPackageName + hostPlatform + lib + config + flags + pname + ; + }); + in + # Basic things callPackage should have available + { + inherit lib pkgs; + inherit config cudaVersion flags; + inherit cudaVersionAtLeast cudaVersionOlder; + inherit cudaMajorVersion cudaMajorMinorVersion; - # NOTE: The attribute values need to be callPackage'd before being used. - fixupFns = builtins.import ../development/cuda-modules/cuda/fixups.nix; - # NOTE: fixupFn is good to go as-is. - metaFixupFn = builtins.import ../development/cuda-modules/cuda/meta-fixup.nix { - inherit lib manifests nullableOr; - }; + # TODO(@connorbaker): `cudaFlags` is an alias for `flags` which should be removed in the future. + cudaFlags = flags; - # Builder function which builds a single redist package for a given platform. - # buildRedistPackage : PackageName -> Derivation - buildRedistPackage = - pname: - genericManifestBuilderFn { - inherit (final) callPackage; - inherit manifests pname redistName; - overrideAttrsFns = [ - metaFixupFn - ] ++ lib.optionals (builtins.hasAttr pname fixupFns) [ (final.callPackage fixupFns.${pname} { }) ]; + # Maintain a reference to the final cudaPackages. + # Without this, if we use `final.callPackage` and a package accepts `cudaPackages` as an argument, + # it's provided with `cudaPackages` from the top-level scope, which is not what we want. We want to + # provide the `cudaPackages` from the final scope -- that is, the *current* scope. + cudaPackages = final; + } + # Loose packages + // { + cudatoolkit = final.callPackage ../development/cuda-modules/cudatoolkit { }; + saxpy = final.callPackage ../development/cuda-modules/saxpy { }; + nccl = final.callPackage ../development/cuda-modules/nccl { }; + nccl-tests = final.callPackage ../development/cuda-modules/nccl-tests { }; + # Exposed as cudaPackages.backendStdenv. + # This is what nvcc uses as a backend, + # and it has to be an officially supported one (e.g. gcc11 for cuda11). + # + # It, however, propagates current stdenv's libstdc++ to avoid "GLIBCXX_* not found errors" + # when linked with other C++ libraries. + # E.g. for cudaPackages_11_8 we use gcc11 with gcc12's libstdc++ + # Cf. https://github.com/NixOS/nixpkgs/pull/218265 for context + backendStdenv = final.callPackage ../development/cuda-modules/backend-stdenv.nix { }; + } + # Setup hooks + // callPackageOnAttrs (builtins.import ../development/cuda-modules/setup-hooks) + # Redistributable packages + // ( + let + redistName = "cuda"; + inherit (config.${redistName}) fixupFns; + manifests = + let + default = { + redistrib = { }; + feature = { }; }; + in + config.${redistName}.manifests.${cudaVersion} or default; + generalFixupFn = final.callPackage fixupFns.generalFixupFn { inherit manifests; }; + indexedFixupFn = callPackageOnAttrs (builtins.import fixupFns.indexedFixupFn); + buildRedistPackage = + pname: + genericManifestBuilderFn { + inherit + manifests + pname + redistName + generalFixupFn + indexedFixupFn + ; + }; + redistPackages = trivial.pipe manifests.feature [ + # Get all the package names + builtins.attrNames + # Build the redist packages + (trivial.flip attrsets.genAttrs buildRedistPackage) + ]; + in + redistPackages + ) + # CuTensor + // ( + let + # A release is supported if it has a libPath that matches our CUDA version for our platform. + # LibPath are not constant across the same release -- one platform may support fewer + # CUDA versions than another. + # redistArch :: Optional String + redistArch = flags.getRedistArch hostPlatform.system; + redistName = "cutensor"; + pname = "libcutensor"; + inherit (config.${redistName}) fixupFns; + # Our cudaVersion tells us which version of CUDA we're building against. + # The subdirectories in lib/ tell us which versions of CUDA are supported. + # Typically the names will look like this: + # + # - 10.2 + # - 11 + # - 11.0 + # - 12 + # libPath :: String + libPath = if cudaVersion == "10.2" then cudaVersion else cudaMajorVersion; + # Our build for cutensor is actually multiplexed -- we build a cutensor package for each + # version of CUDA that cutensor supports. + # We do this by filtering out the leaves of the manifest tree which don't contain the libPath we want. + # manifests :: { ${version} = { redistrib, feature }; } + manifests = + attrsets.filterAttrs + ( + _version: manifests: + lists.all trivial.id [ + # Platform must be supported + (redistArch != null) + ( + attrsets.attrByPath + [ + pname + redistArch + ] + null + manifests.feature != null + ) + ] + ) + config.${redistName}.manifests; - redistPackages = trivial.pipe manifests.feature [ - # Get all the package names - builtins.attrNames - # Build the redist packages - (trivial.flip attrsets.genAttrs buildRedistPackage) - ]; - in - redistPackages - ) - # CUDNN - // (builtins.import ../development/cuda-modules/generic-builders/multiplex.nix { - inherit (final) callPackage; - inherit - cudaVersion - flags - mkVersionedPackageName - hostPlatform - lib - evaluatedModules - ; - pname = "cudnn"; - shimsFn = ../development/cuda-modules/cudnn/shims.nix; - fixupFn = ../development/cuda-modules/cudnn/fixup.nix; - }) - ); - - mkVersionedPackageName = - name: version: - strings.concatStringsSep "_" [ - name - (strings.replaceStrings [ "." ] [ "_" ] (versions.majorMinor version)) - ]; - - composedExtension = - fixedPoints.composeManyExtensions - [ - # (callPackage ../development/cuda-modules/cutensor/extension.nix { - # inherit cudaVersion flags mkVersionedPackageName; - # }) - # (callPackage ../development/cuda-modules/generic-builders/multiplex.nix { - # inherit cudaVersion flags mkVersionedPackageName; - # pname = "tensorrt"; - # releasesModule = ../development/cuda-modules/tensorrt/releases.nix; - # shimsFn = ../development/cuda-modules/tensorrt/shims.nix; - # fixupFn = ../development/cuda-modules/tensorrt/fixup.nix; - # }) - # (callPackage ../test/cuda/cuda-samples/extension.nix { inherit cudaVersion; }) - # (callPackage ../test/cuda/cuda-library-samples/extension.nix { }) - ]; - - # cudaPackages = customisation.makeScope newScope ( - # fixedPoints.extends composedExtension passthruFunction - # ); + generalFixupFn = final.callPackage fixupFns.generalFixupFn { }; - cudaPackages = pkgs.makeScopeWithSplicing' { - otherSplices = pkgs.generateSplicesForMkScope "cudaPackages"; - f = fixedPoints.extends composedExtension passthruFunction; - # f = final: { - # # Recursive reference to the initial cudaPackages' cudaVersion. - # inherit (config.cudaPackages) cudaVersion; - # gpus = builtins.import ../gpus.nix; - # nvccCompatibilities = builtins.import ../nvcc-compatibilities.nix; - # cudaMajorVersion = versions.major final.cudaVersion; - # cudaMajorMinorVersion = versions.majorMinor final.cudaVersion; - # # flags = final.callPackage ../flags.nix { }; - # # Exposed as cudaPackages.backendStdenv. - # # This is what nvcc uses as a backend, - # # and it has to be an officially supported one (e.g. gcc11 for cuda11). - # # - # # It, however, propagates current stdenv's libstdc++ to avoid "GLIBCXX_* not found errors" - # # when linked with other C++ libraries. - # # E.g. for cudaPackages_11_8 we use gcc11 with gcc12's libstdc++ - # # Cf. https://github.com/NixOS/nixpkgs/pull/218265 for context - # backendStdenv = final.callPackage ../backend-stdenv.nix { }; - # }; - }; + redistPackages = + attrsets.mapAttrs' + (version: manifests: { + name = mkVersionedPackageName pname version; + value = genericManifestBuilderFn { + inherit + manifests + pname + redistName + generalFixupFn + libPath + ; + }; + }) + manifests; + in + redistPackages + // attrsets.optionalAttrs (redistPackages != { }) { + ${pname} = + let + # Get the newest version in our pruned manifests. + nameOfNewest = trivial.pipe manifests [ + # Get all the versions + builtins.attrNames + # Sort in descending order + (lists.sort (trivial.flip strings.versionOlder)) + # Get the first element + builtins.head + # Get the name of the newest version + (mkVersionedPackageName pname) + ]; + in + redistPackages.${nameOfNewest}; + } + ) + # CUDNN + // (genericMultiplexBuilderFn "cudnn") + # TensorRT + // (genericMultiplexBuilderFn "tensorrt"); in -cudaPackages // { inherit __attrsFailEvaluation; } +trivial.pipe config.versions [ + (builtins.map ( + cudaVersion: { + name = mkVersionedPackageName "cudaPackages" cudaVersion; + value = + lib.recurseIntoAttrs ( + makeScopeWithSplicing' { + otherSplices = generateSplicesForMkScope "cudaPackages"; + f = makeCudaPackages cudaVersion; + } + ) + // { + inherit __attrsFailEvaluation; + }; + } + )) + builtins.listToAttrs +] + +# TODO(@connorbaker): migrate tests away from extensions. +# composedExtension = +# fixedPoints.composeManyExtensions +# [ +# # (callPackage ../test/cuda/cuda-samples/extension.nix { inherit cudaVersion; }) +# # (callPackage ../test/cuda/cuda-library-samples/extension.nix { }) +# ]; From b8be36118696fa49d86a4b7b59fd4a64dc63db4f Mon Sep 17 00:00:00 2001 From: Connor Baker Date: Tue, 19 Dec 2023 17:30:51 +0000 Subject: [PATCH 3/5] cuda-modules: wip refactor for cross-compilation --- .../cuda-modules/modules/cuda/default.nix | 78 ++--------- .../cuda-modules/modules/cutensor/default.nix | 62 ++------- .../cuda-modules/modules/cutensor/fixup.nix | 1 + .../modules/generic/manifests/default.nix | 46 +++++++ pkgs/top-level/all-packages.nix | 7 +- pkgs/top-level/cuda-packages.nix | 121 ++++++++++++------ 6 files changed, 157 insertions(+), 158 deletions(-) diff --git a/pkgs/development/cuda-modules/modules/cuda/default.nix b/pkgs/development/cuda-modules/modules/cuda/default.nix index fdaee6ffa3e96..fbee0f4248d53 100644 --- a/pkgs/development/cuda-modules/modules/cuda/default.nix +++ b/pkgs/development/cuda-modules/modules/cuda/default.nix @@ -1,72 +1,18 @@ -{ config, lib, ... }: +{ config, lib, options, ... }: let - inherit (lib) - filesystem - options - types - ; + inherit (lib) filesystem modules; + opt = options.generic.manifests; cfg = config.generic.manifests; in { - options.cuda = { - # TODO(@connorbaker): Figure out how to move these general options into `generic.manifests.options` - # and re-use them here, ideally overriding only the default values to get the effects we want. - manifestPathParser = options.mkOption { - description = "Function to parse a manifest path into a manifest"; - default = cfg.utils.manifestPathToManifestMeta; - type = types.functionTo cfg.types.manifestMeta; - }; - manifestPaths = options.mkOption { - description = "List of paths to CUDA redistributable manifests"; - default = filesystem.listFilesRecursive ./manifests; - type = types.listOf types.path; - }; - manifestMetas = options.mkOption { - description = "List of meta information about CUDA redistributable manifests"; - default = builtins.map cfg.utils.manifestPathToManifestMeta config.cuda.manifestPaths; - type = types.listOf cfg.types.manifestMeta; - }; - manifestMetasToManifests = options.mkOption { - description = "Function to convert a list of manifest metas to a list of manifests"; - default = cfg.utils.manifestMetasToManifests; - type = types.functionTo cfg.types.manifests; - }; - manifests = options.mkOption { - description = "Mapping of manifest version (major and minor) to feature and redistributable manifests"; - default = cfg.utils.manifestMetasToManifests config.cuda.manifestMetas; - type = cfg.types.manifests; - }; - fixupFns = options.mkOption { - description = '' - Functions to pass to a derivation's overrideAttrs function to perform fixup after being callPackage'd - ''; - default = { }; - type = types.submodule { - options = { - generalFixupFn = options.mkOption { - description = '' - A general fixup applied to all redistributables's. Note that it requires `callPackage` to provide a - `manifests` argument. - NOTE: The value must be inspectable by `callPackage`. It seems that when functions are exposed via module - configurations, they do not preserve functionArgs, and so callPackage will fail because it cannot supply - arguments by default. - ''; - default = ./fixup.nix; - type = types.path; - }; - indexedFixupFn = options.mkOption { - description = '' - Attribute set of functions where each key is the `pname` of a redistributable and each value is a - function to fixup the derivation's attributes after being callPackage'd - NOTE: The value must be inspectable by `callPackage`. It seems that when functions are exposed via module - configurations, they do not preserve functionArgs, and so callPackage will fail because it cannot supply - arguments by default. - ''; - default = ./fixups.nix; - type = types.path; - }; - }; - }; - }; + options.cuda = opt.options; + config.cuda = { + manifestPaths = modules.mkDefault (filesystem.listFilesRecursive ./manifests); + manifestMetas = modules.mkDefault ( + builtins.map cfg.utils.manifestPathToManifestMeta config.cuda.manifestPaths + ); + manifests = modules.mkDefault (cfg.utils.manifestMetasToManifests config.cuda.manifestMetas); + generalFixupFn = modules.mkDefault ./fixup.nix; + indexedFixupFn = modules.mkDefault ./fixups.nix; }; } diff --git a/pkgs/development/cuda-modules/modules/cutensor/default.nix b/pkgs/development/cuda-modules/modules/cutensor/default.nix index 5b997472323be..c5efffcfce008 100644 --- a/pkgs/development/cuda-modules/modules/cutensor/default.nix +++ b/pkgs/development/cuda-modules/modules/cutensor/default.nix @@ -1,57 +1,17 @@ -{ config, lib, ... }: +{ config, lib, options, ... }: let - inherit (lib) filesystem options types; + inherit (lib) filesystem modules; + opt = options.generic.manifests; cfg = config.generic.manifests; in { - options.cutensor = { - # TODO(@connorbaker): Figure out how to move these general options into `generic.manifests.options` - # and re-use them here, ideally overriding only the default values to get the effects we want. - manifestPathParser = options.mkOption { - description = "Function to parse a manifest path into a manifest"; - default = cfg.utils.manifestPathToManifestMeta; - type = types.functionTo cfg.types.manifestMeta; - }; - manifestPaths = options.mkOption { - description = "List of paths to CUDA redistributable manifests"; - default = filesystem.listFilesRecursive ./manifests; - type = types.listOf types.path; - }; - manifestMetas = options.mkOption { - description = "List of meta information about CUDA redistributable manifests"; - default = builtins.map cfg.utils.manifestPathToManifestMeta config.cutensor.manifestPaths; - type = types.listOf cfg.types.manifestMeta; - }; - manifestMetasToManifests = options.mkOption { - description = "Function to convert a list of manifest metas to a list of manifests"; - default = cfg.utils.manifestMetasToManifests; - type = types.functionTo cfg.types.manifests; - }; - manifests = options.mkOption { - description = "Mapping of manifest version (major and minor) to feature and redistributable manifests"; - default = cfg.utils.manifestMetasToManifests config.cutensor.manifestMetas; - type = cfg.types.manifests; - }; - fixupFns = options.mkOption { - description = '' - Functions to pass to a derivation's overrideAttrs function to perform fixup after being callPackage'd - ''; - default = { }; - type = types.submodule { - options = { - generalFixupFn = options.mkOption { - description = '' - A general fixup applied to all redistributables's. - - NOTE: The value must be inspectable by `callPackage`. It seems that when functions are exposed via module - configurations, they do not preserve functionArgs, and so callPackage will fail because it cannot supply - arguments by default. - ''; - default = ./fixup.nix; - type = types.path; - }; - }; - }; - }; + options.cutensor = opt.options; + config.cutensor = { + manifestPaths = modules.mkDefault (filesystem.listFilesRecursive ./manifests); + manifestMetas = modules.mkDefault ( + builtins.map cfg.utils.manifestPathToManifestMeta config.cutensor.manifestPaths + ); + manifests = modules.mkDefault (cfg.utils.manifestMetasToManifests config.cutensor.manifestMetas); + generalFixupFn = modules.mkDefault ./fixup.nix; }; } diff --git a/pkgs/development/cuda-modules/modules/cutensor/fixup.nix b/pkgs/development/cuda-modules/modules/cutensor/fixup.nix index ae57a016e520c..b814483f35460 100644 --- a/pkgs/development/cuda-modules/modules/cutensor/fixup.nix +++ b/pkgs/development/cuda-modules/modules/cutensor/fixup.nix @@ -4,6 +4,7 @@ cudaVersionAtLeast, cudaPackages, lib, + ... }: let inherit (lib) lists strings; diff --git a/pkgs/development/cuda-modules/modules/generic/manifests/default.nix b/pkgs/development/cuda-modules/modules/generic/manifests/default.nix index 895ae11f581da..2d839badf0cb5 100644 --- a/pkgs/development/cuda-modules/modules/generic/manifests/default.nix +++ b/pkgs/development/cuda-modules/modules/generic/manifests/default.nix @@ -64,6 +64,52 @@ in { }; }; }; + options = { + manifestPathParser = options.mkOption { + description = "Function to parse a manifest path into a manifest"; + default = config.generic.manifests.utils.manifestPathToManifestMeta; + type = types.functionTo config.generic.manifests.types.manifestMeta; + }; + manifestPaths = options.mkOption { + description = "List of paths to CUDA redistributable manifests"; + type = types.listOf types.path; + }; + manifestMetas = options.mkOption { + description = "List of meta information about CUDA redistributable manifests"; + type = types.listOf config.generic.manifests.types.manifestMeta; + }; + manifestMetasToManifests = options.mkOption { + description = "Function to convert a list of manifest metas to a list of manifests"; + default = config.generic.manifests.utils.manifestMetasToManifests; + type = types.functionTo config.generic.manifests.types.manifests; + }; + manifests = options.mkOption { + description = "Mapping of manifest version (major and minor) to feature and redistributable manifests"; + type = config.generic.manifests.types.manifests; + }; + generalFixupFn = options.mkOption { + description = '' + A general fixup applied to all redistributables's. Note that it requires `callPackage` to provide a + `manifests` argument. + NOTE: The value must be inspectable by `callPackage`. It seems that when functions are exposed via module + configurations, they do not preserve functionArgs, and so callPackage will fail because it cannot supply + arguments by default. + ''; + default = _: {}; + type = types.oneOf [types.path (types.functionTo types.attrs)]; + }; + indexedFixupFn = options.mkOption { + description = '' + Attribute set of functions where each key is the `pname` of a redistributable and each value is a + function to fixup the derivation's attributes after being callPackage'd + NOTE: The value must be inspectable by `callPackage`. It seems that when functions are exposed via module + configurations, they do not preserve functionArgs, and so callPackage will fail because it cannot supply + arguments by default. + ''; + default = {}; + type = types.oneOf [types.path types.attrs]; + }; + }; types = options.mkOption { description = "A set of generic types."; type = types.attrsOf types.optionType; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 7bb6251b371a7..b48c9b2383f41 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -7332,6 +7332,7 @@ with pkgs; snooze = callPackage ../tools/system/snooze { }; + # recurseIntoAttrs is applied inside cuda-packages.nix. inherit (callPackage ./cuda-packages.nix {}) # CUDA Packages 10.x cudaPackages_10_0 @@ -7356,9 +7357,9 @@ with pkgs; cudaPackages_12_3 ; - cudaPackages_10 = recurseIntoAttrs cudaPackages_10_2; - cudaPackages_11 = recurseIntoAttrs cudaPackages_11_8; - cudaPackages_12 = recurseIntoAttrs cudaPackages_12_0; + cudaPackages_10 = cudaPackages_10_2; + cudaPackages_11 = cudaPackages_11_8; + cudaPackages_12 = cudaPackages_12_0; # Use the older cudaPackages for tensorflow and jax, as determined by cudnn # compatibility: https://www.tensorflow.org/install/source#gpu diff --git a/pkgs/top-level/cuda-packages.nix b/pkgs/top-level/cuda-packages.nix index 7f85829e84cbf..f0f17dfc7f8e4 100644 --- a/pkgs/top-level/cuda-packages.nix +++ b/pkgs/top-level/cuda-packages.nix @@ -38,19 +38,18 @@ let versions ; - inherit - (modules.evalModules { - modules = [ - { - inherit (pkgs.config) cudaSupport; - cudaCapabilities = pkgs.config.cudaCapabilities or [ ]; - cudaForwardCompat = pkgs.config.cudaForwardCompat or true; - } - ../development/cuda-modules/modules - ]; - }) - config - ; + # cuda-modules contains all the information we need to build our packages. + cuda-modules = modules.evalModules { + modules = [ + { + inherit (pkgs.config) cudaSupport; + cudaCapabilities = pkgs.config.cudaCapabilities or [ ]; + cudaForwardCompat = pkgs.config.cudaForwardCompat or true; + } + ../development/cuda-modules/modules + ]; + }; + config = attrsets.dontRecurseIntoAttrs cuda-modules.config; mkVersionedPackageName = name: version: @@ -59,13 +58,10 @@ let (strings.replaceStrings [ "." ] [ "_" ] (versions.majorMinor version)) ]; + # Create a fixed-point for makeScopeWithSplicing' parameterized by cudaVersion. makeCudaPackages = - cudaVersion: final: + cudaVersion: let - # NOTE: Use of `final.callPackage` for `callPackageOnAttrs` doesn't run the risk of infinite recursion, - # as the `final` argument is not used to compute the attribute names. - callPackageOnAttrs = attrsets.mapAttrs (_: value: final.callPackage value { }); - # Flags used to enable different features of cudaPackages -- we cannot use final.callPackage # because we use `flags` to determine the presence of certain packages, which would cause # infinite recursion. @@ -87,17 +83,44 @@ let cudaVersionOlder = strings.versionOlder cudaVersion; # cudaVersionAtLeast : Version -> Boolean cudaVersionAtLeast = strings.versionAtLeast cudaVersion; + in + final: + let + # NOTE: Use of `final.callPackage` for `callPackageOnAttrs` doesn't run the risk of infinite recursion, + # as the `final` argument is not used to compute the attribute names. + callPackageOnAttrs = attrsets.mapAttrs (_: value: final.callPackage value { }); + + # Helper function to get the general fixup function for a redistributable package. + getGeneralFixupFn = + { + redistName, + args ? { }, + }: + final.callPackage config.${redistName}.generalFixupFn args; + + # Helper function to get the indexed fixup function for a redistributable package. + getIndexedFixupFn = + { redistName }: + trivial.pipe config.${redistName}.indexedFixupFn [ + # If it's a path, we need to import it before passing it along. + # The default value is an empty attrset so we don't need to import it. + (iff: if builtins.isPath iff then builtins.import iff else iff) + callPackageOnAttrs + ]; + + # Helper function which wraps the generic manifest builder. genericManifestBuilderFn = { pname, redistName, manifests, - generalFixupFn ? { }, - indexedFixupFn ? { }, + generalFixupFn, + indexedFixupFn, libPath ? null, }: trivial.pipe ../development/cuda-modules/generic-builders/manifest.nix [ + # Build the package ( path: final.callPackage path { @@ -109,10 +132,13 @@ let ; } ) + # General package fixup (drv: drv.overrideAttrs generalFixupFn) - (drv: drv.overrideAttrs (indexedFixupFn.${pname} or { })) + # Package-specific fixup if it exists + (drv: drv.overrideAttrs (attrsets.attrByPath [ pname ] { } indexedFixupFn)) ]; + # Helper function which wraps the generic multiplex builder. genericMultiplexBuilderFn = pname: (builtins.import ../development/cuda-modules/generic-builders/multiplex.nix { @@ -166,17 +192,31 @@ let // ( let redistName = "cuda"; - inherit (config.${redistName}) fixupFns; + + # Retrieve the manifests for our cudaVersion. + # We unconditionally access manifests.feature so we need to make sure it exists (even if it is empty). + # manifests :: { redistrib, feature } manifests = - let - default = { + attrsets.attrByPath + [ + redistName + "manifests" + cudaVersion + ] + ({ redistrib = { }; feature = { }; - }; - in - config.${redistName}.manifests.${cudaVersion} or default; - generalFixupFn = final.callPackage fixupFns.generalFixupFn { inherit manifests; }; - indexedFixupFn = callPackageOnAttrs (builtins.import fixupFns.indexedFixupFn); + }) + config; + + # Retrieve our fixup functions. + generalFixupFn = getGeneralFixupFn { + inherit redistName; + args = {inherit manifests; }; + }; + indexedFixupFn = getIndexedFixupFn { inherit redistName; }; + + # Build a single redistributable package. buildRedistPackage = pname: genericManifestBuilderFn { @@ -188,6 +228,9 @@ let indexedFixupFn ; }; + + # Map our buildRedistPackage function over all the packages in manifests.feature. + # This is an empty attrset if manifests.feature is empty, which is what we want. redistPackages = trivial.pipe manifests.feature [ # Get all the package names builtins.attrNames @@ -200,14 +243,11 @@ let # CuTensor // ( let - # A release is supported if it has a libPath that matches our CUDA version for our platform. - # LibPath are not constant across the same release -- one platform may support fewer - # CUDA versions than another. # redistArch :: Optional String redistArch = flags.getRedistArch hostPlatform.system; redistName = "cutensor"; pname = "libcutensor"; - inherit (config.${redistName}) fixupFns; + # Our cudaVersion tells us which version of CUDA we're building against. # The subdirectories in lib/ tell us which versions of CUDA are supported. # Typically the names will look like this: @@ -218,9 +258,10 @@ let # - 12 # libPath :: String libPath = if cudaVersion == "10.2" then cudaVersion else cudaMajorVersion; - # Our build for cutensor is actually multiplexed -- we build a cutensor package for each - # version of CUDA that cutensor supports. - # We do this by filtering out the leaves of the manifest tree which don't contain the libPath we want. + + # Our build for cutensor is actually multiplexed -- we build a cutensor package for each version of CUDA that + # cutensor supports. Currently, we don't know ahead of time what the contents of the lib directory are for + # each package, so building these is best-effort. # manifests :: { ${version} = { redistrib, feature }; } manifests = attrsets.filterAttrs @@ -242,7 +283,9 @@ let ) config.${redistName}.manifests; - generalFixupFn = final.callPackage fixupFns.generalFixupFn { }; + # Retrieve our fixup functions. + generalFixupFn = getGeneralFixupFn { inherit redistName; }; + indexedFixupFn = getIndexedFixupFn { inherit redistName; }; redistPackages = attrsets.mapAttrs' @@ -254,6 +297,7 @@ let pname redistName generalFixupFn + indexedFixupFn libPath ; }; @@ -289,7 +333,7 @@ trivial.pipe config.versions [ cudaVersion: { name = mkVersionedPackageName "cudaPackages" cudaVersion; value = - lib.recurseIntoAttrs ( + attrsets.recurseIntoAttrs ( makeScopeWithSplicing' { otherSplices = generateSplicesForMkScope "cudaPackages"; f = makeCudaPackages cudaVersion; @@ -300,6 +344,7 @@ trivial.pipe config.versions [ }; } )) + # Make sure not to recurse into config builtins.listToAttrs ] From 49953cc4885a993cbbaf7a19725e74adf26b385d Mon Sep 17 00:00:00 2001 From: Connor Baker Date: Wed, 20 Dec 2023 03:44:33 +0000 Subject: [PATCH 4/5] cuda-modules: wip refactor for cross-compilation --- .../cuda-modules/modules/cuda/fixup.nix | 4 +- .../modules/generic/manifests/default.nix | 2 +- pkgs/top-level/cuda-packages.nix | 333 +++++++++--------- 3 files changed, 160 insertions(+), 179 deletions(-) diff --git a/pkgs/development/cuda-modules/modules/cuda/fixup.nix b/pkgs/development/cuda-modules/modules/cuda/fixup.nix index 004433b89cfc3..ad1383577f10a 100644 --- a/pkgs/development/cuda-modules/modules/cuda/fixup.nix +++ b/pkgs/development/cuda-modules/modules/cuda/fixup.nix @@ -1,5 +1,5 @@ # A function which when callPackage'd returns a function to be given to overrideAttrs. -{ lib, manifests, ... }: +{ config, redistName, version, lib, ... }: prevAttrs: prevAttrs // { @@ -9,7 +9,7 @@ prevAttrs url = let default = "${prevAttrs.pname}/LICENSE.txt"; - nullableDesired = manifests.redistrib.${prevAttrs.pname}.license_path; + nullableDesired = config.${redistName}.${version}.redistrib.${prevAttrs.pname}.license_path; in "https://developer.download.nvidia.com/compute/cuda/redist/" + (if nullableDesired != null then nullableDesired else default); diff --git a/pkgs/development/cuda-modules/modules/generic/manifests/default.nix b/pkgs/development/cuda-modules/modules/generic/manifests/default.nix index 2d839badf0cb5..417fdd4ce71a5 100644 --- a/pkgs/development/cuda-modules/modules/generic/manifests/default.nix +++ b/pkgs/development/cuda-modules/modules/generic/manifests/default.nix @@ -95,7 +95,7 @@ in configurations, they do not preserve functionArgs, and so callPackage will fail because it cannot supply arguments by default. ''; - default = _: {}; + default = {}: {}; type = types.oneOf [types.path (types.functionTo types.attrs)]; }; indexedFixupFn = options.mkOption { diff --git a/pkgs/top-level/cuda-packages.nix b/pkgs/top-level/cuda-packages.nix index f0f17dfc7f8e4..b113785605e90 100644 --- a/pkgs/top-level/cuda-packages.nix +++ b/pkgs/top-level/cuda-packages.nix @@ -42,6 +42,9 @@ let cuda-modules = modules.evalModules { modules = [ { + # Apply any user-specified overrides. + imports = pkgs.config.extraCudaModules or [ ]; + inherit (pkgs.config) cudaSupport; cudaCapabilities = pkgs.config.cudaCapabilities or [ ]; cudaForwardCompat = pkgs.config.cudaForwardCompat or true; @@ -91,53 +94,152 @@ let # as the `final` argument is not used to compute the attribute names. callPackageOnAttrs = attrsets.mapAttrs (_: value: final.callPackage value { }); - # Helper function to get the general fixup function for a redistributable package. - getGeneralFixupFn = - { - redistName, - args ? { }, - }: - final.callPackage config.${redistName}.generalFixupFn args; - - # Helper function to get the indexed fixup function for a redistributable package. - getIndexedFixupFn = - { redistName }: - trivial.pipe config.${redistName}.indexedFixupFn [ - # If it's a path, we need to import it before passing it along. - # The default value is an empty attrset so we don't need to import it. - (iff: if builtins.isPath iff then builtins.import iff else iff) - callPackageOnAttrs - ]; - # Helper function which wraps the generic manifest builder. + # Manifests have the shape { ${version} = { redistrib, feature }; } + # This builder will create an attribute set containing packages from each pair of redistributable and feature + # manifests for a given redistName. + # + # If `version` is non-null, only the package set for that version will be returned. No suffix will be appended. + # + # Otherwise, package sets for each version will be suffixed with the version and unioned with each other and the + # package set from the newest version of the manifest, which will not be suffixed. + # This means that packages from the newest manifest are available both with and without a suffix, while packages + # from older manifests are only available with a suffix. genericManifestBuilderFn = { - pname, redistName, - manifests, - generalFixupFn, - indexedFixupFn, - libPath ? null, + useLibPath, + # If version is specified, only build the redistributable package for that version. + version ? null, }: - trivial.pipe ../development/cuda-modules/generic-builders/manifest.nix [ - # Build the package - ( - path: - final.callPackage path { - inherit - pname - redistName - manifests - libPath - ; - } - ) - # General package fixup - (drv: drv.overrideAttrs generalFixupFn) - # Package-specific fixup if it exists - (drv: drv.overrideAttrs (attrsets.attrByPath [ pname ] { } indexedFixupFn)) - ]; + let + # TODO(@connorbaker): Update the modules so it's named versionedManifests. + # Retrieve all versions of the manifests available for our redistName. + # versionedManifests :: { ${version} = { redistrib, feature }; } + versionedManifests = + let + allVersionedManifests = + attrsets.attrByPath + [ + redistName + "manifests" + ] + { } + config; + in + if version != null then + # If version is specified, only build the redistributable package for that version. + # If we're specifying the version explicitly, it is assumed that it exists. + # It will be bother the only and the nested manifest. + { "${version}" = allVersionedManifests.${version}; } + else + allVersionedManifests; + + # Our cudaVersion tells us which version of CUDA we're building against. + # The subdirectories in lib/ tell us which versions of CUDA are supported. + # Typically the names will look like this: + # + # - 10.2 + # - 11 + # - 11.0 + # - 12 + # + # Not every package uses this layout, but we can precompute it here. + # libPath :: String + libPath = + if useLibPath then if cudaVersion == "10.2" then cudaVersion else cudaMajorVersion else null; + + # Build a redistributable package given the version and corresponding manifest. + buildPackageSetFromVersionedManifest = + let + # Retrieve our fixup functions which do not rely on the version of the manifest being processed. + indexedFixupFn = trivial.pipe config.${redistName}.indexedFixupFn [ + # If it's a path, we need to import it before passing it along. + # The default value is an empty attrset so we don't need to import it. + (maybePath: if builtins.isPath maybePath then builtins.import maybePath else maybePath) + # Use callPackage on the values in the attrset. + callPackageOnAttrs + ]; + in + # version :: String + version: + let + # Retrieve our indexedFixupFn, which does not rely on the version of the manifest being processed. + generalFixupFn = final.callPackage config.${redistName}.generalFixupFn { + inherit redistName version; + }; + in + # manifests :: { redistrib, feature } + manifests: + + # Map over the attribute names of the feature manifest, which contain only package names. + attrsets.genAttrs (builtins.attrNames manifests.feature) ( + # pname :: String + pname: + trivial.pipe pname [ + # Build the package + ( + pname: + final.callPackage ../development/cuda-modules/generic-builders/manifest.nix { + inherit + pname + redistName + manifests + libPath + ; + } + ) + # General package fixup + (drv: drv.overrideAttrs generalFixupFn) + # Package-specific fixup if it exists + (drv: drv.overrideAttrs (attrsets.attrByPath [ pname ] { } indexedFixupFn)) + ] + ); + # For each version in our manifests, build a package set. + # Do not rename packages yet; that's handled later. + versionedPackageSets = attrsets.mapAttrs buildPackageSetFromVersionedManifest versionedManifests; + + # Fold over any remaining package sets and append a suffix to the package names. + flattenedVersionedSuffixedPackageSets = + attrsets.concatMapAttrs + ( + # version :: String + version: + # packages :: { ${pname} = drv; } + packages: + attrsets.mapAttrs' + (pname: drv: { + name = mkVersionedPackageName pname version; + value = drv; + }) + packages + ) + versionedPackageSets; + in + trivial.throwIf (versionedPackageSets == { }) + '' + No manifests found for ${redistName}. + Please check that there are in fact manifest files present and that there are not filtered out by a version + check or (within the module evaluation) by an incorrect manifest filename (which would case the regex to + fail to match). + '' + ( + let + # Since versionedPackageSets is non-empty, we can safely assume that newestToOldestVersion is non-empty. + newestToOldestVersion = lists.sort (trivial.flip strings.versionOlder) ( + builtins.attrNames versionedManifests + ); + newestVersion = builtins.head newestToOldestVersion; + newestPackageSet = versionedPackageSets.${newestVersion}; + in + if version != null then + # If version is non-null, just return the newest package set. + newestPackageSet + else + # Otherwise, return the flattened package set unioned with the default package set (the newest). + newestPackageSet // flattenedVersionedSuffixedPackageSets + ); # Helper function which wraps the generic multiplex builder. genericMultiplexBuilderFn = pname: @@ -189,144 +291,23 @@ let # Setup hooks // callPackageOnAttrs (builtins.import ../development/cuda-modules/setup-hooks) # Redistributable packages - // ( - let - redistName = "cuda"; - - # Retrieve the manifests for our cudaVersion. - # We unconditionally access manifests.feature so we need to make sure it exists (even if it is empty). - # manifests :: { redistrib, feature } - manifests = - attrsets.attrByPath - [ - redistName - "manifests" - cudaVersion - ] - ({ - redistrib = { }; - feature = { }; - }) - config; - - # Retrieve our fixup functions. - generalFixupFn = getGeneralFixupFn { - inherit redistName; - args = {inherit manifests; }; - }; - indexedFixupFn = getIndexedFixupFn { inherit redistName; }; - - # Build a single redistributable package. - buildRedistPackage = - pname: - genericManifestBuilderFn { - inherit - manifests - pname - redistName - generalFixupFn - indexedFixupFn - ; - }; - - # Map our buildRedistPackage function over all the packages in manifests.feature. - # This is an empty attrset if manifests.feature is empty, which is what we want. - redistPackages = trivial.pipe manifests.feature [ - # Get all the package names - builtins.attrNames - # Build the redist packages - (trivial.flip attrsets.genAttrs buildRedistPackage) - ]; - in - redistPackages - ) + // genericManifestBuilderFn { + redistName = "cuda"; + useLibPath = false; + version = cudaVersion; + } # CuTensor - // ( - let - # redistArch :: Optional String - redistArch = flags.getRedistArch hostPlatform.system; - redistName = "cutensor"; - pname = "libcutensor"; - - # Our cudaVersion tells us which version of CUDA we're building against. - # The subdirectories in lib/ tell us which versions of CUDA are supported. - # Typically the names will look like this: - # - # - 10.2 - # - 11 - # - 11.0 - # - 12 - # libPath :: String - libPath = if cudaVersion == "10.2" then cudaVersion else cudaMajorVersion; - - # Our build for cutensor is actually multiplexed -- we build a cutensor package for each version of CUDA that - # cutensor supports. Currently, we don't know ahead of time what the contents of the lib directory are for - # each package, so building these is best-effort. - # manifests :: { ${version} = { redistrib, feature }; } - manifests = - attrsets.filterAttrs - ( - _version: manifests: - lists.all trivial.id [ - # Platform must be supported - (redistArch != null) - ( - attrsets.attrByPath - [ - pname - redistArch - ] - null - manifests.feature != null - ) - ] - ) - config.${redistName}.manifests; - - # Retrieve our fixup functions. - generalFixupFn = getGeneralFixupFn { inherit redistName; }; - indexedFixupFn = getIndexedFixupFn { inherit redistName; }; - - redistPackages = - attrsets.mapAttrs' - (version: manifests: { - name = mkVersionedPackageName pname version; - value = genericManifestBuilderFn { - inherit - manifests - pname - redistName - generalFixupFn - indexedFixupFn - libPath - ; - }; - }) - manifests; - in - redistPackages - // attrsets.optionalAttrs (redistPackages != { }) { - ${pname} = - let - # Get the newest version in our pruned manifests. - nameOfNewest = trivial.pipe manifests [ - # Get all the versions - builtins.attrNames - # Sort in descending order - (lists.sort (trivial.flip strings.versionOlder)) - # Get the first element - builtins.head - # Get the name of the newest version - (mkVersionedPackageName pname) - ]; - in - redistPackages.${nameOfNewest}; - } - ) + # Our build for cutensor is actually multiplexed -- we build a cutensor package for each version of CUDA that + # cutensor supports. Currently, we don't know ahead of time what the contents of the lib directory are for + # each package, so building these is best-effort. + // genericManifestBuilderFn { + redistName = "cutensor"; + useLibPath = true; + } # CUDNN - // (genericMultiplexBuilderFn "cudnn") + // genericMultiplexBuilderFn "cudnn" # TensorRT - // (genericMultiplexBuilderFn "tensorrt"); + // genericMultiplexBuilderFn "tensorrt"; in trivial.pipe config.versions [ (builtins.map ( From 91f763407f3071baab8194b54a7f0accd3718d23 Mon Sep 17 00:00:00 2001 From: Connor Baker Date: Wed, 20 Dec 2023 04:05:16 +0000 Subject: [PATCH 5/5] cuda-modules: wip refactor for cross-compilation --- pkgs/development/cuda-modules/flags.nix | 60 +++++++------------ .../generic-builders/manifest.nix | 50 ++++++++++------ .../generic-builders/multiplex.nix | 2 +- .../development/cuda-modules/nccl/default.nix | 3 + 4 files changed, 57 insertions(+), 58 deletions(-) diff --git a/pkgs/development/cuda-modules/flags.nix b/pkgs/development/cuda-modules/flags.nix index 3e90c8ac2b86c..2f45c76c6985f 100644 --- a/pkgs/development/cuda-modules/flags.nix +++ b/pkgs/development/cuda-modules/flags.nix @@ -103,44 +103,28 @@ let "-gencode=arch=compute_${dropDot computeCapability},code=${feat}_${dropDot computeCapability}" ); - # Maps Nix system to NVIDIA redist arch. - # NOTE: We swap out the default `linux-sbsa` redist (for server-grade ARM chips) with the - # `linux-aarch64` redist (which is for Jetson devices) if we're building any Jetson devices. - # Since both are based on aarch64, we can only have one or the other, otherwise there's an - # ambiguity as to which should be used. - # getRedistArch :: String -> String - getRedistArch = - nixSystem: - if nixSystem == "aarch64-linux" then - if jetsonTargets != [] then "linux-aarch64" else "linux-sbsa" - else if nixSystem == "x86_64-linux" then - "linux-x86_64" - else if nixSystem == "ppc64le-linux" then - "linux-ppc64le" - else if nixSystem == "x86_64-windows" then - "windows-x86_64" - else - builtins.throw "Unsupported Nix system: ${nixSystem}"; - - # Maps NVIDIA redist arch to Nix system. - # It is imperative that we include the boolean condition based on jetsonTargets to ensure - # we don't advertise availability of packages only available on server-grade ARM - # as being available for the Jetson, since both `linux-sbsa` and `linux-aarch64` are - # mapped to the Nix system `aarch64-linux`. - getNixSystem = - redistArch: - if redistArch == "linux-sbsa" && jetsonTargets == [] then - "aarch64-linux" - else if redistArch == "linux-aarch64" && jetsonTargets != [] then - "aarch64-linux" - else if redistArch == "linux-x86_64" then - "x86_64-linux" - else if redistArch == "linux-ppc64le" then - "ppc64le-linux" - else if redistArch == "windows-x86_64" then - "x86_64-windows" - else - builtins.throw "Unsupported NVIDIA redist arch: ${redistArch}"; + # NOTE: This function *will* be called by unsupported systems because `cudaPackages` is part of + # `all-packages.nix`, which is evaluated on all systems. As such, we need to handle unsupported + # systems gracefully. + # getRedistArch :: String -> Optional String + getRedistArch = nixSystem: attrsets.attrByPath [ nixSystem ] null { + aarch64-linux = if jetsonTargets != [] then "linux-aarch64" else "linux-sbsa"; + x86_64-linux = "linux-x86_64"; + ppc64le-linux = "linux-ppc64le"; + x86_64-windows = "windows-x86_64"; + }; + + # NOTE: This function *will* be called by unsupported systems because `cudaPackages` is part of + # `all-packages.nix`, which is evaluated on all systems. As such, we need to handle unsupported + # systems gracefully. + # getNixSystem :: String -> Optional String + getNixSystem = redistArch: attrsets.attrByPath [ redistArch ] null { + linux-sbsa = "aarch64-linux"; + linux-aarch64 = "aarch64-linux"; + linux-x86_64 = "x86_64-linux"; + linux-ppc64le = "ppc64le-linux"; + windows-x86_64 = "x86_64-windows"; + }; formatCapabilities = { diff --git a/pkgs/development/cuda-modules/generic-builders/manifest.nix b/pkgs/development/cuda-modules/generic-builders/manifest.nix index 33fe02ec51c05..917751d5c2bdd 100644 --- a/pkgs/development/cuda-modules/generic-builders/manifest.nix +++ b/pkgs/development/cuda-modules/generic-builders/manifest.nix @@ -45,6 +45,7 @@ let # Get the redist architectures for which package provides distributables. # These are used by meta.platforms. supportedRedistArchs = builtins.attrNames featureRelease; + # redistArch :: Optional String redistArch = flags.getRedistArch hostPlatform.system; in backendStdenv.mkDerivation ( @@ -65,7 +66,8 @@ backendStdenv.mkDerivation ( strictDeps = true; # NOTE: Outputs are evaluated jointly with meta, so in the case that this is an unsupported platform, - # we still need to provide a list of outputs. + # we still need to provide a list of outputs. The top-level binding is wrapped in `lists.optionals` + # to avoid evaluating attrByPath with a potentially null redistArch. outputs = let # Checks whether the redistributable provides an output. @@ -90,7 +92,7 @@ backendStdenv.mkDerivation ( "python" ]; # The out output is special -- it's the default output and we always include it. - outputs = ["out"] ++ additionalOutputs; + outputs = ["out"] ++ lists.optionals (redistArch != null) additionalOutputs; in outputs; @@ -110,15 +112,27 @@ backendStdenv.mkDerivation ( # Useful for introspecting why something went wrong. # Maps descriptions of why the derivation would be marked broken to # booleans indicating whether that description is true. - brokenConditions = {}; - - src = fetchurl { - url = "https://developer.download.nvidia.com/compute/${redistName}/redist/${ - redistribRelease.${redistArch}.relative_path - }"; - inherit (redistribRelease.${redistArch}) sha256; + brokenConditions = { + # Using an unrecognized redistArch + "Unrecognized NixOS platform ${hostPlatform.system}" = redistArch == null; + # Trying to build for a platform that doesn't have a redistributable + "Unsupported NixOS platform (or configuration) ${hostPlatform.system}" = finalAttrs.src == null; }; + src = trivial.pipe redistArch [ + # If redistArch is null, return null. Otherwise, get the redistributable for that architecture, returning null + # if it doesn't exist. + (trivial.mapNullable (redistArch: redistribRelease.${redistArch} or null)) + # If the release is non-null, fetch the source. + (trivial.mapNullable ( + { relative_path, sha256, ... }: + fetchurl { + url = "https://developer.download.nvidia.com/compute/${redistName}/redist/${relative_path}"; + inherit sha256; + } + )) + ]; + # We do need some other phases, like configurePhase, so the multiple-output setup hook works. dontBuild = true; @@ -246,16 +260,14 @@ backendStdenv.mkDerivation ( meta = { description = "${redistribRelease.name}. By downloading and using the packages you accept the terms and conditions of the ${finalAttrs.meta.license.shortName}"; sourceProvenance = [sourceTypes.binaryNativeCode]; - platforms = - lists.concatMap - ( - redistArch: - let - nixSystem = builtins.tryEval (flags.getNixSystem redistArch); - in - if nixSystem.success then [nixSystem.value] else [] - ) - supportedRedistArchs; + platforms = trivial.pipe supportedRedistArchs [ + # Map each redist arch to the equivalent nix system or null if there is no equivalent. + (trivial.flip attrsets.genAttrs flags.getNixSystem) + # Filter out entries which do not have a corresponding nix system + (attrsets.filterAttrs (_: value: value != null)) + # Get the remaining nix systems + attrsets.attrValues + ]; broken = lists.any trivial.id (attrsets.attrValues finalAttrs.brokenConditions); license = licenses.unfree; maintainers = teams.cuda.members; diff --git a/pkgs/development/cuda-modules/generic-builders/multiplex.nix b/pkgs/development/cuda-modules/generic-builders/multiplex.nix index d931a3cfaa3a8..dbd2898f8f7d2 100644 --- a/pkgs/development/cuda-modules/generic-builders/multiplex.nix +++ b/pkgs/development/cuda-modules/generic-builders/multiplex.nix @@ -15,7 +15,6 @@ let inherit (lib) attrsets lists - modules strings ; cfg = config.${pname}; @@ -37,6 +36,7 @@ let && strings.versionAtLeast package.maxCudaVersion cudaVersion; # Get all of the packages for our given platform. + # redistArch :: Optional String redistArch = flags.getRedistArch hostPlatform.system; # All the supported packages we can build for our platform. diff --git a/pkgs/development/cuda-modules/nccl/default.nix b/pkgs/development/cuda-modules/nccl/default.nix index c56d59cb42068..6e385688d0f8d 100644 --- a/pkgs/development/cuda-modules/nccl/default.nix +++ b/pkgs/development/cuda-modules/nccl/default.nix @@ -100,6 +100,9 @@ backendStdenv.mkDerivation ( homepage = "https://developer.nvidia.com/nccl"; license = licenses.bsd3; platforms = platforms.linux; + # NCCL is not supported on Jetson, because it does not use NVLink or PCI-e for inter-GPU communication. + # https://forums.developer.nvidia.com/t/can-jetson-orin-support-nccl/232845/9 + badPlatforms = lib.optionals cudaFlags.isJetsonBuild [ "aarch64-linux" ]; maintainers = with maintainers; [