Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
119 changes: 80 additions & 39 deletions pkgs/development/libraries/science/math/cudnn/default.nix
Original file line number Diff line number Diff line change
@@ -1,77 +1,118 @@
# The following version combinations are supported:
# * cuDNN 7.4.2, cudatoolkit 10.0
# * cuDNN 7.6.5, cudatoolkit 10.2
# * cuDNN 8.1.1, cudatoolkit 11.0-11.2
# * cuDNN 8.3.0, cudatoolkit 11.0-11.5
# * cuDNN 7.6.5, cudatoolkit 10.0-10.1
# * cuDNN 8.1.1, cudatoolkit 10.2-11.2
# * cuDNN 8.3.2, cudatoolkit 10.2-11.5
{ callPackage
, cudatoolkit_10
, cudatoolkit_10_0
, cudatoolkit_10_1
, cudatoolkit_10_2
, cudatoolkit_11
, cudatoolkit_11_0
, cudatoolkit_11_1
, cudatoolkit_11_2
, cudatoolkit_11_3
, cudatoolkit_11_4
, cudatoolkit_11_5
, fetchurl
, lib
}:

let
generic = args: callPackage (import ./generic.nix (removeAttrs args [ "cudatoolkit" ])) {
inherit (args) cudatoolkit;
};
urlPrefix = "https://developer.download.nvidia.com/compute/redist/cudnn";
in
rec {
# cuDNN 7.x
# Still used by libtensorflow-bin. It should be upgraded at some point.
cudnn_7_4_cudatoolkit_10_0 = generic rec {
version = "7.4.2";
cudatoolkit = cudatoolkit_10_0;
srcName = "cudnn-${cudatoolkit.majorVersion}-linux-x64-v7.4.2.24.tgz";
sha256 = "18ys0apiz9afid2s6lvy9qbyi8g66aimb2a7ikl1f3dm09mciprf";
# See https://docs.nvidia.com/deeplearning/cudnn/archives/cudnn_742/cudnn-support-matrix/index.html#cudnn-cuda-hardware-versions__table-cudnn-cuda-hardware-versions.
minCudaVersion = "9.2.88";
maxCudaVersion = "10.0.99999";
mkSrc = _: fetchurl {
url = "${urlPrefix}/v${version}/cudnn-10.0-linux-x64-v7.4.2.24.tgz";
hash = "sha256-Lt/IagK1DRfojEeJVaMy5qHoF05+U6NFi06lH68C2qM=";
};
};
# The only overlap between supported and packaged CUDA versions is 10.0.

# The `cudnn` alias still points to this in all-packages.nix. It should be
# upgraded at some point.
cudnn_7_6_cudatoolkit_10_2 = generic rec {
cudnn_7_6_cudatoolkit_10_0 = generic rec {
version = "7.6.5";
cudatoolkit = cudatoolkit_10_2;
srcName = "cudnn-${cudatoolkit.majorVersion}-linux-x64-v7.6.5.32.tgz";
sha256 = "084c13vzjdkb5s1996yilybg6dgav1lscjr1xdcgvlmfrbr6f0k0";
cudatoolkit = cudatoolkit_10_0;
# See https://docs.nvidia.com/deeplearning/cudnn/archives/cudnn_765/cudnn-support-matrix/index.html#cudnn-versions-763-765.
minCudaVersion = "9.2.148";
maxCudaVersion = "10.1.243";
mkSrc = cudatoolkit: fetchurl {
url = "${urlPrefix}/v${version}/cudnn-${cudatoolkit.majorVersion}-linux-x64-v7.6.5.32.tgz";
hash = {
"10.0" = "sha256-KDVeOV8LK5OsLIO2E2CzW6bNA3fkTni+GXtrYbS0kro=";
"10.1" = "sha256-fq7IA5osMKsLx1jTA1iHZ2k972v0myJIWiwAvy4TbLM=";
}."${cudatoolkit.majorVersion}";
};
};

cudnn_7_6_cudatoolkit_10 = cudnn_7_6_cudatoolkit_10_2;
cudnn_7_6_cudatoolkit_10_1 = cudnn_7_6_cudatoolkit_10_0.override { cudatoolkit = cudatoolkit_10_1; };

# cuDNN 8.x
# cuDNN 8.1 is still used by tensorflow at the time of writing (2022-02-17).
# See https://github.com/NixOS/nixpkgs/pull/158218 for more info.
cudnn_8_1_cudatoolkit_11_0 = generic rec {
cudnn_8_1_cudatoolkit_10_2 = generic rec {
version = "8.1.1";
cudatoolkit = cudatoolkit_11_0;
# 8.1.0 is compatible with CUDA 11.0-11.2:
# https://docs.nvidia.com/deeplearning/cudnn/archives/cudnn-811/support-matrix/index.html
srcName = "cudnn-11.2-linux-x64-v8.1.1.33.tgz";
hash = "sha256-mKh4TpKGLyABjSDCgbMNSgzZUfk2lPZDPM9K6cUCumo=";
cudatoolkit = cudatoolkit_10_2;
# See https://docs.nvidia.com/deeplearning/cudnn/archives/cudnn-811/support-matrix/index.html#cudnn-versions-810-811.
minCudaVersion = "10.2.00000";
maxCudaVersion = "11.2.99999";
mkSrc = cudatoolkit:
let v = if lib.versions.majorMinor cudatoolkit.version == "10.2" then "10.2" else "11.2"; in
fetchurl {
url = "${urlPrefix}/v${version}/cudnn-${v}-linux-x64-v8.1.1.33.tgz";
hash = {
"10.2" = "sha256-Kkp7mabpv6aQ6xm7QeSVU/KnpJGls6v8rpAOFmxbbr0=";
"11.2" = "sha256-mKh4TpKGLyABjSDCgbMNSgzZUfk2lPZDPM9K6cUCumo=";
}."${v}";
};
};
cudnn_8_1_cudatoolkit_11_1 = cudnn_8_1_cudatoolkit_11_0.override { cudatoolkit = cudatoolkit_11_1; };
cudnn_8_1_cudatoolkit_11_2 = cudnn_8_1_cudatoolkit_11_0.override { cudatoolkit = cudatoolkit_11_2; };
cudnn_8_1_cudatoolkit_11 = cudnn_8_1_cudatoolkit_11_2;
cudnn_8_1_cudatoolkit_11_0 = cudnn_8_1_cudatoolkit_10_2.override { cudatoolkit = cudatoolkit_11_0; };
cudnn_8_1_cudatoolkit_11_1 = cudnn_8_1_cudatoolkit_10_2.override { cudatoolkit = cudatoolkit_11_1; };
cudnn_8_1_cudatoolkit_11_2 = cudnn_8_1_cudatoolkit_10_2.override { cudatoolkit = cudatoolkit_11_2; };

cudnn_8_1_cudatoolkit_10 = cudnn_8_1_cudatoolkit_10_2.override { cudatoolkit = cudatoolkit_10; };
cudnn_8_1_cudatoolkit_11 = cudnn_8_1_cudatoolkit_10_2.override { cudatoolkit = cudatoolkit_11; };

# cuDNN 8.3 is necessary for the latest jaxlib, esp. jaxlib-bin. See
# https://github.com/google/jax/discussions/9455 for more info.
cudnn_8_3_cudatoolkit_11_0 = generic rec {
# 8.3.0 is the last version to respect the folder structure that generic.nix
# expects. Later versions have files in a subdirectory `local_installers`.
# See eg https://developer.download.nvidia.com/compute/redist/cudnn/v8.3.1/.
version = "8.3.0";
cudatoolkit = cudatoolkit_11_0;
# 8.3.0 is compatible with CUDA 11.0-11.5:
# https://docs.nvidia.com/deeplearning/cudnn/archives/cudnn-830/support-matrix/index.html
srcName = "cudnn-11.5-linux-x64-v8.3.0.98.tgz";
hash = "sha256-RMb1rVyxL7dPoMmh58qvTwTXVa3xGi5bbJ5BfaN2srI=";
};
cudnn_8_3_cudatoolkit_11_1 = cudnn_8_3_cudatoolkit_11_0.override { cudatoolkit = cudatoolkit_11_1; };
cudnn_8_3_cudatoolkit_11_2 = cudnn_8_3_cudatoolkit_11_0.override { cudatoolkit = cudatoolkit_11_2; };
cudnn_8_3_cudatoolkit_11_3 = cudnn_8_3_cudatoolkit_11_0.override { cudatoolkit = cudatoolkit_11_3; };
cudnn_8_3_cudatoolkit_11_4 = cudnn_8_3_cudatoolkit_11_0.override { cudatoolkit = cudatoolkit_11_4; };
cudnn_8_3_cudatoolkit_11_5 = cudnn_8_3_cudatoolkit_11_0.override { cudatoolkit = cudatoolkit_11_5; };
cudnn_8_3_cudatoolkit_11 = cudnn_8_3_cudatoolkit_11_5;
cudnn_8_3_cudatoolkit_10_2 =
generic
rec {
version = "8.3.2";
cudatoolkit = cudatoolkit_10_2;
# See https://docs.nvidia.com/deeplearning/cudnn/archives/cudnn-832/support-matrix/index.html#cudnn-cuda-hardware-versions.
minCudaVersion = "10.2.00000";
maxCudaVersion = "11.5.99999";
mkSrc = cudatoolkit:
let v = if lib.versions.majorMinor cudatoolkit.version == "10.2" then "10.2" else "11.5"; in
fetchurl {
# Starting at version 8.3.1 there's a new directory layout including
# a subdirectory `local_installers`.
url = "https://developer.download.nvidia.com/compute/redist/cudnn/v${version}/local_installers/${v}/cudnn-linux-x86_64-8.3.2.44_cuda${v}-archive.tar.xz";
hash = {
"10.2" = "sha256-1vVu+cqM+PketzIQumw9ykm6REbBZhv6/lXB7EC2aaw=";
"11.5" = "sha256-VQCVPAjF5dHd3P2iNPnvvdzb5DpTsm3AqCxyP6FwxFc=";
}."${v}";
};
}
;
cudnn_8_3_cudatoolkit_11_0 = cudnn_8_3_cudatoolkit_10_2.override { cudatoolkit = cudatoolkit_11_0; };
cudnn_8_3_cudatoolkit_11_1 = cudnn_8_3_cudatoolkit_10_2.override { cudatoolkit = cudatoolkit_11_1; };
cudnn_8_3_cudatoolkit_11_2 = cudnn_8_3_cudatoolkit_10_2.override { cudatoolkit = cudatoolkit_11_2; };
cudnn_8_3_cudatoolkit_11_3 = cudnn_8_3_cudatoolkit_10_2.override { cudatoolkit = cudatoolkit_11_3; };
cudnn_8_3_cudatoolkit_11_4 = cudnn_8_3_cudatoolkit_10_2.override { cudatoolkit = cudatoolkit_11_4; };
cudnn_8_3_cudatoolkit_11_5 = cudnn_8_3_cudatoolkit_10_2.override { cudatoolkit = cudatoolkit_11_5; };

cudnn_8_3_cudatoolkit_10 = cudnn_8_3_cudatoolkit_10_2.override { cudatoolkit = cudatoolkit_10; };
cudnn_8_3_cudatoolkit_11 = cudnn_8_3_cudatoolkit_10_2.override { cudatoolkit = cudatoolkit_11; };
}
44 changes: 27 additions & 17 deletions pkgs/development/libraries/science/math/cudnn/generic.nix
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
{ version
, srcName
, hash ? null
, sha256 ? null
{ minCudaVersion
, maxCudaVersion
, mkSrc
, version
}:

assert (hash != null) || (sha256 != null);

{ stdenv
, lib
, cudatoolkit
Expand All @@ -25,19 +23,18 @@ stdenv.mkDerivation {
name = "cudatoolkit-${cudatoolkit.majorVersion}-cudnn-${version}";

inherit version;

src = let
hash_ = if hash != null then { inherit hash; } else { inherit sha256; };
in fetchurl ({
# URL from NVIDIA docker containers: https://gitlab.com/nvidia/cuda/blob/centos7/7.0/runtime/cudnn4/Dockerfile
url = "https://developer.download.nvidia.com/compute/redist/cudnn/v${version}/${srcName}";
} // hash_);
# It's often the case that the src depends on the version of cudatoolkit it's
# being linked against, so we pass in `cudatoolkit` as an argument to `mkSrc`.
src = mkSrc cudatoolkit;

nativeBuildInputs = [ addOpenGLRunpath ];

# Some cuDNN libraries depend on things in cudatoolkit, eg.
# libcudnn_ops_infer.so.8 tries to load libcublas.so.11. So we need to patch
# cudatoolkit into RPATH. See also https://github.com/NixOS/nixpkgs/blob/88a2ad974692a5c3638fcdc2c772e5770f3f7b21/pkgs/development/python-modules/jaxlib/bin.nix#L78-L98.
#
# Note also that version <=8.3.0 contained a subdirectory "lib64/" but in
# version 8.3.2 it seems to have been renamed to simply "lib/".
installPhase = ''
runHook preInstall

Expand All @@ -46,14 +43,16 @@ stdenv.mkDerivation {
patchelf --set-rpath "''${p:+$p:}${lib.makeLibraryPath [ stdenv.cc.cc cudatoolkit.lib ]}:${cudatoolkit}/lib:\$ORIGIN/" $1
}

for lib in lib64/lib*.so; do
fixRunPath $lib
for sofile in {lib,lib64}/lib*.so; do
fixRunPath $sofile
done

mkdir -p $out
cp -a include $out/include
cp -a lib64 $out/lib64
[ -d "lib/" ] && cp -a lib $out/lib
[ -d "lib64/" ] && cp -a lib64 $out/lib64
'' + lib.optionalString removeStatic ''
rm -f $out/lib/*.a
rm -f $out/lib64/*.a
'' + ''
runHook postInstall
Expand All @@ -77,10 +76,21 @@ stdenv.mkDerivation {
};

meta = with lib; {
# Check that the cudatoolkit version satisfies our min/max constraints (both
# inclusive). We mark the package as broken if it fails to satisfies the
# official version constraints (as recorded in default.nix). In some cases
# you _may_ be able to smudge version constraints, just know that you're
# embarking into unknown and unsupported territory when doing so.
broken = let cudaVer = lib.getVersion cudatoolkit; in
!(
lib.versionAtLeast cudaVer minCudaVersion
&& lib.versionAtLeast maxCudaVersion cudaVer
);

description = "NVIDIA CUDA Deep Neural Network library (cuDNN)";
homepage = "https://developer.nvidia.com/cudnn";
license = licenses.unfree;
platforms = [ "x86_64-linux" ];
maintainers = with maintainers; [ mdaiter ];
maintainers = with maintainers; [ mdaiter samuela ];
};
}
11 changes: 8 additions & 3 deletions pkgs/top-level/all-packages.nix
Original file line number Diff line number Diff line change
Expand Up @@ -4589,21 +4589,26 @@ with pkgs;
cudnnPackages = callPackages ../development/libraries/science/math/cudnn { };
inherit (cudnnPackages)
cudnn_7_4_cudatoolkit_10_0
cudnn_7_6_cudatoolkit_10_2
cudnn_7_6_cudatoolkit_10
cudnn_7_6_cudatoolkit_10_0
cudnn_7_6_cudatoolkit_10_1
cudnn_8_1_cudatoolkit_10_2
cudnn_8_1_cudatoolkit_11_0
cudnn_8_1_cudatoolkit_11_1
cudnn_8_1_cudatoolkit_11_2
cudnn_8_1_cudatoolkit_10
cudnn_8_1_cudatoolkit_11
cudnn_8_3_cudatoolkit_10_2
cudnn_8_3_cudatoolkit_11_0
cudnn_8_3_cudatoolkit_11_1
cudnn_8_3_cudatoolkit_11_2
cudnn_8_3_cudatoolkit_11_3
cudnn_8_3_cudatoolkit_11_4
cudnn_8_3_cudatoolkit_11_5
cudnn_8_3_cudatoolkit_10
cudnn_8_3_cudatoolkit_11;

cudnn = cudnn_7_6_cudatoolkit_10;
# TODO(samuela): This is old and should be upgraded to 8.3 at some point.
cudnn = cudnn_7_6_cudatoolkit_10_1;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If it's too old, why do we keep it as a default?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just because no one's gotten around to updating it yet. But that would be great for a follow-up PR!


Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Doesn't this have to be cudnn_8_3_cudatoolkit_10 in order to be compatible with the currently packaged cudatoolkit?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ideally, yes. The issue is that cuDNN 7.6.5 doesn't actually support CUDA 10.2, the current cudatoolkit version. And I didn't want to upgrade cuDNN past 7.6.5 here in the interest of keeping this PR as small as possible. The solution is to upgrade cudnn to 8.3 which supports CUDA 10.2-11.5. (Looks like you're already on that, great!)

In the future, it think it would be nice to explore a solution like cudatoolkit_11.withPackages similar to python3.withPackages, but we have to move one step at a time to get the ball rolling.

cutensorPackages = callPackages ../development/libraries/science/math/cutensor { };
inherit (cutensorPackages)
Expand Down