From 1a61b64dbe1918657e85a4327f403f41ab1c146b Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Fri, 15 Apr 2022 06:36:52 +0300 Subject: [PATCH 1/7] cudaPackages.cudnn: migrate to redist cudaPackages --- .../libraries/science/math/cudnn/generic.nix | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/pkgs/development/libraries/science/math/cudnn/generic.nix b/pkgs/development/libraries/science/math/cudnn/generic.nix index 2e53bdcdfc4ea..f819977384545 100644 --- a/pkgs/development/libraries/science/math/cudnn/generic.nix +++ b/pkgs/development/libraries/science/math/cudnn/generic.nix @@ -1,6 +1,6 @@ { stdenv , lib -, cudatoolkit +, cudaPackages , fetchurl , addOpenGLRunpath , # The distributed version of CUDNN includes both dynamically liked .so files, @@ -23,14 +23,15 @@ assert (hash != null) || (sha256 != null); let + inherit (cudaPackages) cudaMajorVersion libcublas; + inherit (cudaPackages.cudatoolkit ) cc; + majorMinorPatch = version: lib.concatStringsSep "." (lib.take 3 (lib.splitVersion version)); version = majorMinorPatch fullVersion; in stdenv.mkDerivation { - name = "cudatoolkit-${cudatoolkit.majorVersion}-cudnn-${version}"; + name = "cudatoolkit-${cudaMajorVersion}-cudnn-${version}"; inherit version; - # It's often the case that the src depends on the version of cudatoolkit it's - # being linked against, so we pass in `cudatoolkit` as an argument to `mkSrc`. src = fetchurl { inherit url hash sha256; }; @@ -48,7 +49,7 @@ in stdenv.mkDerivation { function fixRunPath { p=$(patchelf --print-rpath $1) - patchelf --set-rpath "''${p:+$p:}${lib.makeLibraryPath [ stdenv.cc.cc cudatoolkit.lib ]}:${cudatoolkit}/lib:\$ORIGIN/" $1 + patchelf --set-rpath "''${p:+$p:}${lib.makeLibraryPath [ cc.cc libcublas ]}:\$ORIGIN/" $1 } for sofile in {lib,lib64}/lib*.so; do @@ -66,6 +67,11 @@ in stdenv.mkDerivation { runHook postInstall ''; + doInstallCheck = true; + installCheckPhase = '' + ! find -iname '*.so' -exec ldd {} + | grep 'not found' + ''; + # Set RUNPATH so that libcuda in /run/opengl-driver(-32)/lib can be found. # See the explanation in addOpenGLRunpath. postFixup = '' @@ -75,11 +81,16 @@ in stdenv.mkDerivation { ''; propagatedBuildInputs = [ - cudatoolkit + # FIXME: we can and should decouple cudnn from runfile-based cudatoolkit + # ...leaving this in propagated inputs just to avoid breaking downstream + # derivations right now + cudaPackages.cudatoolkit ]; passthru = { - inherit cudatoolkit; + cudatoolkit = lib.warn "cudnn.cudatoolkit passthru attribute is deprecated: use cudaPackages" cudaPackages.cudatoolkit; + inherit cudaPackages; + majorVersion = lib.versions.major version; }; @@ -89,7 +100,7 @@ in stdenv.mkDerivation { # official version constraints (as recorded in default.nix). In some cases # you _may_ be able to smudge version constraints, just know that you're # embarking into unknown and unsupported territory when doing so. - broken = !(elem cudatoolkit.majorMinorVersion supportedCudaVersions); + broken = !(elem cudaPackages.cudaVersion supportedCudaVersions); description = "NVIDIA CUDA Deep Neural Network library (cuDNN)"; homepage = "https://developer.nvidia.com/cudnn"; license = licenses.unfree; From 91795ddbb3e5c5b8a304b29bda1c9e8e795eac97 Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Fri, 15 Apr 2022 06:37:45 +0300 Subject: [PATCH 2/7] cudaPackages.cudnn: fix missing zlib in rpath --- pkgs/development/libraries/science/math/cudnn/generic.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkgs/development/libraries/science/math/cudnn/generic.nix b/pkgs/development/libraries/science/math/cudnn/generic.nix index f819977384545..43087d64a4f45 100644 --- a/pkgs/development/libraries/science/math/cudnn/generic.nix +++ b/pkgs/development/libraries/science/math/cudnn/generic.nix @@ -1,5 +1,6 @@ { stdenv , lib +, zlib , cudaPackages , fetchurl , addOpenGLRunpath @@ -49,7 +50,7 @@ in stdenv.mkDerivation { function fixRunPath { p=$(patchelf --print-rpath $1) - patchelf --set-rpath "''${p:+$p:}${lib.makeLibraryPath [ cc.cc libcublas ]}:\$ORIGIN/" $1 + patchelf --set-rpath "''${p:+$p:}${lib.makeLibraryPath [ cc.cc libcublas zlib ]}:\$ORIGIN/" $1 } for sofile in {lib,lib64}/lib*.so; do From c501e5f0454d6f201c29b89152e4aa6bb8f2a893 Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Fri, 15 Apr 2022 07:16:35 +0300 Subject: [PATCH 3/7] cudaPackages.cudnn: remove the propagated cudatoolkit --- pkgs/development/libraries/science/math/cudnn/generic.nix | 7 ------- pkgs/games/katago/default.nix | 1 + 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/pkgs/development/libraries/science/math/cudnn/generic.nix b/pkgs/development/libraries/science/math/cudnn/generic.nix index 43087d64a4f45..a01c3b033558e 100644 --- a/pkgs/development/libraries/science/math/cudnn/generic.nix +++ b/pkgs/development/libraries/science/math/cudnn/generic.nix @@ -81,13 +81,6 @@ in stdenv.mkDerivation { done ''; - propagatedBuildInputs = [ - # FIXME: we can and should decouple cudnn from runfile-based cudatoolkit - # ...leaving this in propagated inputs just to avoid breaking downstream - # derivations right now - cudaPackages.cudatoolkit - ]; - passthru = { cudatoolkit = lib.warn "cudnn.cudatoolkit passthru attribute is deprecated: use cudaPackages" cudaPackages.cudatoolkit; inherit cudaPackages; diff --git a/pkgs/games/katago/default.nix b/pkgs/games/katago/default.nix index 6328b3f9bc820..a3d7ed5cba9df 100644 --- a/pkgs/games/katago/default.nix +++ b/pkgs/games/katago/default.nix @@ -52,6 +52,7 @@ stdenv.mkDerivation rec { eigen ] ++ lib.optionals (enableGPU && enableCuda) [ cudaPackages.cudnn + cudaPackages.cudatoolkit mesa.drivers ] ++ lib.optionals (enableGPU && !enableCuda) [ opencl-headers From 0378f6f7c909d7dbe5331a16d57f7e24aa4513d3 Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Fri, 15 Apr 2022 07:38:29 +0300 Subject: [PATCH 4/7] cudaPackages.cudnn: use autoPatchelfHook instead of custom find ... -exec ldd | grep routine mark libcudnn_cnn_infer.so as needed for libcudnn.so on cudnn>=8.0.5 - a hint for autoPatchelf, as an alternative to manually adding $ORIGIN --- .../libraries/science/math/cudnn/generic.nix | 52 +++++++++---------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/pkgs/development/libraries/science/math/cudnn/generic.nix b/pkgs/development/libraries/science/math/cudnn/generic.nix index a01c3b033558e..69f4d27eef9c9 100644 --- a/pkgs/development/libraries/science/math/cudnn/generic.nix +++ b/pkgs/development/libraries/science/math/cudnn/generic.nix @@ -1,6 +1,7 @@ { stdenv , lib , zlib +, autoPatchelfHook , cudaPackages , fetchurl , addOpenGLRunpath @@ -18,18 +19,19 @@ , url , hash ? null , sha256 ? null -, supportedCudaVersions ? [] +, supportedCudaVersions ? [ ] }: assert (hash != null) || (sha256 != null); let inherit (cudaPackages) cudaMajorVersion libcublas; - inherit (cudaPackages.cudatoolkit ) cc; + inherit (cudaPackages.cudatoolkit) cc; majorMinorPatch = version: lib.concatStringsSep "." (lib.take 3 (lib.splitVersion version)); version = majorMinorPatch fullVersion; -in stdenv.mkDerivation { +in +stdenv.mkDerivation { name = "cudatoolkit-${cudaMajorVersion}-cudnn-${version}"; inherit version; @@ -37,26 +39,22 @@ in stdenv.mkDerivation { inherit url hash sha256; }; - nativeBuildInputs = [ addOpenGLRunpath ]; + nativeBuildInputs = [ autoPatchelfHook addOpenGLRunpath ]; - # Some cuDNN libraries depend on things in cudatoolkit, eg. - # libcudnn_ops_infer.so.8 tries to load libcublas.so.11. So we need to patch - # cudatoolkit into RPATH. See also https://github.com/NixOS/nixpkgs/blob/88a2ad974692a5c3638fcdc2c772e5770f3f7b21/pkgs/development/python-modules/jaxlib/bin.nix#L78-L98. + # Used by autoPatchelfHook + buildInputs = builtins.map lib.getLib [ + cc.cc # libstdc++ + libcublas + zlib + ]; + + # We used to patch Runpath here, but now we use autoPatchelfHook # # Note also that version <=8.3.0 contained a subdirectory "lib64/" but in # version 8.3.2 it seems to have been renamed to simply "lib/". installPhase = '' runHook preInstall - function fixRunPath { - p=$(patchelf --print-rpath $1) - patchelf --set-rpath "''${p:+$p:}${lib.makeLibraryPath [ cc.cc libcublas zlib ]}:\$ORIGIN/" $1 - } - - for sofile in {lib,lib64}/lib*.so; do - fixRunPath $sofile - done - mkdir -p $out cp -a include $out/include [ -d "lib/" ] && cp -a lib $out/lib @@ -68,17 +66,17 @@ in stdenv.mkDerivation { runHook postInstall ''; - doInstallCheck = true; - installCheckPhase = '' - ! find -iname '*.so' -exec ldd {} + | grep 'not found' - ''; - - # Set RUNPATH so that libcuda in /run/opengl-driver(-32)/lib can be found. - # See the explanation in addOpenGLRunpath. - postFixup = '' - for lib in $out/lib/lib*.so; do - addOpenGLRunpath $lib - done + # Check and normalize Runpath against DT_NEEDED using autoPatchelf. + # Prepend /run/opengl-driver/lib using addOpenGLRunpath + # so that libcuda (which is not part of DT_NEEDED) + # can be found at runtime with dlopen(). + # Without --add-needed autoPatchelf forgets $ORIGIN on cuda>=8.0.5. + dontAutoPatchelf = true; + postFixup = lib.optionalString (lib.versionAtLeast fullVersion "8.0.5") '' + patchelf $out/lib/libcudnn.so --add-needed libcudnn_cnn_infer.so + '' ++ '' + autoPatchelf $out + addOpenGLRunpath $out/lib/lib*.so ''; passthru = { From 98121e6dd2aa11d29b62869e4ba7b1b0581be9cd Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Fri, 15 Apr 2022 10:09:55 +0300 Subject: [PATCH 5/7] cudaPackages.cudnn: use automatic hooks for rpath as a more common way to use addOpenGLRunpath and autoPatchelf with cudaPackages --- .../libraries/science/math/cudnn/generic.nix | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/pkgs/development/libraries/science/math/cudnn/generic.nix b/pkgs/development/libraries/science/math/cudnn/generic.nix index 69f4d27eef9c9..e2bea8e4eeded 100644 --- a/pkgs/development/libraries/science/math/cudnn/generic.nix +++ b/pkgs/development/libraries/science/math/cudnn/generic.nix @@ -1,10 +1,10 @@ { stdenv , lib , zlib -, autoPatchelfHook , cudaPackages +, autoPatchelfHook +, autoAddOpenGLRunpathHook , fetchurl -, addOpenGLRunpath , # The distributed version of CUDNN includes both dynamically liked .so files, # as well as statically linked .a files. However, CUDNN is quite large # (multiple gigabytes), so you can save some space in your nix store by @@ -39,7 +39,14 @@ stdenv.mkDerivation { inherit url hash sha256; }; - nativeBuildInputs = [ autoPatchelfHook addOpenGLRunpath ]; + # Check and normalize Runpath against DT_NEEDED using autoPatchelf. + # Prepend /run/opengl-driver/lib using addOpenGLRunpath + # so that libcuda (which is not part of DT_NEEDED) + # can be found at runtime with dlopen(). + nativeBuildInputs = [ + autoPatchelfHook + autoAddOpenGLRunpathHook + ]; # Used by autoPatchelfHook buildInputs = builtins.map lib.getLib [ @@ -66,17 +73,9 @@ stdenv.mkDerivation { runHook postInstall ''; - # Check and normalize Runpath against DT_NEEDED using autoPatchelf. - # Prepend /run/opengl-driver/lib using addOpenGLRunpath - # so that libcuda (which is not part of DT_NEEDED) - # can be found at runtime with dlopen(). # Without --add-needed autoPatchelf forgets $ORIGIN on cuda>=8.0.5. - dontAutoPatchelf = true; postFixup = lib.optionalString (lib.versionAtLeast fullVersion "8.0.5") '' patchelf $out/lib/libcudnn.so --add-needed libcudnn_cnn_infer.so - '' ++ '' - autoPatchelf $out - addOpenGLRunpath $out/lib/lib*.so ''; passthru = { From 0c75c477a792e8ef3531af59dcc2a91f1a170e03 Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Fri, 15 Apr 2022 10:19:57 +0300 Subject: [PATCH 6/7] cudaPackages.cudnn: consume individual cuda packages ...since cudnn is part of the cuda package set - introduces the scary useCudatoolkitRunfile function argument to discourage usage of the runfile-based cudatoolkit - instead of the rather hidden useRedist term in let ... in - repeats cudatoolkit_root pattern after cuda_joined in pytorch &c (the "toolchain view") - redist packages are marked optional to support cuda<11.4 where the attributes for redist packages do not exist --- .../science/math/cudnn/extension.nix | 6 ++- .../libraries/science/math/cudnn/generic.nix | 39 ++++++++++++------- 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/pkgs/development/libraries/science/math/cudnn/extension.nix b/pkgs/development/libraries/science/math/cudnn/extension.nix index 172935fc97949..f1bdfb9836ed3 100644 --- a/pkgs/development/libraries/science/math/cudnn/extension.nix +++ b/pkgs/development/libraries/science/math/cudnn/extension.nix @@ -5,7 +5,11 @@ final: prev: let ### CuDNN - buildCuDnnPackage = args: callPackage ./generic.nix {} args; + buildCuDnnPackage = args: + let + useCudatoolkitRunfile = lib.versionOlder cudaVersion "11.3.999"; + in + callPackage ./generic.nix { inherit useCudatoolkitRunfile; } args; toUnderscore = str: lib.replaceStrings ["."] ["_"] str; diff --git a/pkgs/development/libraries/science/math/cudnn/generic.nix b/pkgs/development/libraries/science/math/cudnn/generic.nix index e2bea8e4eeded..a0c9345a4e54f 100644 --- a/pkgs/development/libraries/science/math/cudnn/generic.nix +++ b/pkgs/development/libraries/science/math/cudnn/generic.nix @@ -1,7 +1,11 @@ { stdenv , lib , zlib -, cudaPackages +, useCudatoolkitRunfile ? false +, cudaVersion +, cudaMajorVersion +, cudatoolkit # if cuda>=11: only used for .cc +, libcublas ? null # cuda <11 doesn't ship redist packages , autoPatchelfHook , autoAddOpenGLRunpathHook , fetchurl @@ -24,35 +28,38 @@ assert (hash != null) || (sha256 != null); +assert useCudatoolkitRunfile || (libcublas != null); + let - inherit (cudaPackages) cudaMajorVersion libcublas; - inherit (cudaPackages.cudatoolkit) cc; + inherit (cudatoolkit) cc; majorMinorPatch = version: lib.concatStringsSep "." (lib.take 3 (lib.splitVersion version)); version = majorMinorPatch fullVersion; + + cudatoolkit_root = + if useCudatoolkitRunfile + then cudatoolkit + else libcublas; in stdenv.mkDerivation { name = "cudatoolkit-${cudaMajorVersion}-cudnn-${version}"; - inherit version; src = fetchurl { inherit url hash sha256; }; # Check and normalize Runpath against DT_NEEDED using autoPatchelf. - # Prepend /run/opengl-driver/lib using addOpenGLRunpath - # so that libcuda (which is not part of DT_NEEDED) - # can be found at runtime with dlopen(). + # Prepend /run/opengl-driver/lib using addOpenGLRunpath for dlopen("libcudacuda.so") nativeBuildInputs = [ autoPatchelfHook autoAddOpenGLRunpathHook ]; # Used by autoPatchelfHook - buildInputs = builtins.map lib.getLib [ - cc.cc # libstdc++ - libcublas + buildInputs = [ + cc.cc.lib # libstdc++ zlib + cudatoolkit_root ]; # We used to patch Runpath here, but now we use autoPatchelfHook @@ -79,8 +86,13 @@ stdenv.mkDerivation { ''; passthru = { - cudatoolkit = lib.warn "cudnn.cudatoolkit passthru attribute is deprecated: use cudaPackages" cudaPackages.cudatoolkit; - inherit cudaPackages; + inherit useCudatoolkitRunfile; + + cudatoolkit = lib.warn '' + cudnn.cudatoolkit passthru attribute is deprecated; + if your derivation uses cudnn directly, it should probably consume cudaPackages instead + '' + cudatoolkit; majorVersion = lib.versions.major version; }; @@ -91,9 +103,10 @@ stdenv.mkDerivation { # official version constraints (as recorded in default.nix). In some cases # you _may_ be able to smudge version constraints, just know that you're # embarking into unknown and unsupported territory when doing so. - broken = !(elem cudaPackages.cudaVersion supportedCudaVersions); + broken = !(elem cudaVersion supportedCudaVersions); description = "NVIDIA CUDA Deep Neural Network library (cuDNN)"; homepage = "https://developer.nvidia.com/cudnn"; + # TODO: consider marking unfreRedistributable when not using runfile license = licenses.unfree; platforms = [ "x86_64-linux" ]; maintainers = with maintainers; [ mdaiter samuela ]; From 3e3e1e26b144728465ecb6eadce0dc96460e73d7 Mon Sep 17 00:00:00 2001 From: Serge K Date: Sat, 16 Apr 2022 07:47:29 +0300 Subject: [PATCH 7/7] cudaPackages.cudnn: update to pname+version Co-authored-by: Sandro --- pkgs/development/libraries/science/math/cudnn/generic.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkgs/development/libraries/science/math/cudnn/generic.nix b/pkgs/development/libraries/science/math/cudnn/generic.nix index a0c9345a4e54f..1e9bac5efed29 100644 --- a/pkgs/development/libraries/science/math/cudnn/generic.nix +++ b/pkgs/development/libraries/science/math/cudnn/generic.nix @@ -42,7 +42,8 @@ let else libcublas; in stdenv.mkDerivation { - name = "cudatoolkit-${cudaMajorVersion}-cudnn-${version}"; + pname = "cudatoolkit-${cudaMajorVersion}-cudnn"; + inherit version; src = fetchurl { inherit url hash sha256;