rPackages: compile r-torch from source

pkgs/development/python-modules/torch/default.nix

@@ -119,7 +119,7 @@ let
         "8.7"
         "8.9"
         "9.0"
-        "9.0a"
+        # "9.0a"
       ];
       ptx = lists.map (x: "${x}+PTX") real;
     in
@@ -214,7 +214,7 @@ in
 buildPythonPackage rec {
   pname = "torch";
   # Don't forget to update torch-bin to the same version.
-  version = "2.4.1";
+  version = "2.0.1";
   pyproject = true;
 
   disabled = pythonOlder "3.8.0";
@@ -232,16 +232,24 @@ buildPythonPackage rec {
     repo = "pytorch";
     rev = "refs/tags/v${version}";
     fetchSubmodules = true;
-    hash = "sha256-x/zM/57syr46CP1TfGaefSjzvNm4jJbWFZGVGyzPMg8=";
+    hash = "sha256-xUj77yKz3IQ3gd/G32pI4OhL3LoN1zS7eFg0/0nZp5I=";
   };
 
   patches =
     [
+      ./pytorch_2_0_1_c10_fix.patch
+      ./pytorch_2_0_1_aten_fix.patch
+      ./pytorch_2_0_1_quant_fix.patch
+      ./pytorch_2_0_1_jit_fix.patch
+      ./pytorch_2_0_1_lazy_fix.patch
+
+      ./pytorch_2_0_1_kineto_fix.patch
+      ./pytorch_2_0_1_fbgemmm_fix.patch
       # Allow setting PYTHON_LIB_REL_PATH with an environment variable.
       # https://github.com/pytorch/pytorch/pull/128419
       ./passthrough-python-lib-rel-path.patch
     ]
-    ++ lib.optionals cudaSupport [ ./fix-cmake-cuda-toolkit.patch ]
+    #++ lib.optionals cudaSupport [ ./fix-cmake-cuda-toolkit.patch ]
     ++ lib.optionals (stdenv.hostPlatform.isDarwin && stdenv.hostPlatform.isx86_64) [
       # pthreadpool added support for Grand Central Dispatch in April
       # 2020. However, this relies on functionality (DISPATCH_APPLY_AUTO)
@@ -290,9 +298,9 @@ buildPythonPackage rec {
     # NOTE: Parts of pytorch rely on unmaintained FindCUDA.cmake with custom patches to support e.g.
     # newer architectures (sm_90a). We do want to delete vendored patches, but have to keep them
     # until https://github.com/pytorch/pytorch/issues/76082 is addressed
-    + lib.optionalString cudaSupport ''
-      rm cmake/Modules/FindCUDAToolkit.cmake
-    ''
+    # + lib.optionalString cudaSupport ''
+    #   rm cmake/Modules/FindCUDAToolkit.cmake
+    # ''
     # error: no member named 'aligned_alloc' in the global namespace; did you mean simply 'aligned_alloc'
     # This lib overrided aligned_alloc hence the error message. Tltr: his function is linkable but not in header.
     +

pkgs/development/python-modules/torch/pytorch_2_0_1_aten_fix.patch

@@ -0,0 +1,23 @@
+From bb0b283e5ace42b65a6180956bdff6af1b560e4c Mon Sep 17 00:00:00 2001
+From: Nikita Shulga <nshulga@meta.com>
+Date: Fri, 28 Jul 2023 07:08:59 -0700
+Subject: [PATCH] Do not force -Werror on Pooling.cpp
+
+As new versions of compilers are likely find new types of violation s as shown in https://github.com/pytorch/pytorch/issues/105728
+---
+ caffe2/CMakeLists.txt | 2 --
+ 1 file changed, 2 deletions(-)
+
+diff --git a/caffe2/CMakeLists.txt b/caffe2/CMakeLists.txt
+index cc6ecdc7415f6..955bc67e2299c 100644
+--- a/caffe2/CMakeLists.txt
++++ b/caffe2/CMakeLists.txt
+@@ -530,8 +530,6 @@ endif()
+ # Required workaround for LLVM 9 includes.
+ if(NOT MSVC)
+   set_source_files_properties(${TORCH_SRC_DIR}/csrc/jit/tensorexpr/llvm_jit.cpp PROPERTIES COMPILE_FLAGS -Wno-noexcept-type)
+-  # Force -Werror on several files
+-  set_source_files_properties(${CMAKE_CURRENT_LIST_DIR}/../aten/src/ATen/native/mkldnn/Pooling.cpp PROPERTIES COMPILE_FLAGS "-Werror")
+ endif()
+ # Disable certain warnings for GCC-9.X
+ if(CMAKE_COMPILER_IS_GNUCXX AND (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 9.0.0))

pkgs/development/python-modules/torch/pytorch_2_0_1_c10_fix.patch

@@ -0,0 +1,11 @@
+diff --git a/c10/util/Registry.h b/c10/util/Registry.h
+--- a/c10/util/Registry.h
++++ b/c10/util/Registry.h
+@@ -11,2 +11,3 @@
+ #include <c10/macros/Macros.h>
+ #include <c10/util/Type.h>
++#include <stdexcept>
+
+ namespace c10 {
+
+ template <typename KeyType>

pkgs/development/python-modules/torch/pytorch_2_0_1_fbgemmm_fix.patch

@@ -0,0 +1,12 @@
+diff --git a/third_party/fbgemm/include/fbgemm/UtilsAvx2.h b/third_party/fbgemm/include/fbgemm/UtilsAvx2.h
+index a1af6078a..4fb1220eb 100644
+--- a/third_party/fbgemm/include/fbgemm/UtilsAvx2.h
++++ b/third_party/fbgemm/include/fbgemm/UtilsAvx2.h
+@@ -8,4 +8,5 @@
+ // This file defines common utilities used in code compiled with avx2/avx512
+ // flags.
+
++#include <cstdint>
+ #include <string>
+
+ namespace fbgemm {

pkgs/development/python-modules/torch/pytorch_2_0_1_jit_fix.patch

@@ -0,0 +1,17 @@
+diff --git a/torch/csrc/jit/runtime/logging.h b/torch/csrc/jit/runtime/logging.h
+--- a/torch/csrc/jit/runtime/logging.h
++++ b/torch/csrc/jit/runtime/logging.h
+@@ -11,12 +11,13 @@
+ #pragma once
+
+ #include <mutex>
+ #include <string>
+ #include <unordered_map>
+ #include <vector>
++#include <stdexcept>
+
+ #include <torch/csrc/Export.h>
+
+ namespace torch {
+ namespace jit {
+ namespace logging {

pkgs/development/python-modules/torch/pytorch_2_0_1_kineto_fix.patch

@@ -0,0 +1,10 @@
+diff --git a/third_party/kineto/libkineto/src/SampleListener.h b/a/third_party/kineto/libkineto/src/SampleListener.h
+--- a/third_party/kineto/libkineto/src/SampleListener.h
++++ b/third_party/kineto/libkineto/src/SampleListener.h
+@@ -11,5 +11,6 @@
+ #include <assert.h>
+ #include <cmath>
+ #include <iostream>
++#include <cstdint>
+ #include <string>
+ #include <vector>

pkgs/development/python-modules/torch/pytorch_2_0_1_lazy_fix.patch

@@ -0,0 +1,14 @@
+diff --git a/torch/csrc/lazy/core/multi_wait.h b/torch/csrc/lazy/core/multi_wait.h
+--- a/torch/csrc/lazy/core/multi_wait.h
++++ b/torch/csrc/lazy/core/multi_wait.h
+@@ -11,9 +11,10 @@
+ #include <condition_variable>
+ #include <functional>
+ #include <memory>
+ #include <mutex>
++#include <stdexcept>
+
+ #include <c10/macros/Export.h>
+
+ namespace torch {
+ namespace lazy {

pkgs/development/python-modules/torch/pytorch_2_0_1_quant_fix.patch

@@ -0,0 +1,11 @@
+diff --git a/torch/csrc/jit/passes/quantization/quantization_type.h b/torch/csrc/jit/passes/quantization/quantization_type.h
+index a1af6078a..4fb1220eb 100644
+--- a/torch/csrc/jit/passes/quantization/quantization_type.h
++++ b/torch/csrc/jit/passes/quantization/quantization_type.h
+@@ -8,4 +8,5 @
+ #pragma once
+ #include <ostream>
++#include <cstdint>
+
+ namespace torch {
+ namespace jit {

pkgs/development/r-modules/default.nix

@@ -596,6 +596,10 @@ let
     symbolicQspray = [ pkgs.pkg-config ];
     sphereTessellation = [ pkgs.pkg-config ];
     vapour = [ pkgs.pkg-config ];
+    torch = with pkgs; [
+      cmake
+      gcc11
+    ];
   };
 
   packagesWithBuildInputs = {
@@ -803,6 +807,23 @@ let
     MedianaDesigner = [ pkgs.zlib.dev ];
     ChemmineOB = [ pkgs.eigen ];
     DGP4LCF = [ pkgs.lapack pkgs.blas ];
+    #Wrap with mkIf cuda=true or similar
+    torch = [
+      pkgs.cudaPackages_11_8.cuda_cudart.dev
+      pkgs.cudaPackages_11_8.cuda_nvrtc.dev
+      # pkgs.cudaPackages_11_8.cuda_nvrtc.lib
+      pkgs.cudaPackages_11_8.cuda_cccl.dev
+      pkgs.cudaPackages_11_8.cuda_nvtx.dev
+      pkgs.cudaPackages_11_8.cuda_nvcc # no lib or dev?
+      pkgs.cudaPackages_11_8.libcusparse.dev
+      pkgs.cudaPackages_11_8.libcusolver.dev
+      pkgs.cudaPackages_11_8.libcublas.dev
+      pkgs.cudaPackages_11_8.libcurand.dev
+      pkgs.cudaPackages_11_8.libcufft.lib
+      pkgs.cudaPackages_11_8.cudnn.dev
+
+      # pkgs.python3Packages.torchWithCuda.dev # pulled in by setting env var TORCH_PATH in torch override
+    ];
   };
 
   packagesRequiringX = [
@@ -1824,9 +1845,78 @@ let
       '';
     });
 
-    torch = old.torch.overrideAttrs (attrs: {
+    torch =
+      let
+        libtorch_hack_2_0_1 = pkgs.python311Packages.torch.override
+          {
+            # protobuf = pkgs.python3Packages.protobuf.override {
+            #   protobuf = pkgs.protobuf.override {
+            #     stdenv = pkgs.gcc13Stdenv;
+            #     abseil-cpp = pkgs.abseil-cpp.override {
+            #       stdenv = pkgs.gcc13Stdenv;
+            #     };
+            #     gtest = pkgs.gtest.override {
+            #       stdenv = pkgs.gcc13Stdenv;
+            #     };
+            #   };
+            # };
+            # tensorboard = pkgs.python3Packages.tensorboard.override {
+
+            #   protobuf = pkgs.python3Packages.protobuf.override {
+            #                   # stdenv = pkgs.gcc13Stdenv;
+
+            #     protobuf = pkgs.protobuf.override {
+            #       stdenv = pkgs.gcc13Stdenv;
+            #       abseil-cpp = pkgs.abseil-cpp.override {
+            #         stdenv = pkgs.gcc13Stdenv;
+            #       };
+            #       gtest = pkgs.gtest.override {
+            #         stdenv = pkgs.gcc13Stdenv;
+            #       };
+            #     };
+            #   };
+            # };
+            cudaSupport = true;
+            rocmSupport = false;
+            cudaPackages = pkgs.cudaPackages_11_8.overrideScope (cu-fi: _: {
+              # CuDNN 9 is not supported:
+              # https://github.com/cupy/cupy/issues/8215
+              cudnn = cu-fi.cudnn_8_9;
+            });
+
+            # cudaPackages = pkgs.cudaPackages_11_8;
+            effectiveMagma = pkgs.magma-cuda-static.override {
+              cudaPackages = pkgs.cudaPackages_11_8;
+            };
+            #stdenv = pkgs.gcc11Stdenv;
+          };
+      in
+      old.torch.overrideAttrs (attrs: {
+      # CRAN source is modified, and does not contain lantern source
+      src = pkgs.fetchFromGitHub {
+        owner = "mlverse";
+        repo = "torch";
+        rev = "v${lib.strings.removePrefix "r-torch-" attrs.name}";
+        sha256 ="sha256-AqigcWB3mZUsC9sQ7WKqPd7sTAMEHE7mqbjEmuztxfs=";
+      };
+      env = {
+        BUILD_LANTERN = "1";
+        CUDA = "11.8";
+        # May need libtorch 2.0.1
+        # Trialled libtorch-bin, missing a header file
+        TORCH_PATH = "${libtorch_hack_2_0_1.dev}";
+      };
+#      cmakeFlags = [
+#        "-DCMAKE_C_COMPILER=${pkgs.gcc11}/bin/cc"
+#        "-DCMAKE_CXX_COMPILER=${pkgs.gcc11}/bin/c++"
+#      ];
       preConfigure = ''
         patchShebangs configure
+        substituteInPlace configure \
+             --replace-fail "\$R_PACKAGE_DIR" "$out"
+        # I think this might be a bug
+        substituteInPlace src/lantern/CMakeLists.txt \
+             --replace-fail '"ENV{TORCH_PATH}"' '"$ENV{TORCH_PATH}"'
       '';
     });
 
@@ -1845,5 +1935,6 @@ let
       '';
     });
   };
+
 in
   self

pkgs/top-level/all-packages.nix

@@ -16606,7 +16606,8 @@ with pkgs;
 
   pythonInterpreters = callPackage ./../development/interpreters/python { };
   inherit (pythonInterpreters) python27 python39 python310 python311 python312 python313 python3Minimal pypy27 pypy310 pypy39 rustpython;
-
+  # inherit (pythonInterpreters) python27 python39 python310 python311  python313 python3Minimal pypy27 pypy310 pypy39 rustpython;
+  # python312 = pythonInterpreters.python312.override {stdenv = gcc11Stdenv;};
   # List of extensions with overrides to apply to all Python package sets.
   pythonPackagesExtensions = [ ];