Build Tensorflow and libtensorflow from source

pkgs/build-support/build-bazel-package/default.nix

@@ -4,19 +4,19 @@
 , lib
 }:
 
-args@{ name, bazelFlags ? [], bazelTarget, buildAttrs, fetchAttrs, ... }:
+args@{ name, bazelFlags ? [], bazelBuildFlags ? [], bazelFetchFlags ? [], bazelTarget, buildAttrs, fetchAttrs, ... }:
 
 let
   fArgs = removeAttrs args [ "buildAttrs" "fetchAttrs" ];
   fBuildAttrs = fArgs // buildAttrs;
   fFetchAttrs = fArgs // removeAttrs fetchAttrs [ "sha256" ];
 
 in stdenv.mkDerivation (fBuildAttrs // {
-  inherit name bazelFlags bazelTarget;
+  inherit name bazelFlags bazelBuildFlags bazelFetchFlags bazelTarget;
 
   deps = stdenv.mkDerivation (fFetchAttrs // {
     name = "${name}-deps";
-    inherit bazelFlags bazelTarget;
+    inherit bazelFlags bazelBuildFlags bazelFetchFlags bazelTarget;
 
     nativeBuildInputs = fFetchAttrs.nativeBuildInputs or [] ++ [ bazel ];
 
@@ -49,6 +49,7 @@ in stdenv.mkDerivation (fBuildAttrs // {
         fetch \
         --loading_phase_threads=1 \
         $bazelFlags \
+        $bazelFetchFlags \
         $bazelTarget
 
       runHook postBuild
@@ -149,6 +150,7 @@ in stdenv.mkDerivation (fBuildAttrs // {
       "''${host_linkopts[@]}" \
       '' + ''
       $bazelFlags \
+      $bazelBuildFlags \
       $bazelTarget
 
     runHook postBuild

pkgs/development/libraries/science/math/tensorflow/bin.nix

@@ -0,0 +1,72 @@
+{ stdenv
+, fetchurl
+, patchelf
+, cudaSupport ? false, symlinkJoin, cudatoolkit, cudnn, nvidia_x11
+}:
+
+with stdenv.lib;
+let
+  unavailable = throw "libtensorflow is not available for this platform!";
+
+  tfType = if cudaSupport then "gpu" else "cpu";
+
+  system = 
+    if      stdenv.isLinux  then "linux"
+    else if stdenv.isDarwin then "darwin"
+    else unavailable;
+
+  platform =
+    if stdenv.isx86_64 then "x86_64"
+    else unavailable;
+
+  rpath = makeLibraryPath ([stdenv.cc.libc stdenv.cc.cc.lib] ++
+            optionals cudaSupport [ cudatoolkit.out cudatoolkit.lib cudnn nvidia_x11 ]);
+
+  packages = import ./binary-hashes.nix;
+  packageName = "${tfType}-${system}-${platform}";
+  url = packages.${packageName} or unavailable;
+
+  patchLibs =
+    if stdenv.isDarwin
+    then ''
+      install_name_tool -id $out/lib/libtensorflow.dylib $out/lib/libtensorflow.dylib
+      install_name_tool -id $out/lib/libtensorflow_framework.dylib $out/lib/libtensorflow_framework.dylib
+    ''
+    else ''
+      patchelf --set-rpath "${rpath}:$out/lib" $out/lib/libtensorflow.so
+      patchelf --set-rpath "${rpath}" $out/lib/libtensorflow_framework.so
+    '';
+
+in stdenv.mkDerivation rec {
+  pname = "libtensorflow";
+  inherit (packages) version;
+
+  src = fetchurl url;
+
+  # Patch library to use our libc, libstdc++ and others
+  buildCommand = ''
+    mkdir -pv $out
+    tar -C $out -xzf $src
+    chmod -R +w $out
+    ${patchLibs}
+
+    # Write pkgconfig file.
+    mkdir $out/lib/pkgconfig
+    cat > $out/lib/pkgconfig/tensorflow.pc << EOF
+    Name: TensorFlow
+    Version: ${version}
+    Description: Library for computation using data flow graphs for scalable machine learning
+    Requires:
+    Libs: -L$out/lib -ltensorflow
+    Cflags: -I$out/include/tensorflow
+    EOF
+  '';
+
+  meta = {
+    description = "C API for TensorFlow";
+    homepage = https://www.tensorflow.org/install/lang_c;
+    license = licenses.asl20;
+    platforms = [ "x86_64-linux" "x86_64-darwin" ];
+    maintainers = with maintainers; [ basvandijk ];
+  };
+}

pkgs/development/libraries/science/math/tensorflow/binary-hashes.nix

@@ -0,0 +1,15 @@
+{
+version = "1.14.0";
+"cpu-linux-x86_64" = {
+  url = "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-linux-x86_64-1.14.0.tar.gz";
+  sha256 = "04bi3ijq4sbb8c5vk964zlv0j9mrjnzzxd9q9knq3h273nc1a36k";
+};
+"gpu-linux-x86_64" = {
+  url = "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-gpu-linux-x86_64-1.14.0.tar.gz";
+  sha256 = "1ffnpyj9jjgwxpjfiyjvq4dm3n6nwiksim5jld9zw7fdswh215x6";
+};
+"cpu-darwin-x86_64" = {
+  url = "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-darwin-x86_64-1.14.0.tar.gz";
+  sha256 = "0zsd5ils1a17j6jzh0c7q1z56fw46gkzybbnms7h2rgg8al0rh92";
+};
+}

pkgs/development/libraries/science/math/tensorflow/prefetcher.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+version=1.14.0
+hashfile=binary-hashes.nix
+rm -f $hashfile
+echo "{" >> $hashfile
+echo "version = \"$version\";" >> $hashfile
+for sys in "linux" "darwin"; do
+    for tfpref in "cpu" "gpu"; do
+        for platform in "x86_64"; do
+            if [ $sys = "darwin" ] && [ $tfpref = "gpu" ]; then
+               continue
+            fi
+            url=https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-$tfpref-$sys-$platform-$version.tar.gz
+            hash=$(nix-prefetch-url $url)
+            echo "\"${tfpref}-${sys}-${platform}\" = {" >> $hashfile
+            echo "  url = \"$url\";" >> $hashfile
+            echo "  sha256 = \"$hash\";" >> $hashfile
+            echo "};" >> $hashfile
+        done
+    done
+done
+echo "}" >> $hashfile
+

pkgs/development/python-modules/tensorflow/bin.nix

@@ -35,15 +35,19 @@
 assert cudaSupport -> cudatoolkit != null
                    && cudnn != null
                    && nvidia_x11 != null;
+
+# unsupported combination
+assert ! (stdenv.isDarwin && cudaSupport);
+
 let
-  cudatoolkit_joined = symlinkJoin {
-    name = "unsplit_cudatoolkit";
-    paths = [ cudatoolkit.out
-              cudatoolkit.lib ];};
+  packages = import ./binary-hashes.nix;
+
+  variant = if cudaSupport then "-gpu" else "";
+  pname = "tensorflow${variant}";
 
 in buildPythonPackage rec {
-  pname = "tensorflow";
-  version = "1.14.0";
+  inherit pname;
+  inherit (packages) version;
   format = "wheel";
 
   src = let
@@ -52,8 +56,7 @@ in buildPythonPackage rec {
     platform = if stdenv.isDarwin then "mac" else "linux";
     unit = if cudaSupport then "gpu" else "cpu";
     key = "${platform}_py_${pyver}_${unit}";
-    dls = import (./. + "/tf${version}-hashes.nix");
-  in fetchurl dls.${key};
+  in fetchurl packages.${key};
 
   propagatedBuildInputs = [
     protobuf
@@ -86,9 +89,9 @@ in buildPythonPackage rec {
   # patchelf --shrink-rpath will remove the cuda libraries.
   postFixup = let
     rpath = stdenv.lib.makeLibraryPath
-      ([ stdenv.cc.cc.lib zlib ] ++ lib.optionals cudaSupport [ cudatoolkit_joined cudnn nvidia_x11 ]);
+      ([ stdenv.cc.cc.lib zlib ] ++ lib.optionals cudaSupport [ cudatoolkit.out cudatoolkit.lib cudnn nvidia_x11 ]);
   in
-  lib.optionalString (stdenv.isLinux) ''
+  lib.optionalString stdenv.isLinux ''
     rrPath="$out/${python.sitePackages}/tensorflow/:$out/${python.sitePackages}/tensorflow/contrib/tensor_forest/:${rpath}"
     internalLibPath="$out/${python.sitePackages}/tensorflow/python/_pywrap_tensorflow_internal.so"
     find $out \( -name '*.so' -or -name '*.so.*' \) -exec patchelf --set-rpath "$rrPath" {} \;
@@ -100,7 +103,7 @@ in buildPythonPackage rec {
     homepage = http://tensorflow.org;
     license = licenses.asl20;
     maintainers = with maintainers; [ jyp abbradar ];
-    platforms = with platforms; linux ++ lib.optionals (!cudaSupport) darwin;
+    platforms = [ "x86_64-linux" "x86_64-darwin" ];
     # Python 2.7 build uses different string encoding.
     # See https://github.com/NixOS/nixpkgs/pull/37044#issuecomment-373452253
     broken = stdenv.isDarwin && !isPy3k;

pkgs/development/python-modules/tensorflow/tf1.14.0-hashes.nix → pkgs/development/python-modules/tensorflow/binary-hashes.nix

@@ -1,4 +1,5 @@
 {
+version = "1.14.0";
 linux_py_27_cpu = {
   url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp27-none-linux_x86_64.whl";
   sha256 = "0yywdrfk97dh1bxhibspg0raz70fx9lcczj6xlimqy4xb60clx7k";