Julia build system

doc/languages-frameworks/index.xml

@@ -26,6 +26,7 @@
  <xi:include href="java.section.xml" />
  <xi:include href="javascript.section.xml" />
  <xi:include href="lua.section.xml" />
+ <xi:include href="julia.section.xml" />
  <xi:include href="maven.section.xml" />
  <xi:include href="nim.section.xml" />
  <xi:include href="ocaml.section.xml" />

doc/languages-frameworks/julia.section.md

@@ -0,0 +1,72 @@
+# Julia {#sec-julia}
+
+## Introduction {#ssec-julia-introduction}
+
+The [Julia programming language](https://julialang.org/) was developed
+with scientific applications in mind. It is distributed with a package
+manager and the vast majority of packages are available through the
+main package server
+[pkg.julialang.org](https://pkg.julialang.org). While the native
+package manager can be used to easily and declaratively install
+packages, several ones come with binary blobs (called artifacts in
+Julia parlance) that do not work in NixOS. To remedy this shortcoming,
+Julia packages can be installed as described below. This way the
+binary blobs are patched to work as expected and all software can be
+declared in a single file.
+
+### Installing Julia Packages {#sssec-installing-julia-packages}
+
+Most Julia compilers available in Nixpkgs have a passthru attribute
+called `withPackages`. This function permits creating Julia
+environments with sets of packages ready for use. Compared with the
+similarly named function available in other Nixpkgs language
+frameworks, the one of Julia takes two arguments. The first one is a
+function specifying a list of Julia packages available in Nixpkgs. The
+second one is also a function, and it's used to specify a list of
+extra package definitions not available in Nixpkgs. These definitions
+can be generated with the help of `julia2nix`. As an example, to
+install `julia-bin` with the Nixpkgs Julia package `Plots`, all its
+dependencies and the upstream package `CSTParser` with its dependency
+`Tokenize`, one can use the following command
+
+```ShellSession
+$ nix-shell -p 'julia-bin.withPackages
+   (p: with p; [ Plots ])
+   (p: with p; [
+     { pname = "Tokenize";
+       version = "0.5.25";
+       src = fetchurl {
+         url = "https://pkg.julialang.org/package/0796e94c-ce3b-5d07-9a54-7f471281c624/90538bf898832b6ebd900fa40f223e695970e3a5";
+         name = "julia-bin-1.8.3-Tokenize-0.5.25.tar.gz";
+         sha256 = "00437718f09d81958e86c2131f3f8b63e0975494e505f6c7b62f872a5a102f51";
+       };
+     }
+
+     { pname = "CSTParser";
+       version = "3.3.6";
+       src = fetchurl {
+         url = "https://pkg.julialang.org/package/00ebfdb7-1f24-5e51-bd34-a7502290713f/3ddd48d200eb8ddf9cb3e0189fc059fd49b97c1f";
+         name = "julia-bin-1.8.3-CSTParser-3.3.6.tar.gz";
+         sha256 = "647fc5588cb87362d216401a0a4124d41b80a85618a94098a737ad38ae5786c4";
+       };
+       requiredJuliaPackages = [ Tokenize ];
+     }
+   ])
+'
+```
+
+This is not very practical as a `nix-shell` command, but the same
+expression can be used in a `shell.nix` file, or other places
+expecting a Nix expression as described elsewhere in this
+manual. After executing this command and starting Julia, the packages
+can be directly imported with `using` or `import` without first having
+to use `import Pkg; Pkg.add(...)`.
+
+The list of Julia packages in Nixpkgs is accessible through the
+compiler passthru attribute set `juliaPackages`, for example
+`julia-bin.juliaPackages`.
+
+Note that the above upstream package definitions are addressed with an
+`url` specifying a unique package identifier (uuid) and the hash of
+the package content. For this reason we don't lose reproducibility by
+using upstream packages.

nixos/doc/manual/from_md/release-notes/rl-2305.section.xml

@@ -10,14 +10,25 @@
       In addition to numerous new and upgraded packages, this release
       has the following highlights:
     </para>
-    <itemizedlist spacing="compact">
+    <itemizedlist>
       <listitem>
         <para>
           Cinnamon has been updated to 5.6, see
           <link xlink:href="https://github.com/NixOS/nixpkgs/pull/201328#issue-1449910204">the
           pull request</link> for what is changed.
         </para>
       </listitem>
+      <listitem>
+        <para>
+          It’s now possible to make
+          <link xlink:href="https://julialang.org">Julia</link>
+          environments with a set of packages using e.g.
+          <literal>julia-bin.withPackages</literal>. Definitions for
+          packages in the main Julia package repository not available in
+          NixOS can be generated with <literal>julia2nix</literal> and
+          used directly with <literal>withPackages</literal>.
+        </para>
+      </listitem>
     </itemizedlist>
   </section>
   <section xml:id="sec-release-23.05-new-services">

nixos/doc/manual/release-notes/rl-2305.section.md

@@ -10,6 +10,8 @@ In addition to numerous new and upgraded packages, this release has the followin
 
 - Cinnamon has been updated to 5.6, see [the pull request](https://github.com/NixOS/nixpkgs/pull/201328#issue-1449910204) for what is changed.
 
+- It's now possible to make [Julia](https://julialang.org) environments with a set of packages using e.g. `julia-bin.withPackages`. Definitions for packages in the main Julia package repository not available in NixOS can be generated with `julia2nix` and used directly with `withPackages`.
+
 ## New Services {#sec-release-23.05-new-services}
 
 <!-- To avoid merge conflicts, consider adding your item at an arbitrary place in the list instead. -->

pkgs/development/compilers/julia/build-env.nix

@@ -0,0 +1,24 @@
+{ buildEnv
+, makeWrapper
+, extraPackages ? []
+, julia
+, computeRequiredJuliaPackages
+}:
+
+buildEnv {
+    name = "julia-env";
+    paths = computeRequiredJuliaPackages extraPackages;
+
+    nativeBuildInputs = [ makeWrapper ];
+
+    pathsToLink = ["/share/julia" ];
+
+    postBuild = ''
+      makeWrapper ${julia}/bin/julia $out/bin/julia \
+        --set JULIA_LOAD_PATH "$JULIA_LOAD_PATH:$out/share/julia/packages" \
+        --set JULIA_DEPOT_PATH "$JULIA_DEPOT_PATH:$out/share/julia"
+    '';
+
+    passthru = { inherit julia; };
+}
+

pkgs/development/compilers/julia/build-julia-package.nix

@@ -0,0 +1,161 @@
+{ lib
+, stdenv
+, config
+, patchelf
+, julia
+, computeRequiredJuliaPackages
+, computeJuliaDepotPath
+, computeJuliaLoadPath
+, computeJuliaArtifacts
+}:
+
+# Build an individual package
+{ fullPkgName ? "${attrs.pname}-${attrs.version}"
+
+, src
+
+, dontPatch ? false
+, patches ? []
+, patchPhase ? ""
+
+, enableParallelBuilding ? true
+# Build-time dependencies for the package, which were compiled for the system compiling this.
+, nativeBuildInputs ? []
+
+# Build-time dependencies for the package, which may not have been compiled for the system compiling this.
+, buildInputs ? []
+
+# Propagate build dependencies so in case we have A -> B -> C,
+# C can import package A propagated by B
+# Run-time dependencies for the package.
+, propagatedBuildInputs ? []
+
+# Julia packages that are required at runtime for this one.
+# These behave similarly to propagatedBuildInputs, where if
+# package A is needed by B, and C needs B, then C also requires A.
+# The main difference between these and propagatedBuildInputs is
+# during the package's installation into Julia, where all
+# requiredJuliaPackages are ALSO installed into Julia.
+, requiredJuliaPackages ? []
+
+, preBuild ? ""
+, meta ? {}
+, passthru ? {}
+
+# Julia packages can distribute 'Artifacts'.  These can be some data,
+# binary, ...  We Package 'Artifacts' as independent derivations. They
+# differ from the rest of the packages in that they don't get compiled
+# and the package providing it expect to find them in a specific location.
+, isJuliaArtifact ? false
+, ... } @ attrs:
+
+let
+  requiredJuliaPackages' = computeRequiredJuliaPackages requiredJuliaPackages;
+  juliaDepotPath = computeJuliaDepotPath requiredJuliaPackages';
+  juliaLoadPath = computeJuliaLoadPath requiredJuliaPackages';
+  juliaArtifacts = computeJuliaArtifacts requiredJuliaPackages';
+
+
+  # Must use attrs.nativeBuildInputs before they are removed by the removeAttrs
+  # below, or everything fails.
+  nativeBuildInputs' = [ julia patchelf ] ++ nativeBuildInputs;
+
+  # This step is required because when
+  # a = { test = [ "a" "b" ]; }; b = { test = [ "c" "d" ]; };
+  # (a // b).test = [ "c" "d" ];
+  # This used to mean that if a package defined extra nativeBuildInputs, it
+  # would override the ones for building a Julia package (Julia itself)
+  # causing everything to fail.
+  attrs' = builtins.removeAttrs attrs [ "nativeBuildInputs" ];
+
+in stdenv.mkDerivation ({
+  packageName = "${fullPkgName}";
+  # The name of the package ends up being
+  # "julia-version-package-version"
+  name = "${julia.pname}-${julia.version}-${fullPkgName}";
+
+  # This states that any package built with the function that this returns
+  # will be a julia package. This is used for ensuring other julia
+  # packages are installed into julia during the environment building phase.
+  isJuliaPackage = true;
+  # Julia Artifacts need to be treated specially.
+  isJuliaArtifact = isJuliaArtifact;
+
+  inherit src;
+
+  inherit dontPatch patches patchPhase;
+
+  dontConfigure = true;
+
+  enableParallelBuilding = enableParallelBuilding;
+
+  requiredJuliaPackages = requiredJuliaPackages';
+
+  nativeBuildInputs = nativeBuildInputs';
+
+  buildInputs = buildInputs ++ requiredJuliaPackages' ++ [ stdenv.cc.libc ];
+
+  propagatedBuildInputs = propagatedBuildInputs;
+
+  preUnpack = ''
+    mkdir ${attrs.pname}
+    cd ${attrs.pname}
+  '';
+
+  setSourceRoot = ''
+    sourceRoot=$PWD
+  '';
+
+  # Julia installs compiled cache in the first path in DEPOT_PATH.
+  # Packages are pre-complied when imported.
+  buildPhase = if isJuliaArtifact then ''
+    runHook preBuild
+
+    mkdir -p $out/share/julia/artifacts/${attrs.juliaPath}
+    cp -r * $out/share/julia/artifacts/${attrs.juliaPath}
+
+    runHook postBuild
+  '' else ''
+    runHook preBuild
+
+    mkdir -p $out/share/julia/packages/${attrs.pname}
+    cp -r * $out/share/julia/packages/${attrs.pname}
+
+    export JULIA_LOAD_PATH=$out/share/julia/packages:${juliaLoadPath}:$JULIA_LOAD_PATH
+    export JULIA_DEPOT_PATH=$out/share/julia:${julia}/share/julia:${juliaDepotPath}
+
+    mkdir -p $out/share/julia/artifacts
+    for path in ${lib.concatMapStringsSep " " (p: p + "/share/julia/artifacts/*") juliaArtifacts}; do
+        ln -s $path $out/share/julia/artifacts/$(basename $path);
+    done
+
+    pushd $out/share/julia/packages/${attrs.pname}
+    if [[ -f ./deps/build.jl ]]; then
+      pushd deps
+      $NIX_JULIA_PRECMD julia -- build.jl
+      popd
+    fi
+    $NIX_JULIA_PRECMD julia -e 'import ${attrs.pname}'
+    popd
+
+    # these may cause collisions
+    rm -r $out/share/julia/logs || true
+    rm -r $out/share/julia/scratchspaces || true
+
+    runHook postBuild
+  '';
+
+  # Need to install before building
+  dontInstall = true;
+
+  # Patch interpreter of bundled binary files
+  postFixup = if isJuliaArtifact  then ''
+    if [[ -d "$out"/share/julia/artifacts/${attrs.juliaPath}/bin ]]; then
+      for fn in $(echo $out/share/julia/artifacts/${attrs.juliaPath}/bin/*); do
+          isELF $fn && patchelf --set-interpreter ${stdenv.cc.bintools.dynamicLinker} $fn || true
+      done;
+    fi
+  '' else "";
+
+  inherit meta;
+} // attrs')

pkgs/development/compilers/julia/with-packages.nix

@@ -0,0 +1,68 @@
+{ lib, callPackage, julia, fetchurl }:
+
+# The function `withPackages` can be used to create a Julia environment
+# with a specified set of packages as shown by the following example
+#
+# pkgs.julia-bin.withPackages
+#   (p: with p; [ Plots ])
+#   (p: with p; [
+#     { pname = "Tokenize";
+#       version = "0.5.25";
+#       src = fetchurl {
+#         url = "https://pkg.julialang.org/package/0796e94c-ce3b-5d07-9a54-7f471281c624/90538bf898832b6ebd900fa40f223e695970e3a5";
+#         name = "julia-bin-1.8.3-Tokenize-0.5.25.tar.gz";
+#         sha256 = "00437718f09d81958e86c2131f3f8b63e0975494e505f6c7b62f872a5a102f51";
+#       };
+#     }
+#
+#     { pname = "CSTParser";
+#       version = "3.3.6";
+#       src = fetchurl {
+#         url = "https://pkg.julialang.org/package/00ebfdb7-1f24-5e51-bd34-a7502290713f/3ddd48d200eb8ddf9cb3e0189fc059fd49b97c1f";
+#         name = "julia-bin-1.8.3-CSTParser-3.3.6.tar.gz";
+#         sha256 = "647fc5588cb87362d216401a0a4124d41b80a85618a94098a737ad38ae5786c4";
+#       };
+#       requiredJuliaPackages = [ Tokenize ];
+#     }
+#   ])
+#
+# The first argument specifies a list of packages available in `<nixpkgs>`. The second one
+# a list of upstream packages specified by a package definition. These definitions can be
+# automatically generated by `julia2nix`. For all of them the package attribute name
+# is equal to the `pname` of the package.
+#
+# The reason for using the attribute `requiredJuliaPackages` rather than
+# `propagatedBuildInputs` is that the dependencies must be installed as well.
+#
+# Packages artifacts are specified in a similar way with the addition of the
+# `isArtifact` attribute. This is so because they need to be treated differently.
+
+let juliaPackages = lib.recurseIntoAttrs
+      (callPackage ../../../top-level/julia-packages.nix { inherit julia; }).pkgs;
+
+    withPackages = nix: upstream:
+      let nixPackages = (nix juliaPackages);
+          upstreamPackages = (packagesFromUpstream upstreamPkgsList);
+
+          packagesFromUpstream = ps: builtins.map (p: upstreamPkgs."${p.pname}") ps;
+
+          upstreamPkgsList = upstream upstreamPkgs;
+
+          upstreamPkgs = juliaPackages.juliaPackagesFromList upstreamPkgsList;
+
+      in callPackage ./build-env.nix {
+        inherit julia;
+        extraPackages = lib.unique nixPackages ++ upstreamPackages;
+        computeRequiredJuliaPackages = juliaPackages.computeRequiredJuliaPackages;
+      };
+
+    juliaWithPkgs = julia.overrideAttrs (finalAttrs: previousAttrs:
+      { passthru = let passthru' = { juliaPackages = juliaPackages;
+                                     withPackages = withPackages;
+                                   };
+                   in if previousAttrs ? passthru
+                      then previousAttrs.passthru // passthru'
+                      else passthru';
+      });
+
+in juliaWithPkgs