Build arbitrary Julia package environments in Nixpkgs

doc/languages-frameworks/index.md

@@ -24,6 +24,7 @@ idris.section.md
 ios.section.md
 java.section.md
 javascript.section.md
+julia.section.md
 lisp.section.md
 lua.section.md
 maven.section.md

doc/languages-frameworks/julia.section.md

@@ -0,0 +1,54 @@
+# Julia {#language-julia}
+
+## Introduction {#julia-introduction}
+
+Nixpkgs includes Julia as the `julia` derivation. You can get specific versions by looking at the other `julia*` top-level derivations available. For example, `julia_19` corresponds to Julia 1.9. We also provide the current stable version as `julia-stable`, and an LTS version as `julia-lts`.
+
+Occasionally, a Julia version has been too difficult to build from source in Nixpkgs and has been fetched prebuilt instead. These Julia versions are differentiated with the `*-bin` suffix; for example, `julia-stable-bin`.
+
+## julia.withPackages {#julia-withpackage}
+
+The basic Julia derivations only provide the built-in packages that come with the distribution.
+
+You can build Julia environments with additional packages using the `julia.withPackages` command. This function accepts a list of strings representing Julia package names. For example, you can build a Julia environment with the `Plots` package as follows.
+
+```nix
+julia.withPackages ["Plots"]
+```
+
+Arguments can be passed using `.override`. For example:
+
+```nix
+(julia.withPackages.override {
+  precompile = false; # Turn off precompilation
+}) ["Plots"]
+```
+
+Here's a nice way to run a Julia environment with a shell one-liner:
+
+```sh
+nix-shell -p 'julia.withPackages ["Plots"]' --run julia
+```
+
+### Arguments {#julia-withpackage-arguments}
+
+* `precompile`: Whether to run `Pkg.precompile()` on the generated environment.
+
+  This will make package imports faster, but may fail in some cases. For example, there is an upstream issue with `Gtk.jl` that prevents precompilation from working in the Nix build sandbox, because the precompiled code tries to access a display. Packages like this will work fine if you build with `precompile=false`, and then precompile as needed once your environment starts. Defaults to `true`.
+
+* `extraLibs`: Extra library dependencies that will be placed on the `LD_LIBRARY_PATH` for Julia.
+
+  Should not be needed as we try to obtain library dependencies automatically using Julia's artifacts system.
+
+* `makeWrapperArgs`: Extra arguments to pass to the `makeWrapper` call which we use to wrap the Julia binary.
+* `setDefaultDepot`: Whether to automatically prepend `$HOME/.julia` to the `JULIA_DEPOT_PATH`.
+
+  This is useful because Julia expects a writable depot path as the first entry, which the one we build in Nixpkgs is not. If there's no writable depot, then Julia will show a warning and be unable to save command history logs etc. Defaults to `true`.
+
+* `packageOverrides`: Allows you to override packages by name by passing an alternative source.
+
+  For example, you can use a custom version of the `LanguageServer` package by passing `packageOverrides = { "LanguageServer" = fetchFromGitHub {...}; }`.
+
+* `augmentedRegistry`: Allows you to change the registry from which Julia packages are drawn.
+
+  This normally points at a special augmented version of the Julia [General packages registry](https://github.com/JuliaRegistries/General). If you want to use a bleeding-edge version to pick up the latest package updates, you can plug in a later revision than the one in Nixpkgs.

nixos/doc/manual/release-notes/rl-2405.section.md

@@ -14,6 +14,8 @@ In addition to numerous new and upgraded packages, this release has the followin
    - This can be disabled through the `environment.stub-ld.enable` option.
    - If you use `programs.nix-ld.enable`, no changes are needed. The stub will be disabled automatically.
 
+- Julia environments can now be built with arbitrary packages from the ecosystem using the `.withPackages` function. For example: `julia.withPackages ["Plots"]`.
+
 ## New Services {#sec-release-24.05-new-services}
 
 <!-- To avoid merge conflicts, consider adding your item at an arbitrary place in the list instead. -->

pkgs/development/compilers/julia/default.nix

@@ -0,0 +1,20 @@
+{ callPackage }:
+
+let
+  juliaWithPackages = callPackage ../../julia-modules {};
+
+  wrapJulia = julia: julia.overrideAttrs (oldAttrs: {
+    passthru = (oldAttrs.passthru or {}) // {
+      withPackages = juliaWithPackages.override { inherit julia; };
+    };
+  });
+
+in
+
+{
+  julia_16-bin = wrapJulia (callPackage ./1.6-bin.nix {});
+  julia_18-bin = wrapJulia (callPackage ./1.8-bin.nix {});
+  julia_19-bin = wrapJulia (callPackage ./1.9-bin.nix {});
+  julia_18 = wrapJulia (callPackage ./1.8.nix {});
+  julia_19 = wrapJulia (callPackage ./1.9.nix {});
+}

pkgs/development/julia-modules/default.nix

@@ -0,0 +1,193 @@
+{ lib
+, callPackage
+, runCommand
+, fetchFromGitHub
+, fetchgit
+, fontconfig
+, git
+, makeWrapper
+, writeText
+, writeTextFile
+, python3
+
+# Artifacts dependencies
+, fetchurl
+, glibc
+, pkgs
+, stdenv
+
+, julia
+
+# Special registry which is equal to JuliaRegistries/General, but every Versions.toml
+# entry is augmented with a Nix sha256 hash
+, augmentedRegistry ? callPackage ./registry.nix {}
+
+# Other overridable arguments
+, extraLibs ? []
+, precompile ? true
+, setDefaultDepot ? true
+, makeWrapperArgs ? ""
+, packageOverrides ? {}
+, makeTransitiveDependenciesImportable ? false # Used to support symbol indexing
+}:
+
+packageNames:
+
+let
+  util = callPackage ./util.nix {};
+
+in
+
+let
+  # Some Julia packages require access to Python. Provide a Nixpkgs version so it
+  # doesn't try to install its own.
+  pythonToUse = let
+    extraPythonPackages = ((callPackage ./extra-python-packages.nix { inherit python3; }).getExtraPythonPackages packageNames);
+  in (if extraPythonPackages == [] then python3
+      else util.addPackagesToPython python3 (map (pkg: lib.getAttr pkg python3.pkgs) extraPythonPackages));
+
+  # Start by wrapping Julia so it has access to Python and any other extra libs.
+  # Also, prevent various packages (CondaPkg.jl, PythonCall.jl) from trying to do network calls.
+  juliaWrapped = runCommand "julia-${julia.version}-wrapped" { buildInputs = [makeWrapper]; inherit makeWrapperArgs; } ''
+    mkdir -p $out/bin
+    makeWrapper ${julia}/bin/julia $out/bin/julia \
+      --suffix LD_LIBRARY_PATH : "${lib.makeLibraryPath extraLibs}" \
+      --set FONTCONFIG_FILE ${fontconfig.out}/etc/fonts/fonts.conf \
+      --set PYTHONHOME "${pythonToUse}" \
+      --prefix PYTHONPATH : "${pythonToUse}/${pythonToUse.sitePackages}" \
+      --set PYTHON ${pythonToUse}/bin/python $makeWrapperArgs \
+      --set JULIA_CONDAPKG_OFFLINE yes \
+      --set JULIA_CONDAPKG_BACKEND Null \
+      --set JULIA_PYTHONCALL_EXE "@PyCall"
+  '';
+
+  # If our closure ends up with certain packages, add others.
+  packageImplications = {
+    # Because we want to put PythonCall in PyCall mode so it doesn't try to download
+    # Python packages
+    PythonCall = ["PyCall"];
+  };
+
+  # Invoke Julia resolution logic to determine the full dependency closure
+  packageOverridesRepoified = lib.mapAttrs util.repoifySimple packageOverrides;
+  closureYaml = callPackage ./package-closure.nix {
+    inherit augmentedRegistry julia packageNames packageImplications;
+    packageOverrides = packageOverridesRepoified;
+  };
+
+  # Generate a Nix file consisting of a map from dependency UUID --> package info with fetchgit call:
+  # {
+  #   "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" = {
+  #     src = fetchgit {...};
+  #     name = "...";
+  #     version = "...";
+  #     treehash = "...";
+  #   };
+  #   ...
+  # }
+  dependencies = runCommand "julia-sources.nix" { buildInputs = [(python3.withPackages (ps: with ps; [toml pyyaml])) git]; } ''
+    python ${./python}/sources_nix.py \
+      "${augmentedRegistry}" \
+      '${lib.generators.toJSON {} packageOverridesRepoified}' \
+      "${closureYaml}" \
+      "$out"
+  '';
+
+  # Import the Nix file from the previous step (IFD) and turn each dependency repo into
+  # a dummy Git repository, as Julia expects. Format the results as a YAML map from
+  # dependency UUID -> Nix store location:
+  # {
+  #   "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3":"/nix/store/...-NaNMath.jl-0877504",
+  #   ...
+  # }
+  # This is also the point where we apply the packageOverrides.
+  dependencyUuidToInfo = import dependencies { inherit fetchgit; };
+  fillInOverrideSrc = uuid: info:
+    if lib.hasAttr info.name packageOverrides then (info // { src = lib.getAttr info.name packageOverrides; }) else info;
+  dependencyUuidToRepo = lib.mapAttrs util.repoifyInfo (lib.mapAttrs fillInOverrideSrc dependencyUuidToInfo);
+  dependencyUuidToRepoYaml = writeTextFile {
+    name = "dependency-uuid-to-repo.yml";
+    text = lib.generators.toYAML {} dependencyUuidToRepo;
+  };
+
+  # Given the augmented registry, closure info yaml, and dependency path yaml, construct a complete
+  # Julia registry containing all the necessary packages
+  dependencyUuidToInfoYaml = writeTextFile {
+    name = "dependency-uuid-to-info.yml";
+    text = lib.generators.toYAML {} dependencyUuidToInfo;
+  };
+  fillInOverrideSrc' = uuid: info:
+    if lib.hasAttr info.name packageOverridesRepoified then (info // { src = lib.getAttr info.name packageOverridesRepoified; }) else info;
+  overridesOnly = lib.mapAttrs fillInOverrideSrc' (lib.filterAttrs (uuid: info: info.src == null) dependencyUuidToInfo);
+  minimalRegistry = runCommand "minimal-julia-registry" { buildInputs = [(python3.withPackages (ps: with ps; [toml pyyaml])) git]; } ''
+    python ${./python}/minimal_registry.py \
+      "${augmentedRegistry}" \
+      "${closureYaml}" \
+      '${lib.generators.toJSON {} overridesOnly}' \
+      "${dependencyUuidToRepoYaml}" \
+      "$out"
+  '';
+
+  # Next, deal with artifacts. Scan each artifacts file individually and generate a Nix file that
+  # produces the desired Overrides.toml.
+  artifactsNix = runCommand "julia-artifacts.nix" { buildInputs = [(python3.withPackages (ps: with ps; [toml pyyaml]))]; } ''
+    python ${./python}/extract_artifacts.py \
+      "${dependencyUuidToRepoYaml}" \
+      "${closureYaml}" \
+      "${juliaWrapped}/bin/julia" \
+      "${if lib.versionAtLeast julia.version "1.7" then ./extract_artifacts.jl else ./extract_artifacts_16.jl}" \
+      '${lib.generators.toJSON {} (import ./extra-libs.nix)}' \
+      "$out"
+  '';
+
+  # Import the artifacts Nix to build Overrides.toml (IFD)
+  artifacts = import artifactsNix { inherit lib fetchurl pkgs glibc stdenv; };
+  overridesJson = writeTextFile {
+    name = "Overrides.json";
+    text = lib.generators.toJSON {} artifacts;
+  };
+  overridesToml = runCommand "Overrides.toml" { buildInputs = [(python3.withPackages (ps: with ps; [toml]))]; } ''
+    python ${./python}/format_overrides.py \
+      "${overridesJson}" \
+      "$out"
+  '';
+
+  # Build a Julia project and depot. The project contains Project.toml/Manifest.toml, while the
+  # depot contains package build products (including the precompiled libraries, if precompile=true)
+  projectAndDepot = callPackage ./depot.nix {
+    inherit closureYaml extraLibs overridesToml packageImplications precompile;
+    julia = juliaWrapped;
+    registry = minimalRegistry;
+    packageNames = if makeTransitiveDependenciesImportable
+      then lib.mapAttrsToList (uuid: info: info.name) dependencyUuidToInfo
+      else packageNames;
+  };
+
+in
+
+runCommand "julia-${julia.version}-env" {
+  buildInputs = [makeWrapper];
+
+  inherit julia;
+  inherit juliaWrapped;
+
+  # Expose the steps we used along the way in case the user wants to use them, for example to build
+  # expressions and build them separately to avoid IFD.
+  inherit dependencies;
+  inherit closureYaml;
+  inherit dependencyUuidToInfoYaml;
+  inherit dependencyUuidToRepoYaml;
+  inherit minimalRegistry;
+  inherit artifactsNix;
+  inherit overridesJson;
+  inherit overridesToml;
+  inherit projectAndDepot;
+} (''
+  mkdir -p $out/bin
+  makeWrapper ${juliaWrapped}/bin/julia $out/bin/julia \
+    --suffix JULIA_DEPOT_PATH : "${projectAndDepot}/depot" \
+    --set-default JULIA_PROJECT "${projectAndDepot}/project" \
+    --set-default JULIA_LOAD_PATH '@:${projectAndDepot}/project/Project.toml:@v#.#:@stdlib'
+'' + lib.optionalString setDefaultDepot ''
+  sed -i '2 i\JULIA_DEPOT_PATH=''${JULIA_DEPOT_PATH-"$HOME/.julia"}' $out/bin/julia
+'')

pkgs/development/julia-modules/depot.nix

@@ -0,0 +1,85 @@
+{ lib
+, runCommand
+
+, cacert
+, curl
+, git
+, julia
+, python3
+
+, closureYaml
+, extraLibs
+, overridesToml
+, packageNames
+, packageImplications
+, precompile
+, registry
+}:
+
+runCommand "julia-depot" {
+    nativeBuildInputs = [curl git julia (python3.withPackages (ps: with ps; [pyyaml]))] ++ extraLibs;
+    inherit precompile registry;
+  } ''
+  export HOME=$(pwd)
+
+  echo "Building Julia depot and project with the following inputs"
+  echo "Julia: ${julia}"
+  echo "Registry: $registry"
+  echo "Overrides ${overridesToml}"
+
+  mkdir -p $out/project
+  export JULIA_PROJECT="$out/project"
+
+  mkdir -p $out/depot/artifacts
+  export JULIA_DEPOT_PATH="$out/depot"
+  cp ${overridesToml} $out/depot/artifacts/Overrides.toml
+
+  # These can be useful to debug problems
+  # export JULIA_DEBUG=Pkg
+  # export JULIA_DEBUG=loading
+
+  export JULIA_SSL_CA_ROOTS_PATH="${cacert}/etc/ssl/certs/ca-bundle.crt"
+
+  # Only precompile if configured to below
+  export JULIA_PKG_PRECOMPILE_AUTO=0
+
+  # Prevent a warning where Julia tries to download package server info
+  export JULIA_PKG_SERVER=""
+
+  # See if we need to add any extra package names based on the closure
+  # and the packageImplications. We're using the full closure YAML here since
+  # it's available, which is slightly weird, but it should work just as well
+  # for finding the extra packages we need to add
+  python ${./python}/find_package_implications.py "${closureYaml}" '${lib.generators.toJSON {} packageImplications}' extra_package_names.txt
+
+  # git config --global --add safe.directory '/nix'
+  export JULIA_PKG_USE_CLI_GIT="true"
+
+  julia -e ' \
+    import Pkg
+    import Pkg.Types: PRESERVE_NONE
+
+    Pkg.Registry.add(Pkg.RegistrySpec(path="${registry}"))
+
+    input = ${lib.generators.toJSON {} packageNames} ::Vector{String}
+
+    if isfile("extra_package_names.txt")
+      append!(input, readlines("extra_package_names.txt"))
+    end
+
+    input = unique(input)
+
+    if !isempty(input)
+      println("Adding packages: " * join(input, " "))
+      Pkg.add(input; preserve=PRESERVE_NONE)
+      Pkg.instantiate()
+
+      if "precompile" in keys(ENV) && ENV["precompile"] != "0" && ENV["precompile"] != ""
+        Pkg.precompile()
+      end
+    end
+
+    # Remove the registry to save space
+    Pkg.Registry.rm("General")
+  '
+''

pkgs/development/julia-modules/extra-libs.nix

@@ -0,0 +1,15 @@
+# A map from a Julia package (typically a JLL package) to extra libraries
+# that they require from Nix.
+# The libraries should be strings evaluated in a "with pkgs" context.
+
+{
+  # Qt5Base_jll
+  # Needs access to dbus or you get "Cannot find libdbus-1 in your system"
+  # Repro: build environment with ["Plots"]
+  # > using Plots; plot(cos, 0, 2pi)
+  "ea2cea3b-5b76-57ae-a6ef-0a8af62496e1" = ["dbus.lib"];
+
+  # Qt6Base_jll
+  # Same reason as Qt5Base_jll
+  "c0090381-4147-56d7-9ebc-da0b1113ec56" = ["dbus.lib"];
+}