runInMkShell: init

doc/builders/special.md

@@ -6,6 +6,7 @@ This chapter describes several special builders.
 special/fhs-environments.section.md
 special/makesetuphook.section.md
 special/mkshell.section.md
+special/runinmkshell.section.md
 special/darwin-builder.section.md
 special/vm-tools.section.md
 ```

doc/builders/special/runinmkshell.section.md

@@ -0,0 +1,89 @@
+# pkgs.runInMkShell {#sec-pkgs-runInMkShell}
+
+`pkgs.runInMkShell` is a function that generates a wrapper script that
+receives the command to be run as arguments and then run it inside a special
+environment created with [`mkShell`](#sec-pkgs-mkShell).
+
+It's basically like the `--run` flag of `nix-shell`, but using Nix expressions.
+
+`nix-shell` works by setting some specific environment variables and then
+source the setup script from stdenv.
+
+The setup script from stdenv implements primitives such as the phase and hook
+system that allows modular logic for derivations.
+
+Because `nix-shell` is coupled with this behaviour that is implemented in
+nixpkgs it isn't used in the "nix3 cli" (in this case, `nix develop` and `nix
+shell`) that uses a much simpler but incomplete approach: just add the
+derivations bin folder to $PATH.
+
+To have the `nix-shell` behaviour in a `nix3` utility one must wrap a list of
+programs in `mkShell`, which is what `nix-shell` does under the hood
+automatically.
+
+And this function modularizes it to a wrapper script. It creates a script that
+runs the command passed as arguments in an environment built the same way as
+nix-shell does.
+
+The generated script can also be embedded in other scripts using the `source`
+command.
+
+## Usage {#sec-pkgs-runInMkShell-usage}
+
+```nix
+{ pkgs ? import <nixpkgs> { } }:
+pkgs.runInMkShell {
+  shell = pkgs.bash + "/bin/bash"
+
+  name = "demo-environment";
+
+  drv = pkgs.mkShell {
+    buildInputs = with pkgs.python3Packages; [ numpy pandas ];
+  };
+
+  prelude = ''
+    echo Hello world
+  '';
+};
+```
+
+## Attributes {#sec-pkgs-runInMkShell-attributes}
+* `shell` (default: `${pkgs.bash} + "/bin/bash"`): Which shell to use as the
+ script shebang.
+* `name` (default: `"${drv.name}-wrapper"`): Name of the derivation for the
+ script.
+* `drv`: Result of [`mkShell`](#sec-pkgs-mkShell). Can be also only the
+ attrset of the arguments for `mkShell` or the list of the packages.
+* `prelude`: Code to be pasted just before the script runs the command
+ passed with the arguments. It has access to the arguments using the
+ standard variables such as `$@`.
+
+## Function result {#sec-pkgs-runInMkShell-result}
+
+This function will always generate a folder with a executable script in bin.
+
+The absolute path of the executable script can always be obtained by using
+`lib.getExe`.
+
+This executable script will accept a command as the arguments and will setup an
+environment like `nix-shell` does, then run the code in `prelude`, and lastly
+the command passed as parameter.
+
+Running it without parameters does nothing by default.
+
+Sourcing the script will run everything but the command that would be passed as
+argument.
+
+Let the output of the code [right above](#sec-pkgs-runInMkShell-usage) be
+defined as `$script`, an example usage of the script can be:
+
+```shell
+$script python -c 'import numpy as np; import pandas as pd; print(np, pd)'
+```
+
+In this case, the script will print "Hello world" because of the prelude, and
+the representations of `np` and `pd` because of Python.
+
+More usage examples can be found in the function tests at
+`pkgs/build-support/runinmkshell/tests.nix`. All the tests expose the script
+generated in the test.

pkgs/build-support/docker/default.nix

@@ -19,6 +19,7 @@
 , pigz
 , rsync
 , runCommand
+, runInMkShell
 , runtimeShell
 , shadow
 , skopeo
@@ -1234,4 +1235,111 @@ rec {
         passthru = { inherit (stream) imageTag; };
         nativeBuildInputs = [ pigz ];
       } "${stream} | pigz -nTR > $out";
+
+  # This function streams a docker image that runs the arguments inside a nix-shell wrapper
+  streamWrapperImage = lib.makeOverridable (
+    { # The derivation whose environment this docker image should be based on
+      drv
+    , # Image name
+      name ? drv.name + "-env"
+    , # Arguments for the wrapper script
+      args  ? [ ]
+    , # User id to run the container as. Defaults to 1000, because many
+      # binaries don't like to be run as root
+      uid ? 1000
+    , # Group id to run the container as, see also uid
+      gid ? 1000
+    , # Commands to be run right before the payload command runs
+      prelude ? ""
+    , # The home directory of the user
+      homeDirectory ? "/build"
+    , # The temporary directory
+      temporaryDirectory ? "/tmp"
+    }@rest:
+
+      let
+
+        normalizedRest = builtins.removeAttrs rest [
+          "drv"
+          "args"
+          "uid"
+          "gid"
+          "homeDirectory"
+          "temporaryDirectory"
+          "prelude"
+        ];
+
+        wrapper = runInMkShell {
+          name = name + "-wrapper";
+          inherit drv prelude;
+        };
+
+        # Environment variables set in the image
+        envVars = {
+
+          # Root certificates for internet access
+          SSL_CERT_FILE = "${cacert}/etc/ssl/certs/ca-bundle.crt";
+
+          # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/build/local-derivation-goal.cc#L1040-L1044
+          NIX_STORE = storeDir;
+
+          # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/build/local-derivation-goal.cc#L1032-L1038
+          HOME = homeDirectory;
+
+          # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/build/local-derivation-goal.cc#L1012-L1013
+          TMPDIR = temporaryDirectory;
+          TEMPDIR = temporaryDirectory;
+          TMP = temporaryDirectory;
+          TEMP = temporaryDirectory;
+
+          # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/build/local-derivation-goal.cc#L1046-L1047
+          # TODO: Make configurable?
+          NIX_BUILD_CORES = "1";
+
+          # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/build/local-derivation-goal.cc#L1076-L1077
+          TERM = "xterm-256color";
+        };
+
+      in streamLayeredImage (normalizedRest // {
+          contents = (normalizedRest.contents or []) ++ [
+            binSh
+            usrBinEnv
+            (fakeNss.override {
+              # Allows programs to look up the build user's home directory
+              # https://github.com/NixOS/nix/blob/ffe155abd36366a870482625543f9bf924a58281/src/libstore/build/local-derivation-goal.cc#L906-L910
+              # Slightly differs however: We use the passed-in homeDirectory instead of sandboxBuildDir.
+              # We're doing this because it's arguably a bug in Nix that sandboxBuildDir is used here: https://github.com/NixOS/nix/issues/6379
+              extraPasswdLines = [
+                "nixbld:x:${toString uid}:${toString gid}:Build user:${homeDirectory}:/noshell"
+              ];
+              extraGroupLines = [
+                "nixbld:!:${toString gid}:"
+              ];
+            })
+          ];
+        fakeRootCommands = ''
+          # Effectively a single-user installation of Nix, giving the user full
+          # control over the Nix store. Needed for building the derivation this
+          # shell is for, but also in case one wants to use Nix inside the
+          # image
+          mkdir -p ./nix/{store,var/nix} ./etc/nix
+          chown -R ${toString uid}:${toString gid} ./nix ./etc/nix
+
+          # Gives the user control over the build directory
+          mkdir -p .${homeDirectory}
+          chown -R ${toString uid}:${toString gid} .${homeDirectory}
+
+          mkdir -p .${temporaryDirectory}
+          chown -R ${toString uid}:${toString gid} .${temporaryDirectory}
+        '';
+
+        config = (normalizedRest.config or {}) // {
+          # Run this image as the given uid/gid
+          User = "${toString uid}:${toString gid}";
+          WorkingDir = homeDirectory;
+          Env = (lib.mapAttrsToList (name: value: "${name}=${toString value}") envVars) ++ (normalizedRest.config.Env or []);
+          Entrypoint = [ (lib.getExe wrapper) ] ++ args;
+        };
+      }));
 }
+

pkgs/build-support/runinmkshell/default.nix

@@ -0,0 +1,105 @@
+{ mkShell
+, bash
+, coreutils
+, lib
+, writeScript
+, writeScriptBin
+, writeText
+}:
+
+# This function creates a wrapper script that runs a command in a shell context.
+{ # The shell that will execute the wrapper script
+  shell ? bash + "/bin/bash"
+, # The result of calling mkShell
+  drv
+, # Which commands to run before the payload call
+  prelude ? ""
+, # Name for the script derivation
+  name ? null
+, ...
+}:
+
+let
+  shellDrv =
+    if lib.isDerivation drv then drv
+      else if lib.isList drv then mkShell { buildInputs = drv; }
+      else mkShell drv;
+
+  drvName = if name != null then name else "${shellDrv.name}-wrapper";
+
+  # This function closely mirrors what this Nix code does:
+  # https://github.com/NixOS/nix/blob/2.8.0/src/libexpr/primops.cc#L1102
+  # https://github.com/NixOS/nix/blob/2.8.0/src/libexpr/eval.cc#L1981-L2036
+  stringValue = value:
+    # We can't just use `toString` on all derivation attributes because that
+    # would not put path literals in the closure. So we explicitly copy
+    # those into the store here
+    if builtins.typeOf value == "path" then "${value}"
+    else if builtins.typeOf value == "list" then toString (map stringValue value)
+    else toString value;
+
+  # A binary that calls the command to build the derivation
+  builder = writeScript "buildDerivation" ''
+    exec ${lib.escapeShellArg (stringValue shellDrv.drvAttrs.builder)} ${lib.escapeShellArgs (map stringValue shellDrv.drvAttrs.args)}
+  '';
+
+  # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/build/local-derivation-goal.cc#L992-L1004
+  drvEnv = lib.mapAttrs' (name: value:
+    let str = stringValue value;
+    in if lib.elem name (shellDrv.drvAttrs.passAsFile or [])
+    then lib.nameValuePair "${name}Path" (writeText "pass-as-text-${name}" str)
+    else lib.nameValuePair name str
+  ) shellDrv.drvAttrs //
+    # A mapping from output name to the nix store path where they should end up
+    # https://github.com/NixOS/nix/blob/2.8.0/src/libexpr/primops.cc#L1253
+    lib.genAttrs shellDrv.outputs (output: builtins.unsafeDiscardStringContext shellDrv.${output}.outPath);
+
+  staticPath = "${dirOf shell}:${lib.makeBinPath [ builder coreutils ]}";
+
+  env = drvEnv // {
+    # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/build/local-derivation-goal.cc#L1027-L1030
+    # PATH = "/path-not-set";
+    # Allows calling bash and `buildDerivation` as the Cmd
+    PATH = staticPath;
+  };
+
+  variablePrelude = lib.pipe env [
+    (builtins.mapAttrs (k: v: "export ${k}=${lib.escapeShellArg v}"))
+    (builtins.attrValues)
+    (builtins.concatStringsSep "\n")
+  ];
+
+  entrypointScript = writeScriptBin drvName ''
+    #!${shell}
+    unset PATH
+    dontAddDisableDepTrack=1
+
+    ${variablePrelude}
+    # Required or source setup fails
+    NIX_BUILD_TOP=$(mktemp -d)
+
+    # TODO: https://github.com/NixOS/nix/blob/2.8.0/src/nix-build/nix-build.cc#L506
+    [ -e $stdenv/setup ] && source $stdenv/setup
+    PATH=${staticPath}:"$PATH"
+    SHELL=${lib.escapeShellArg shell}
+    BASH=${lib.escapeShellArg shell}
+    set +e
+    [ -n "$PS1" -a -z "$NIX_SHELL_PRESERVE_PROMPT" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '
+    if [ "$(type -t runHook)" = function ]; then
+      runHook shellHook
+    fi
+    unset NIX_ENFORCE_PURITY
+    shopt -u nullglob
+    shopt -s execfail
+    ${prelude}
+
+    if [[ "${"$"}{BASH_SOURCE[0]}" == "${"$"}{0}" ]]; then
+      "$@"
+    fi
+  '';
+in entrypointScript.overrideAttrs (old: {
+  passthru = ((shellDrv.passthru or {}) // {
+    inherit shellDrv;
+  });
+  meta = (old.meta // (shellDrv.meta or {}));
+})