diff --git a/doc/builders/special.md b/doc/builders/special.md index 6d07fa87f3f3e..47625d8f2e3d1 100644 --- a/doc/builders/special.md +++ b/doc/builders/special.md @@ -6,6 +6,7 @@ This chapter describes several special builders. special/fhs-environments.section.md special/makesetuphook.section.md special/mkshell.section.md +special/runinmkshell.section.md special/darwin-builder.section.md special/vm-tools.section.md ``` diff --git a/doc/builders/special/runinmkshell.section.md b/doc/builders/special/runinmkshell.section.md new file mode 100644 index 0000000000000..09587cf7176ef --- /dev/null +++ b/doc/builders/special/runinmkshell.section.md @@ -0,0 +1,89 @@ +# pkgs.runInMkShell {#sec-pkgs-runInMkShell} + +`pkgs.runInMkShell` is a function that generates a wrapper script that +receives the command to be run as arguments and then run it inside a special +environment created with [`mkShell`](#sec-pkgs-mkShell). + +It's basically like the `--run` flag of `nix-shell`, but using Nix expressions. + +`nix-shell` works by setting some specific environment variables and then +source the setup script from stdenv. + +The setup script from stdenv implements primitives such as the phase and hook +system that allows modular logic for derivations. + +Because `nix-shell` is coupled with this behaviour that is implemented in +nixpkgs it isn't used in the "nix3 cli" (in this case, `nix develop` and `nix +shell`) that uses a much simpler but incomplete approach: just add the +derivations bin folder to $PATH. + +To have the `nix-shell` behaviour in a `nix3` utility one must wrap a list of +programs in `mkShell`, which is what `nix-shell` does under the hood +automatically. + +And this function modularizes it to a wrapper script. It creates a script that +runs the command passed as arguments in an environment built the same way as +nix-shell does. + +The generated script can also be embedded in other scripts using the `source` +command. + +## Usage {#sec-pkgs-runInMkShell-usage} + +```nix +{ pkgs ? import { } }: +pkgs.runInMkShell { + shell = pkgs.bash + "/bin/bash" + + name = "demo-environment"; + + drv = pkgs.mkShell { + buildInputs = with pkgs.python3Packages; [ numpy pandas ]; + }; + + prelude = '' + echo Hello world + ''; +}; +``` + +## Attributes {#sec-pkgs-runInMkShell-attributes} +* `shell` (default: `${pkgs.bash} + "/bin/bash"`): Which shell to use as the + script shebang. +* `name` (default: `"${drv.name}-wrapper"`): Name of the derivation for the + script. +* `drv`: Result of [`mkShell`](#sec-pkgs-mkShell). Can be also only the + attrset of the arguments for `mkShell` or the list of the packages. +* `prelude`: Code to be pasted just before the script runs the command + passed with the arguments. It has access to the arguments using the + standard variables such as `$@`. + +## Function result {#sec-pkgs-runInMkShell-result} + +This function will always generate a folder with a executable script in bin. + +The absolute path of the executable script can always be obtained by using +`lib.getExe`. + +This executable script will accept a command as the arguments and will setup an +environment like `nix-shell` does, then run the code in `prelude`, and lastly +the command passed as parameter. + +Running it without parameters does nothing by default. + +Sourcing the script will run everything but the command that would be passed as +argument. + +Let the output of the code [right above](#sec-pkgs-runInMkShell-usage) be +defined as `$script`, an example usage of the script can be: + +```shell +$script python -c 'import numpy as np; import pandas as pd; print(np, pd)' +``` + +In this case, the script will print "Hello world" because of the prelude, and +the representations of `np` and `pd` because of Python. + +More usage examples can be found in the function tests at +`pkgs/build-support/runinmkshell/tests.nix`. All the tests expose the script +generated in the test. diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix index 70fd3635b7454..d316a0f2ef4e8 100644 --- a/pkgs/build-support/docker/default.nix +++ b/pkgs/build-support/docker/default.nix @@ -19,6 +19,7 @@ , pigz , rsync , runCommand +, runInMkShell , runtimeShell , shadow , skopeo @@ -1234,4 +1235,111 @@ rec { passthru = { inherit (stream) imageTag; }; nativeBuildInputs = [ pigz ]; } "${stream} | pigz -nTR > $out"; + + # This function streams a docker image that runs the arguments inside a nix-shell wrapper + streamWrapperImage = lib.makeOverridable ( + { # The derivation whose environment this docker image should be based on + drv + , # Image name + name ? drv.name + "-env" + , # Arguments for the wrapper script + args ? [ ] + , # User id to run the container as. Defaults to 1000, because many + # binaries don't like to be run as root + uid ? 1000 + , # Group id to run the container as, see also uid + gid ? 1000 + , # Commands to be run right before the payload command runs + prelude ? "" + , # The home directory of the user + homeDirectory ? "/build" + , # The temporary directory + temporaryDirectory ? "/tmp" + }@rest: + + let + + normalizedRest = builtins.removeAttrs rest [ + "drv" + "args" + "uid" + "gid" + "homeDirectory" + "temporaryDirectory" + "prelude" + ]; + + wrapper = runInMkShell { + name = name + "-wrapper"; + inherit drv prelude; + }; + + # Environment variables set in the image + envVars = { + + # Root certificates for internet access + SSL_CERT_FILE = "${cacert}/etc/ssl/certs/ca-bundle.crt"; + + # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/build/local-derivation-goal.cc#L1040-L1044 + NIX_STORE = storeDir; + + # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/build/local-derivation-goal.cc#L1032-L1038 + HOME = homeDirectory; + + # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/build/local-derivation-goal.cc#L1012-L1013 + TMPDIR = temporaryDirectory; + TEMPDIR = temporaryDirectory; + TMP = temporaryDirectory; + TEMP = temporaryDirectory; + + # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/build/local-derivation-goal.cc#L1046-L1047 + # TODO: Make configurable? + NIX_BUILD_CORES = "1"; + + # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/build/local-derivation-goal.cc#L1076-L1077 + TERM = "xterm-256color"; + }; + + in streamLayeredImage (normalizedRest // { + contents = (normalizedRest.contents or []) ++ [ + binSh + usrBinEnv + (fakeNss.override { + # Allows programs to look up the build user's home directory + # https://github.com/NixOS/nix/blob/ffe155abd36366a870482625543f9bf924a58281/src/libstore/build/local-derivation-goal.cc#L906-L910 + # Slightly differs however: We use the passed-in homeDirectory instead of sandboxBuildDir. + # We're doing this because it's arguably a bug in Nix that sandboxBuildDir is used here: https://github.com/NixOS/nix/issues/6379 + extraPasswdLines = [ + "nixbld:x:${toString uid}:${toString gid}:Build user:${homeDirectory}:/noshell" + ]; + extraGroupLines = [ + "nixbld:!:${toString gid}:" + ]; + }) + ]; + fakeRootCommands = '' + # Effectively a single-user installation of Nix, giving the user full + # control over the Nix store. Needed for building the derivation this + # shell is for, but also in case one wants to use Nix inside the + # image + mkdir -p ./nix/{store,var/nix} ./etc/nix + chown -R ${toString uid}:${toString gid} ./nix ./etc/nix + + # Gives the user control over the build directory + mkdir -p .${homeDirectory} + chown -R ${toString uid}:${toString gid} .${homeDirectory} + + mkdir -p .${temporaryDirectory} + chown -R ${toString uid}:${toString gid} .${temporaryDirectory} + ''; + + config = (normalizedRest.config or {}) // { + # Run this image as the given uid/gid + User = "${toString uid}:${toString gid}"; + WorkingDir = homeDirectory; + Env = (lib.mapAttrsToList (name: value: "${name}=${toString value}") envVars) ++ (normalizedRest.config.Env or []); + Entrypoint = [ (lib.getExe wrapper) ] ++ args; + }; + })); } + diff --git a/pkgs/build-support/runinmkshell/default.nix b/pkgs/build-support/runinmkshell/default.nix new file mode 100644 index 0000000000000..1b80a32c06ef9 --- /dev/null +++ b/pkgs/build-support/runinmkshell/default.nix @@ -0,0 +1,105 @@ +{ mkShell +, bash +, coreutils +, lib +, writeScript +, writeScriptBin +, writeText +}: + +# This function creates a wrapper script that runs a command in a shell context. +{ # The shell that will execute the wrapper script + shell ? bash + "/bin/bash" +, # The result of calling mkShell + drv +, # Which commands to run before the payload call + prelude ? "" +, # Name for the script derivation + name ? null +, ... +}: + +let + shellDrv = + if lib.isDerivation drv then drv + else if lib.isList drv then mkShell { buildInputs = drv; } + else mkShell drv; + + drvName = if name != null then name else "${shellDrv.name}-wrapper"; + + # This function closely mirrors what this Nix code does: + # https://github.com/NixOS/nix/blob/2.8.0/src/libexpr/primops.cc#L1102 + # https://github.com/NixOS/nix/blob/2.8.0/src/libexpr/eval.cc#L1981-L2036 + stringValue = value: + # We can't just use `toString` on all derivation attributes because that + # would not put path literals in the closure. So we explicitly copy + # those into the store here + if builtins.typeOf value == "path" then "${value}" + else if builtins.typeOf value == "list" then toString (map stringValue value) + else toString value; + + # A binary that calls the command to build the derivation + builder = writeScript "buildDerivation" '' + exec ${lib.escapeShellArg (stringValue shellDrv.drvAttrs.builder)} ${lib.escapeShellArgs (map stringValue shellDrv.drvAttrs.args)} + ''; + + # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/build/local-derivation-goal.cc#L992-L1004 + drvEnv = lib.mapAttrs' (name: value: + let str = stringValue value; + in if lib.elem name (shellDrv.drvAttrs.passAsFile or []) + then lib.nameValuePair "${name}Path" (writeText "pass-as-text-${name}" str) + else lib.nameValuePair name str + ) shellDrv.drvAttrs // + # A mapping from output name to the nix store path where they should end up + # https://github.com/NixOS/nix/blob/2.8.0/src/libexpr/primops.cc#L1253 + lib.genAttrs shellDrv.outputs (output: builtins.unsafeDiscardStringContext shellDrv.${output}.outPath); + + staticPath = "${dirOf shell}:${lib.makeBinPath [ builder coreutils ]}"; + + env = drvEnv // { + # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/build/local-derivation-goal.cc#L1027-L1030 + # PATH = "/path-not-set"; + # Allows calling bash and `buildDerivation` as the Cmd + PATH = staticPath; + }; + + variablePrelude = lib.pipe env [ + (builtins.mapAttrs (k: v: "export ${k}=${lib.escapeShellArg v}")) + (builtins.attrValues) + (builtins.concatStringsSep "\n") + ]; + + entrypointScript = writeScriptBin drvName '' + #!${shell} + unset PATH + dontAddDisableDepTrack=1 + + ${variablePrelude} + # Required or source setup fails + NIX_BUILD_TOP=$(mktemp -d) + + # TODO: https://github.com/NixOS/nix/blob/2.8.0/src/nix-build/nix-build.cc#L506 + [ -e $stdenv/setup ] && source $stdenv/setup + PATH=${staticPath}:"$PATH" + SHELL=${lib.escapeShellArg shell} + BASH=${lib.escapeShellArg shell} + set +e + [ -n "$PS1" -a -z "$NIX_SHELL_PRESERVE_PROMPT" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] ' + if [ "$(type -t runHook)" = function ]; then + runHook shellHook + fi + unset NIX_ENFORCE_PURITY + shopt -u nullglob + shopt -s execfail + ${prelude} + + if [[ "${"$"}{BASH_SOURCE[0]}" == "${"$"}{0}" ]]; then + "$@" + fi + ''; +in entrypointScript.overrideAttrs (old: { + passthru = ((shellDrv.passthru or {}) // { + inherit shellDrv; + }); + meta = (old.meta // (shellDrv.meta or {})); +}) diff --git a/pkgs/build-support/runinmkshell/tests.nix b/pkgs/build-support/runinmkshell/tests.nix new file mode 100644 index 0000000000000..7e02b4ba346ee --- /dev/null +++ b/pkgs/build-support/runinmkshell/tests.nix @@ -0,0 +1,107 @@ +{ runInMkShell +, mkShell +, runCommand +, lib +, pkgs +, writeText +, writeShellScriptBin +}: + +let + mkTest = { + # passed directly as name parameter to runInMkShell + name ? "test" + , # rest of flags for runInMkShell + args ? {} + , # can be either a shell script as string with $script or a list of arguments to $script + command ? [] + , # passed directly as drv parameter to runInMkShell + drv + , touchOut ? false + }: + let + # generated wrapper script + script = runInMkShell (args // { inherit drv name; }); + + payload = if lib.isList command then "$script ${lib.escapeShellArgs command}" else toString command; + in runCommand "runInMkShell-${name}" { + passthru = { inherit script; }; + } '' + script=${lib.getExe script} + + ${payload} + + ${lib.optionalString touchOut "touch $out"} + ''; + +in lib.recurseIntoAttrs (lib.mapAttrs (k: v: mkTest (v // { name = k; })) { + + identity = { # as basic as possible test + command = [ "true" ]; + drv = []; + touchOut = true; + }; + test-the-test = { # not using the list form because it automatically creates $out + command = '' + $script test-payload 2 ${placeholder "out"} + ''; + drv = [ # this way we can confirm that the wrapper is receiving the args and reacting properly + (writeShellScriptBin "test-payload" '' + [ $1 -eq 2 ] + echo Inner script called + touch $2 + '') + ]; + }; + + bc = { # note that you need to pass the command to run inside the wrapper context + command = "echo 2+2 | $script bc > $out"; + drv = [ pkgs.bc ]; + }; + + python-numpy = { # note that in nix-shell python is propagated from the library + command = [ "python" (writeText "script.py" '' + import numpy as np + from sys import argv + + d = np.ones(10, dtype='int') + + with open(argv[1], "w") as f: + print(d, file=f) + '') (placeholder "out") ]; + drv = [ pkgs.python3Packages.numpy ]; + }; + + bring-variables = { # extra variables can be passed as the other mkShell parameters + command = '' + theVar=$($script sh -c 'echo $theVar') + [ $theVar == 2 ] && touch $out + ''; + drv = { + theVar = 2; + }; + }; + + bring-variables-long = { # the same previous test can also be written using the long form + command = '' + theVar=$($script sh -c 'echo $theVar') + [ $theVar == 2 ] && touch $out + ''; + drv = mkShell { + theVar = 2; + }; + }; + + hook-simple = { # the shell primitives available in mkDerivation are also available + command = [ "eval" '' + outDir=${placeholder "out"} + runHook mkShellTestingHook + '']; + drv = { + mkShellTestingHook = '' + touch $outDir + ''; + }; + }; + +}) diff --git a/pkgs/test/default.nix b/pkgs/test/default.nix index 4cc9ecc0db3e9..f0d21540904ef 100644 --- a/pkgs/test/default.nix +++ b/pkgs/test/default.nix @@ -153,6 +153,8 @@ with pkgs; dotnet = recurseIntoAttrs (callPackages ./dotnet { }); + runInMkShell = callPackage ../build-support/runinmkshell/tests.nix { }; + makeHardcodeGsettingsPatch = callPackage ./make-hardcode-gsettings-patch { }; makeWrapper = callPackage ./make-wrapper { }; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index f4176c9d68d73..9980ba9a49484 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -1306,6 +1306,12 @@ with pkgs; mkBinaryCache = callPackage ../build-support/binary-cache { }; mkShell = callPackage ../build-support/mkshell { }; + + runInMkShell = (callPackage ../build-support/runinmkshell { }) + // { + tests = pkgs.tests.runInMkShell; + }; + mkShellNoCC = mkShell.override { stdenv = stdenvNoCC; }; mokutil = callPackage ../tools/security/mokutil { };