[WIP] devShellTools.buildShellEnv

pkgs/build-support/dev-shell-tools/default.nix

@@ -1,5 +1,8 @@
 {
   lib,
+  runtimeShell,
+  bashInteractive,
+  stdenv,
   writeTextFile,
 }:
 let
@@ -72,4 +75,101 @@ rec {
     # https://github.com/NixOS/nix/blob/2.8.0/src/libexpr/primops.cc#L1253
     lib.genAttrs outputList (output: builtins.unsafeDiscardStringContext outputMap.${output}.outPath);
 
+  toBashEnv =
+    { env }:
+    lib.concatStringsSep "\n" (
+      lib.mapAttrsToList (
+        name: value:
+        if lib.isValidPosixName name then ''export ${name}=${lib.escapeShellArg value}'' else ""
+      ) env
+    );
+
+  buildShellEnv =
+    {
+      drvAttrs,
+      promptPrefix ? "build shell",
+      promptName ? null,
+    }:
+    let
+      name = drvAttrs.pname or drvAttrs.name or "shell";
+      env = unstructuredDerivationInputEnv { inherit drvAttrs; };
+    in
+    stdenv.mkDerivation (finalAttrs: {
+      name = "${name}-env";
+      passAsFile = [
+        "bashEnv"
+        "bashrc"
+        "runShell"
+      ];
+      bashEnv = toBashEnv { inherit env; };
+      bashrc = ''
+        export NIXPKGS_SHELL_TMP="$(mktemp -d --tmpdir nixpkgs-shell-${name}.XXXXXX)"
+        export TMPDIR="$NIXPKGS_SHELL_TMP"
+        export TEMPDIR="$NIXPKGS_SHELL_TMP"
+        export TMP="$NIXPKGS_SHELL_TMP"
+        export TEMP="$NIXPKGS_SHELL_TMP"
+
+        echo "Using TMPDIR=$TMPDIR"
+
+        source @envbash@
+
+        mkdir -p $TMP/outputs
+        for _output in $outputs; do
+          export "''${_output}=$TMP/outputs/''${_output}"
+        done
+
+        source @stdenv@/setup
+
+        # Set a distinct prompt to make it clear that we are in a build shell
+        case "$PS1" in
+          *"(build shell $name)"*)
+            echo "It looks like your running a build shell inside a build shell."
+            echo "It might work, but this is probably not what you want."
+            echo "You may want to exit your shell before loading a new one."
+            ;;
+        esac
+
+        # Prefix a line to the prompt to indicate that we are in a build shell
+        PS1=$"\n(\[\033[1;33m\]"${lib.escapeShellArg promptPrefix}$": \[\033[1;34m\]"${
+          if promptName != null then lib.escapeShellArg promptName else ''"$name"''
+        }"\[\033[1;33m\]\[\033[0m\]) $PS1"
+
+        runHook shellHook
+      '';
+      buildCommand = ''
+        mkdir -p $out/lib $out/bin
+        bashrc="$out/lib/bashrc"
+        envbash="$out/lib/env.bash"
+
+        mv "$bashEnvPath" "$envbash"
+        substitute "$bashrcPath" "$bashrc" \
+          --replace-fail "@envbash@" "$envbash" \
+          --replace-fail "@stdenv@" "$stdenv" \
+          ;
+
+        substitute ${./run-shell.sh} "$out/bin/run-shell" \
+          --replace-fail "@bashrc@" "$bashrc" \
+          --replace-fail "@runtimeShell@" "${runtimeShell}" \
+          --replace-fail "@bashInteractive@" "${bashInteractive}" \
+          ;
+
+        # NOTE: most other files are script for the source command, and not
+        #       standalone executables, so they should not be made executable.
+        chmod a+x $out/bin/run-shell
+      '';
+      preferLocalBuild = true;
+      passthru = {
+        # Work this as a shell environment, so that commands like `nix-shell`
+        # will be able to check it and use it correctly. This also lets Nix know
+        # to stop when the user requests pkg.devShell explicitly, or a different
+        # attribute containing a shell environment.
+        isShellEnv = true;
+        devShell = throw "You're trying to access the devShell attribute of a shell environment. We appreciate that this is very \"meta\" and interesting, but it's usually just not what you want. Most likely you've selected one `.devShell` to deep in an expression or on the command line. Try removing the last one.";
+      };
+      meta = {
+        description = "An environment similar to the build environment of ${name}";
+        # TODO longDescription
+        mainProgram = "run-shell";
+      };
+    });
 }

pkgs/build-support/dev-shell-tools/run-shell.sh

@@ -0,0 +1,87 @@
+#!@runtimeShell@
+
+# baked variables, not exported
+bashrc='@bashrc@'
+bash='@bashInteractive@/bin/bash'
+
+# detected variables and option defaults
+shell="$(basename $SHELL)"
+mode=interactive
+
+# remaining args
+args=()
+
+# parse arguments
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -c)
+      mode=commands
+      shift
+      break
+      ;;
+    *)
+      mode=exec
+      break
+      ;;
+  esac
+done
+
+while [[ $# -gt 0 ]]; do
+  args+=("$1")
+  shift
+done
+
+case "$mode" in
+  interactive)
+    ;;
+  commands)
+    if ${#args[@]} -eq 0; then
+      echo "nixpkgs: You've requested a command shell, but didn't provide a command."
+      echo "         Please provide a command to run."
+      exit 1
+    fi
+    ;;
+  exec)
+    ;;
+esac
+
+invokeWithArgs() {
+  bash -c 'source "'"$bashrc"'"; _script="$(shift)"; _args=("$@"); eval "$_script"' -- "$@"
+}
+
+# For bash, we'll run the interactive variant of the version Nixpkgs uses.
+# For other shells, version correctness can't be a goal, so it's best
+# to launch the user's shell from $SHELL. This also avoids bringing in
+# dependencies of which N-1 aren't needed. Keeps it quick.
+launchBash() {
+  case "$mode" in
+    interactive)
+      exec "$bash" --rcfile "$bashrc" "${args[@]}"
+      ;;
+    commands)
+      exec "$bash" -c 'source "'"$bashrc"'"; _script="$1"; shift; eval "$_script"' -- "${args[@]}"
+      ;;
+    exec)
+      exec "$bash" -c 'source "'"$bashrc"'"; "$@"' -- "${args[@]}"
+      ;;
+  esac
+}
+
+launchShell() {
+  case "$shell" in
+    bash|"")
+      launchBash
+      ;;
+    *)
+      (
+        echo "nixpkgs: I see that you weren't running bash. That's cool, but"
+        echo "         stdenv derivations are built with bash, so that's what"
+        echo "         I'll run for you by default. We'd love to have support"
+        echo "         for many shells, so PRs are welcome!"
+      ) >&2
+      launchBash
+      ;;
+  esac
+}
+
+launchShell

pkgs/build-support/dev-shell-tools/tests/default.nix

@@ -1,7 +1,11 @@
 {
+  bash,
+  coreutils,
   devShellTools,
   emptyFile,
+  gnugrep,
   lib,
+  runCommand,
   stdenv,
   hello,
   writeText,
@@ -187,4 +191,70 @@ lib.recurseIntoAttrs {
         "args"
       ]
     );
+
+  # derivation: Call it directly so that we get a minimal environment to run in
+  buildHelloInShell =
+    derivation {
+      inherit (stdenv.buildPlatform) system;
+      name = "buildHelloInShell";
+      builder = "${bash}/bin/bash";
+      PATH = lib.makeBinPath [
+        coreutils
+        gnugrep
+      ];
+      args = [
+        "-euo"
+        "pipefail"
+        "-c"
+        ''
+          ${lib.getExe hello.devShell} -c 'genericBuild && ln -s $out "'"$PWD"'/result"'
+          ls -al
+          ./result/bin/hello | grep -i 'hello, world'
+          (set -x; [[ ! -e $out ]])
+          touch $out
+        ''
+      ];
+    }
+    // {
+      # Required package attributes for Nixpkgs (CI)
+      meta = { };
+    };
+
+  various =
+    let
+      hello2 = hello.overrideAttrs (o: {
+        shellHook = ''
+          echo 'shellHook says hi :)'
+        '';
+      });
+      inherit (hello2) devShell;
+    in
+    derivation {
+      inherit (stdenv.buildPlatform) system;
+      name = "various";
+      builder = "${bash}/bin/bash";
+      PATH = lib.makeBinPath [
+        coreutils
+        gnugrep
+      ];
+      args = [
+        "-euo"
+        "pipefail"
+        "-c"
+        ''
+          ${lib.getExe devShell} -c 'echo "hello, world"' \
+            | grep -F 'hello, world'
+          ${lib.getExe devShell} -c 'echo "hello, world"' \
+            | grep -F 'hello, world'
+
+          ls -al
+          (set -x; [[ ! -e $out ]])
+          touch $out
+        ''
+      ];
+    }
+    // {
+      # Required package attributes for Nixpkgs (CI)
+      meta = { };
+    };
 }

pkgs/build-support/mkshell/default.nix

@@ -2,6 +2,7 @@
   lib,
   stdenv,
   buildEnv,
+  devShellTools,
 }:
 
 # A special kind of derivation that is only meant to be consumed by the
@@ -16,6 +17,7 @@
   nativeBuildInputs ? [ ],
   propagatedBuildInputs ? [ ],
   propagatedNativeBuildInputs ? [ ],
+  passthru ? { },
   ...
 }@attrs:
 let
@@ -42,6 +44,7 @@ let
 in
 
 stdenv.mkDerivation (
+  finalAttrs:
   {
     inherit name;
 
@@ -68,6 +71,33 @@ stdenv.mkDerivation (
     '';
 
     preferLocalBuild = true;
+
+    passthru = {
+      devShell =
+        let
+          inherit (finalAttrs.finalPackage) drvAttrs;
+        in
+        devShellTools.buildShellEnv {
+          inherit drvAttrs;
+
+          # The default prefix is "build shell", but this shell is not derived
+          # directly from a derivation, so we set a more generic title.
+          promptPrefix = "nix";
+
+          # Change the default name
+          promptName =
+            if
+              # name was passed originally
+              attrs ? name
+              # or with overrideAttrs
+              || drvAttrs.name != "nix-shell"
+            then
+              null
+            else
+              "mkShell";
+        };
+    }
+    // passthru;
   }
   // rest
 )

pkgs/development/haskell-modules/generic-builder.nix

@@ -11,6 +11,7 @@
   ghcWithHoogle,
   ghcWithPackages,
   nodejs,
+  devShellTools,
 }:
 
 let
@@ -1045,6 +1046,8 @@ lib.fix (
             // env';
           } "echo $nativeBuildInputs $buildInputs > $out";
 
+        # Specialise the devShell attribute, so we get our improved shell.
+        devShell = env.devShell;
         env = envFunc { };
 
       };

pkgs/stdenv/booter.nix

@@ -97,11 +97,13 @@ let
     let
       args = stageFun prevStage;
       args' = args // {
-        stdenv = args.stdenv // {
-          # For debugging
-          __bootPackages = prevStage;
-          __hatPackages = nextStage;
-        };
+        stdenv = args.stdenv.override (prevArgs: {
+          # Extra package attributes for debugging
+          extraAttrs = prevArgs.extraAttrs or { } // {
+            __bootPackages = prevStage;
+            __hatPackages = nextStage;
+          };
+        });
       };
       thisStage =
         if args.__raw or false then