nixos/binfmt: Add support for using statically-linked QEMU (continuation of #160802)

lib/systems/default.nix

@@ -293,6 +293,10 @@ let
           then selectEmulator pkgs
           else throw "Don't know how to run ${final.config} executables.";
 
+        staticEmulator = pkgs:
+          if builtins.elem final.system pkgs.qemu-user-static.passthru.qemuUserPlatforms then
+            "${pkgs.qemu-user-static}/bin/qemu-${final.qemuArch}"
+          else null;
     }) // mapAttrs (n: v: v final.parsed) inspect.predicates
       // mapAttrs (n: v: v final.gcc.arch or "default") architectures.predicates
       // args // {

nixos/modules/system/boot/binfmt.nix

@@ -30,6 +30,7 @@ let
 
   getEmulator = system: (lib.systems.elaborate { inherit system; }).emulator pkgs;
   getQemuArch = system: (lib.systems.elaborate { inherit system; }).qemuArch;
+  getStaticEmulator = system: (lib.systems.elaborate { inherit system; }).staticEmulator pkgs;
 
   # Mapping of systems to “magicOrExtension” and “mask”. Mostly taken from:
   # - https://github.com/cleverca22/nixos-configs/blob/master/qemu.nix
@@ -277,39 +278,67 @@ in {
         '';
         type = types.listOf (types.enum (builtins.attrNames magics));
       };
+
+      preferStaticEmulators = mkOption {
+        default = false;
+        description = lib.mdDoc ''
+          Whether to use static emulators when available.
+
+          This enables the kernel to preload the emulator binaries when
+          the binfmt registrations are added, obviating the need to make
+          the emulator binaries available inside chroots and chroot-like
+          sandboxes.
+        '';
+        type = types.bool;
+      };
     };
   };
 
   config = {
+    assertions = lib.mapAttrsToList (name: reg: {
+      assertion = reg.fixBinary -> !reg.wrapInterpreterInShell;
+      message = "boot.binfmt.registrations.\"${name}\" cannot have fixBinary when the interpreter is invoked through a shell.";
+    }) cfg.registrations;
+
     boot.binfmt.registrations = builtins.listToAttrs (map (system: {
       name = system;
       value = { config, ... }: let
-        interpreter = getEmulator system;
+        staticEmulator = getStaticEmulator system;
+        interpreter =
+          if cfg.preferStaticEmulators && staticEmulator != null then staticEmulator
+          else getEmulator system;
         qemuArch = getQemuArch system;
 
-        preserveArgvZero = "qemu-${qemuArch}" == baseNameOf interpreter;
+        isQemu = "qemu-${qemuArch}" == baseNameOf interpreter;
+        isStaticEmulator = interpreter == staticEmulator;
+
         interpreterReg = let
           wrapperName = "qemu-${qemuArch}-binfmt-P";
           wrapper = pkgs.wrapQemuBinfmtP wrapperName interpreter;
         in
-          if preserveArgvZero then "${wrapper}/bin/${wrapperName}"
+          if isQemu && !isStaticEmulator then "${wrapper}/bin/${wrapperName}"
           else interpreter;
       in ({
-        preserveArgvZero = mkDefault preserveArgvZero;
+        preserveArgvZero = mkDefault isQemu;
 
         interpreter = mkDefault interpreterReg;
-        wrapInterpreterInShell = mkDefault (!config.preserveArgvZero);
-        interpreterSandboxPath = mkDefault (dirOf (dirOf config.interpreter));
+        fixBinary = mkDefault isStaticEmulator;
+        wrapInterpreterInShell = mkDefault (!config.preserveArgvZero && !config.fixBinary);
+        interpreterSandboxPath = mkDefault
+          (if config.fixBinary then null else dirOf (dirOf config.interpreter));
       } // (magics.${system} or (throw "Cannot create binfmt registration for system ${system}")));
     }) cfg.emulatedSystems);
     nix.settings = lib.mkIf (cfg.emulatedSystems != []) {
       extra-platforms = cfg.emulatedSystems ++ lib.optional pkgs.stdenv.hostPlatform.isx86_64 "i686-linux";
       extra-sandbox-paths = let
         ruleFor = system: cfg.registrations.${system};
         hasWrappedRule = lib.any (system: (ruleFor system).wrapInterpreterInShell) cfg.emulatedSystems;
-      in [ "/run/binfmt" ]
+        hasNonFixedRule = lib.any (system: !(ruleFor system).fixBinary) cfg.emulatedSystems;
+      in [ ]
+        ++ lib.optional hasNonFixedRule "/run/binfmt"
         ++ lib.optional hasWrappedRule "${pkgs.bash}"
-        ++ (map (system: (ruleFor system).interpreterSandboxPath) cfg.emulatedSystems);
+        ++ (lib.filter (p: p != null)
+          (map (system: (ruleFor system).interpreterSandboxPath) cfg.emulatedSystems));
     };
 
     environment.etc."binfmt.d/nixos.conf".source = builtins.toFile "binfmt_nixos.conf"

nixos/tests/systemd-binfmt.nix

@@ -87,4 +87,27 @@ in {
       ).lower()
     '';
   };
+
+  chroot = makeTest {
+    name = "systemd-binfmt-chroot";
+    nodes.machine = { pkgs, ... }: {
+      boot.binfmt.emulatedSystems = [
+        "aarch64-linux"
+      ];
+      boot.binfmt.preferStaticEmulators = true;
+
+      environment.systemPackages = [
+        (pkgs.writeShellScriptBin "test-chroot" ''
+          set -euo pipefail
+          mkdir -p /tmp/chroot
+          cp ${pkgs.pkgsCross.aarch64-multiplatform.pkgsStatic.busybox}/bin/busybox /tmp/chroot/busybox
+          chroot /tmp/chroot /busybox uname -m | grep aarch64
+        '')
+      ];
+    };
+    testScript = ''
+      machine.start()
+      machine.succeed("test-chroot")
+    '';
+  };
 }

pkgs/applications/virtualization/qemu/aio-find-static-library.patch

@@ -0,0 +1,26 @@
+diff --git a/meson.build b/meson.build
+index c9c3217ba4..1a8de03f4e 100644
+--- a/meson.build
++++ b/meson.build
+@@ -986,7 +986,8 @@ zlib = dependency('zlib', required: true)
+ libaio = not_found
+ if not get_option('linux_aio').auto() or have_block
+   libaio = cc.find_library('aio', has_headers: ['libaio.h'],
+-                           required: get_option('linux_aio'))
++                           required: get_option('linux_aio'),
++                           dirs: [get_option('linux_aio_path')])
+ endif
+
+ linux_io_uring_test = '''
+diff --git a/meson_options.txt b/meson_options.txt
+index 0a99a059ec..84d2449de9 100644
+--- a/meson_options.txt
++++ b/meson_options.txt
+@@ -186,6 +186,7 @@ option('libusb', type : 'feature', value : 'auto',
+        description: 'libusb support for USB passthrough')
+ option('linux_aio', type : 'feature', value : 'auto',
+        description: 'Linux AIO support')
++option('linux_aio_path', type: 'string', value: '', description: 'Path for libaio.a')
+ option('linux_io_uring', type : 'feature', value : 'auto',
+        description: 'Linux io_uring support')
+ option('lzfse', type : 'feature', value : 'auto',

pkgs/applications/virtualization/qemu/default.nix

@@ -31,7 +31,10 @@
 , uringSupport ? stdenv.isLinux, liburing
 , canokeySupport ? false, canokey-qemu
 , capstoneSupport ? !toolsOnly, capstone
+, pluginsSupport ? !stdenv.hostPlatform.isStatic
 , enableDocs ? true
+, enableTools ? true
+, enableBlobs ? true
 , hostCpuOnly ? false
 , hostCpuTargets ? (if toolsOnly
                     then [ ]
@@ -70,8 +73,9 @@ stdenv.mkDerivation (finalAttrs: {
     pkg-config flex bison dtc meson ninja
 
     # Don't change this to python3 and python3.pkgs.*, breaks cross-compilation
-    python3Packages.python python3Packages.sphinx python3Packages.sphinx-rtd-theme
+    python3Packages.python
   ]
+    ++ lib.optionals enableDocs [ python3Packages.sphinx python3Packages.sphinx-rtd-theme ]
     ++ lib.optionals gtkSupport [ wrapGAppsHook ]
     ++ lib.optionals hexagonSupport [ glib ]
     ++ lib.optionals stdenv.isDarwin [ sigtool ];
@@ -108,6 +112,7 @@ stdenv.mkDerivation (finalAttrs: {
     ++ lib.optionals capstoneSupport [ capstone ];
 
   dontUseMesonConfigure = true; # meson's configurePhase isn't compatible with qemu build
+  dontAddStaticConfigureFlags = true;
 
   outputs = [ "out" ] ++ lib.optional guestAgentSupport "ga";
   # On aarch64-linux we would shoot over the Hydra's 2G output limit.
@@ -136,7 +141,13 @@ stdenv.mkDerivation (finalAttrs: {
       revert = true;
     })
   ]
-  ++ lib.optional nixosTestRunner ./force-uid0-on-9p.patch;
+  ++ lib.optional nixosTestRunner ./force-uid0-on-9p.patch
+
+  ## FIXME: libaio does not provide a pkg-info file;
+  # and meson does not support static libraries lookup path using -L, relying on LIBRARY_PATH (https://github.com/mesonbuild/meson/issues/10172);
+  # and musl-gcc does not support LIBRARY_PATH overrides (https://www.openwall.com/lists/musl/2017/02/22/7);
+  # so we have to patch the meson.build to add the libaio path to the search path manually.
+  ++ lib.optional stdenv.hostPlatform.isStatic ./aio-find-static-library.patch;
 
   postPatch = ''
     # Otherwise tries to ensure /var/run exists.
@@ -160,7 +171,7 @@ stdenv.mkDerivation (finalAttrs: {
   configureFlags = [
     "--disable-strip" # We'll strip ourselves after separating debug info.
     (lib.enableFeature enableDocs "docs")
-    "--enable-tools"
+    (lib.enableFeature enableTools "tools")
     "--localstatedir=/var"
     "--sysconfdir=/etc"
     "--cross-prefix=${stdenv.cc.targetPrefix}"
@@ -184,7 +195,15 @@ stdenv.mkDerivation (finalAttrs: {
     ++ lib.optional smbdSupport "--smbd=${samba}/bin/smbd"
     ++ lib.optional uringSupport "--enable-linux-io-uring"
     ++ lib.optional canokeySupport "--enable-canokey"
-    ++ lib.optional capstoneSupport "--enable-capstone";
+    ++ lib.optional capstoneSupport "--enable-capstone"
+    ++ lib.optional (!pluginsSupport) "--disable-plugins"
+    ++ lib.optional (!enableBlobs) "--disable-install-blobs"
+    ++ lib.optionals stdenv.hostPlatform.isStatic [
+      "--static"
+      # FIXME: "multiple definition of `strtoll'" with libnbcompat
+      "--extra-ldflags=-Wl,--allow-multiple-definition"
+      "-Dlinux_aio_path=${libaio}/lib"
+    ];
 
   dontWrapGApps = true;
 
@@ -248,7 +267,9 @@ stdenv.mkDerivation (finalAttrs: {
 
   # Add a ‘qemu-kvm’ wrapper for compatibility/convenience.
   postInstall = lib.optionalString (!toolsOnly) ''
-    ln -s $out/bin/qemu-system-${stdenv.hostPlatform.qemuArch} $out/bin/qemu-kvm
+    if [ -f $out/bin/qemu-system-${stdenv.hostPlatform.qemuArch} ]; then
+      ln -s $out/bin/qemu-system-${stdenv.hostPlatform.qemuArch} $out/bin/qemu-kvm
+    fi
   '';
 
   passthru = {

pkgs/applications/virtualization/qemu/user-static.nix

@@ -0,0 +1,53 @@
+{ lib, pkgsStatic }:
+
+let
+  qemuUserPlatforms = [
+    "aarch64-linux"
+    "armv7l-linux"
+    "i386-linux"
+    "powerpc-linux"
+    "powerpc64-linux"
+    "powerpc64le-linux"
+    "riscv32-linux"
+    "riscv64-linux"
+    "x86_64-linux"
+  ];
+
+  qemuTargets = map (system: (lib.systems.elaborate { inherit system; }).qemuArch + "-linux-user") qemuUserPlatforms;
+
+  static = (pkgsStatic.qemu.override {
+    guestAgentSupport = false;
+    numaSupport = false;
+    seccompSupport = false;
+    alsaSupport = false;
+    pulseSupport = false;
+    sdlSupport = false;
+    jackSupport = false;
+    pipewireSupport = false;
+    gtkSupport = false;
+    vncSupport = false;
+    smartcardSupport = false;
+    spiceSupport = false;
+    ncursesSupport = false;
+    libiscsiSupport = false;
+    smbdSupport = false;
+    tpmSupport = false;
+    uringSupport = false;
+    capstoneSupport = false;
+    enableDocs = false;
+    enableTools = false;
+    enableBlobs = false;
+    hostCpuTargets = qemuTargets;
+  }).overrideAttrs (old: {
+    # HACK: Otherwise the result will have the entire buildinput closure
+    # injected by the pkgsStatic stdenv
+    # <https://github.com/NixOS/nixpkgs/issues/83667>
+    postFixup = (old.postFixup or "") + ''
+      rm -f $out/nix-support/propagated-build-inputs
+    '';
+  });
+in static // {
+  passthru = (static.passthru or {}) // {
+    inherit qemuUserPlatforms;
+  };
+}

pkgs/top-level/all-packages.nix

@@ -34246,6 +34246,8 @@ with pkgs;
     toolsOnly = true;
   };
 
+  qemu-user-static = callPackage ../applications/virtualization/qemu/user-static.nix { };
+
   canokey-qemu = callPackage ../applications/virtualization/qemu/canokey-qemu.nix { };
 
   wrapQemuBinfmtP = callPackage ../applications/virtualization/qemu/binfmt-p-wrapper.nix { };