Make nix/unpack-channel.nix a builtin builder

.gitignore

@@ -86,6 +86,7 @@ perl/Makefile.config
 /tests/restricted-innocent
 /tests/shell
 /tests/shell.drv
+/tests/config.nix
 
 # /tests/lang/
 /tests/lang/*.out
@@ -119,3 +120,5 @@ GPATH
 GRTAGS
 GSYMS
 GTAGS
+
+nix-rust/target

Makefile

@@ -1,6 +1,7 @@
 makefiles = \
   mk/precompiled-headers.mk \
   local.mk \
+  nix-rust/local.mk \
   src/libutil/local.mk \
   src/libstore/local.mk \
   src/libmain/local.mk \

configure.ac

@@ -117,26 +117,15 @@ fi
 ])
 
 NEED_PROG(bash, bash)
-NEED_PROG(patch, patch)
 AC_PATH_PROG(xmllint, xmllint, false)
 AC_PATH_PROG(xsltproc, xsltproc, false)
 AC_PATH_PROG(flex, flex, false)
 AC_PATH_PROG(bison, bison, false)
-NEED_PROG(sed, sed)
-NEED_PROG(tar, tar)
-NEED_PROG(bzip2, bzip2)
-NEED_PROG(gzip, gzip)
-NEED_PROG(xz, xz)
 AC_PATH_PROG(dot, dot)
 AC_PATH_PROG(lsof, lsof, lsof)
 
 
-NEED_PROG(cat, cat)
-NEED_PROG(tr, tr)
-AC_ARG_WITH(coreutils-bin, AC_HELP_STRING([--with-coreutils-bin=PATH],
-  [path of cat, mkdir, etc.]),
-  coreutils=$withval, coreutils=$(dirname $cat))
-AC_SUBST(coreutils)
+AC_SUBST(coreutils, [$(dirname $(type -p cat))])
 
 
 AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH],
@@ -167,7 +156,8 @@ if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then
     LIBS="-latomic $LIBS"
 fi
 
-# Look for OpenSSL, a required dependency.
+# Look for OpenSSL, a required dependency. FIXME: this is only (maybe)
+# used by S3BinaryCacheStore.
 PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"])
 
 
@@ -177,11 +167,9 @@ AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true],
 AC_CHECK_HEADERS([bzlib.h], [true],
   [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])])
 
-
 # Look for SQLite, a required dependency.
 PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CXXFLAGS"])
 
-
 # Look for libcurl, a required dependency.
 PKG_CHECK_MODULES([LIBCURL], [libcurl], [CXXFLAGS="$LIBCURL_CFLAGS $CXXFLAGS"])
 
@@ -204,13 +192,11 @@ PKG_CHECK_MODULES([SODIUM], [libsodium],
    have_sodium=1], [have_sodium=])
 AC_SUBST(HAVE_SODIUM, [$have_sodium])
 
-
 # Look for liblzma, a required dependency.
 PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"])
 AC_CHECK_LIB([lzma], [lzma_stream_encoder_mt],
   [AC_DEFINE([HAVE_LZMA_MT], [1], [xz multithreaded compression support])])
 
-
 # Look for libbrotli{enc,dec}.
 PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"])
 

corepkgs/config.nix.in

@@ -1,29 +1,13 @@
+# FIXME: remove this file?
 let
   fromEnv = var: def:
     let val = builtins.getEnv var; in
     if val != "" then val else def;
 in rec {
-  shell = "@bash@";
-  coreutils = "@coreutils@";
-  bzip2 = "@bzip2@";
-  gzip = "@gzip@";
-  xz = "@xz@";
-  tar = "@tar@";
-  tarFlags = "@tarFlags@";
-  tr = "@tr@";
   nixBinDir = fromEnv "NIX_BIN_DIR" "@bindir@";
   nixPrefix = "@prefix@";
   nixLibexecDir = fromEnv "NIX_LIBEXEC_DIR" "@libexecdir@";
   nixLocalstateDir = "@localstatedir@";
   nixSysconfDir = "@sysconfdir@";
   nixStoreDir = fromEnv "NIX_STORE_DIR" "@storedir@";
-
-  # If Nix is installed in the Nix store, then automatically add it as
-  # a dependency to the core packages. This ensures that they work
-  # properly in a chroot.
-  chrootDeps =
-    if dirOf nixPrefix == builtins.storeDir then
-      [ (builtins.storePath nixPrefix) ]
-    else
-      [ ];
 }

corepkgs/unpack-channel.nix

@@ -1,39 +1,12 @@
-with import <nix/config.nix>;
-
-let
-
-  builder = builtins.toFile "unpack-channel.sh"
-    ''
-      mkdir $out
-      cd $out
-      xzpat="\.xz\$"
-      gzpat="\.gz\$"
-      if [[ "$src" =~ $xzpat ]]; then
-        ${xz} -d < $src | ${tar} xf - ${tarFlags}
-      elif [[ "$src" =~ $gzpat ]]; then
-        ${gzip} -d < $src | ${tar} xf - ${tarFlags}
-      else
-        ${bzip2} -d < $src | ${tar} xf - ${tarFlags}
-      fi
-      if [ * != $channelName ]; then
-        mv * $out/$channelName
-      fi
-    '';
-
-in
-
 { name, channelName, src }:
 
 derivation {
-  system = builtins.currentSystem;
-  builder = shell;
-  args = [ "-e" builder ];
-  inherit name channelName src;
+  builder = "builtin:unpack-channel";
+
+  system = "builtin";
 
-  PATH = "${nixBinDir}:${coreutils}";
+  inherit name channelName src;
 
   # No point in doing this remotely.
   preferLocalBuild = true;
-
-  inherit chrootDeps;
 }

nix-rust/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "nix-rust"
+version = "0.1.0"
+authors = ["Eelco Dolstra <edolstra@gmail.com>"]
+edition = "2018"
+
+[lib]
+name = "nixrust"
+crate-type = ["cdylib"]
+
+[dependencies]
+tar = "0.4"
+libc = "0.2"

nix-rust/local.mk

@@ -0,0 +1,38 @@
+ifeq ($(OPTIMIZE), 1)
+  RUST_MODE = --release
+  RUST_DIR = release
+else
+  RUST_MODE =
+  RUST_DIR = debug
+endif
+
+libnixrust_PATH := $(d)/target/$(RUST_DIR)/libnixrust.$(SO_EXT)
+libnixrust_INSTALL_PATH := $(libdir)/libnixrust.$(SO_EXT)
+libnixrust_LDFLAGS_USE := -L$(d)/target/$(RUST_DIR) -lnixrust -ldl
+libnixrust_LDFLAGS_USE_INSTALLED := -L$(libdir) -lnixrust -ldl
+
+ifeq ($(OS), Darwin)
+libnixrust_BUILD_FLAGS = NIX_LDFLAGS="-undefined dynamic_lookup"
+else
+libnixrust_LDFLAGS_USE += -Wl,-rpath,$(abspath $(d)/target/$(RUST_DIR))
+libnixrust_LDFLAGS_USE_INSTALLED += -Wl,-rpath,$(libdir)
+endif
+
+$(libnixrust_PATH): $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml
+	$(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) \
+	$(libnixrust_BUILD_FLAGS) \
+	  cargo build $(RUST_MODE) $$(if [[ -d vendor ]]; then echo --offline; fi) \
+	&& touch target/$(RUST_DIR)/libnixrust.$(SO_EXT)
+
+$(libnixrust_INSTALL_PATH): $(libnixrust_PATH)
+	$(target-gen) cp $^ $@
+ifeq ($(OS), Darwin)
+	install_name_tool -id $@ $@
+endif
+
+dist-files += $(d)/vendor
+
+clean: clean-rust
+
+clean-rust:
+	$(suppress) rm -rfv nix-rust/target

nix-rust/src/error.rs

@@ -0,0 +1,31 @@
+#[derive(Debug)]
+pub enum Error {
+    IOError(std::io::Error),
+    Misc(String),
+    Foreign(CppException),
+}
+
+impl From<std::io::Error> for Error {
+    fn from(err: std::io::Error) -> Self {
+        Error::IOError(err)
+    }
+}
+
+impl From<Error> for CppException {
+    fn from(err: Error) -> Self {
+        match err {
+            Error::Foreign(ex) => ex,
+            Error::Misc(s) => unsafe { make_error(&s) },
+            Error::IOError(err) => unsafe { make_error(&err.to_string()) },
+        }
+    }
+}
+
+#[repr(C)]
+#[derive(Debug)]
+pub struct CppException(*const libc::c_void); // == std::exception_ptr*
+
+extern "C" {
+    #[allow(improper_ctypes)] // YOLO
+    fn make_error(s: &str) -> CppException;
+}

nix-rust/src/foreign.rs

@@ -0,0 +1,14 @@
+/// A wrapper around Nix's Source class that provides the Read trait.
+#[repr(C)]
+pub struct Source {
+    fun: extern "C" fn(this: *mut libc::c_void, data: &mut [u8]) -> usize,
+    this: *mut libc::c_void,
+}
+
+impl std::io::Read for Source {
+    fn read(&mut self, buf: &mut [u8]) -> std::result::Result<usize, std::io::Error> {
+        let n = (self.fun)(self.this, buf);
+        assert!(n <= buf.len());
+        Ok(n)
+    }
+}

nix-rust/src/lib.rs

@@ -0,0 +1,32 @@
+mod error;
+mod foreign;
+mod tarfile;
+
+pub use error::Error;
+
+pub struct CBox<T> {
+    pub ptr: *mut libc::c_void,
+    phantom: std::marker::PhantomData<T>,
+}
+
+impl<T> CBox<T> {
+    fn new(t: T) -> Self {
+        unsafe {
+            let size = std::mem::size_of::<T>();
+            let ptr = libc::malloc(size);
+            *(ptr as *mut T) = t; // FIXME: probably UB
+            Self {
+                ptr,
+                phantom: std::marker::PhantomData,
+            }
+        }
+    }
+}
+
+#[no_mangle]
+pub extern "C" fn unpack_tarfile(
+    source: foreign::Source,
+    dest_dir: &str,
+) -> CBox<Result<(), error::CppException>> {
+    CBox::new(tarfile::unpack_tarfile(source, dest_dir).map_err(|err| err.into()))
+}