wip: Attempting to also build lix with nix-ninja

crates/nix-ninja/src/task.rs

@@ -125,13 +125,9 @@ impl Runner {
             }
 
             let path = entry.into_path();
-            let relative_path = match relative_from(&path, &self.config.build_dir) {
-                Some(p) => p,
-                None => path,
-            };
-
-            let derived_file = new_opaque_file(&self.tools.nix, relative_path.clone())?;
-            let fid = self.add_derived_file(files, derived_file.clone(), &relative_path);
+            let derived_file =
+                new_opaque_file(&self.tools.nix, &self.config.build_dir, path.clone())?;
+            let fid = self.add_derived_file(files, derived_file.clone());
             self.build_dir_inputs.insert(fid, derived_file);
         }
         Ok(())
@@ -170,8 +166,12 @@ impl Runner {
                 None => Vec::new(),
             };
 
-            let derived_file = new_opaque_file(&self.tools.nix, extra_input_path.clone())?;
-            self.add_derived_file(files, derived_file.clone(), &extra_input_path);
+            let derived_file = new_opaque_file(
+                &self.tools.nix,
+                &self.config.build_dir,
+                extra_input_path.clone(),
+            )?;
+            self.add_derived_file(files, derived_file.clone());
 
             extra_inputs.push(derived_file);
             self.extra_inputs.insert(bid, extra_inputs);
@@ -227,7 +227,7 @@ impl Runner {
         }
 
         for derived_file in result.derived_files {
-            self.add_derived_file(files, derived_file.clone(), &derived_file.source);
+            self.add_derived_file(files, derived_file.clone());
         }
 
         Ok(result.bid)
@@ -237,11 +237,8 @@ impl Runner {
         &mut self,
         files: &mut graph::GraphFiles,
         derived_file: DerivedFile,
-        path: &PathBuf,
     ) -> FileId {
-        let mut path_str = path.to_string_lossy().into_owned();
-        canon::canonicalize_path(&mut path_str);
-
+        let path_str = derived_file.source.to_string_lossy().into_owned();
         let fid = match files.lookup(&path_str) {
             Some(fid) => fid,
             None => files.id_from_canonical(path_str),
@@ -290,12 +287,12 @@ impl Runner {
                         continue;
                     }
 
-                    let input = new_opaque_file(&self.tools.nix, file.name.clone().into())?;
-                    self.add_derived_file(
-                        files,
-                        input.clone().to_owned(),
-                        &file.name.clone().into(),
-                    );
+                    let input = new_opaque_file(
+                        &self.tools.nix,
+                        &self.config.build_dir,
+                        file.name.clone().into(),
+                    )?;
+                    self.add_derived_file(files, input.clone().to_owned());
                     input.to_owned()
                 }
             };
@@ -430,14 +427,14 @@ fn build_task_derivation(tools: Tools, task: Task) -> Result<Vec<DerivedFile>> {
         .add_input_src(&tools.nix_ninja_task.to_string());
 
     // Add all ninja build inputs.
-    let mut inputs: Vec<String> = Vec::new();
+    let mut input_set: HashSet<String> = HashSet::new();
     for input in &task.inputs {
         // Declare input for derivation.
         add_derived_path(&mut drv, input);
 
         // Encode input for nix-ninja-task.
         let encoded = &input.to_encoded();
-        inputs.push(encoded.clone());
+        input_set.insert(encoded.clone());
     }
 
     // Handle when rule's dep = gcc, which means we need to find all the
@@ -469,30 +466,24 @@ fn build_task_derivation(tools: Tools, task: Task) -> Result<Vec<DerivedFile>> {
                     }
                 }
 
-                // Make it relative to the build directory.
-                let relative_include = match relative_from(&include, &task.build_dir) {
-                    Some(p) => p,
-                    None => include,
-                };
-                let mut path = relative_include.to_string_lossy().into_owned();
-                canon::canonicalize_path(&mut path);
-
+                let derived_file = new_opaque_file(&tools.nix, &task.build_dir, include)?;
                 // Skip paths that are already in the task inputs.
-                if file_set.contains(&PathBuf::from(path.clone())) {
+                if file_set.contains(&derived_file.source) {
                     continue;
                 }
 
-                let derived_file = new_opaque_file(&tools.nix, path.into())?;
                 let encoded = &derived_file.to_encoded();
                 // Should be source-linked.
-                inputs.push(encoded.clone());
+                input_set.insert(encoded.clone());
                 // Should be included as an input to derivation.
                 add_derived_path(&mut drv, &derived_file);
                 // Should be returned back to the Runner as a discovered input.
                 discovered_inputs.push(derived_file);
             }
         }
     }
+
+    let inputs: Vec<String> = input_set.into_iter().collect();
     drv.add_env("NIX_NINJA_INPUTS", &inputs.join(" "));
 
     // Add all ninja build outputs.
@@ -554,7 +545,11 @@ fn build_task_derivation(tools: Tools, task: Task) -> Result<Vec<DerivedFile>> {
 }
 
 fn process_phony(_: Tools, _: Task) -> Result<Vec<DerivedFile>> {
-    Err(anyhow!("Unimplemented"))
+    Ok(Vec::new())
+    // Err(anyhow!(
+    //     "Phony targets not yet supported for {}",
+    //     &task.name
+    // ))
 }
 
 pub fn which_store_path(binary_name: &str) -> Result<StorePath> {
@@ -587,12 +582,19 @@ fn extract_store_paths(store_regex: &Regex, s: &str) -> Result<Vec<StorePath>> {
     Ok(store_paths)
 }
 
-fn new_opaque_file(nix: &NixTool, path: PathBuf) -> Result<DerivedFile> {
+fn new_opaque_file(nix: &NixTool, build_dir: &PathBuf, path: PathBuf) -> Result<DerivedFile> {
+    let relative_path = match relative_from(&path, build_dir) {
+        Some(p) => p,
+        None => path,
+    };
+    let mut path = relative_path.to_string_lossy().into_owned();
+    canon::canonicalize_path(&mut path);
+
     let canonical_path = fs::canonicalize(&path)?;
     let store_path = nix.store_add(&canonical_path)?;
     Ok(DerivedFile {
         path: SingleDerivedPath::Opaque(store_path.clone()),
-        source: path,
+        source: relative_path,
     })
 }
 

examples/lix/default.nix

@@ -0,0 +1,169 @@
+{ inputs
+, aws-sdk-cpp
+, boehmgc
+, boost
+, brotli
+, busybox-sandbox-shell
+, bzip2
+, cmake
+, curl
+, doxygen
+, editline
+, flex
+, git
+, gtest
+, jq
+, lib
+, libarchive
+, libcpuid
+, libsodium
+, lowdown
+, lowdown-unsandboxed
+, lsof
+, mdbook
+, mdbook-linkcheck
+, mercurial
+, mkMesonPackage
+, nlohmann_json
+, openssl
+, pegtl
+, pkg-config
+, python3
+, rapidcheck
+, sqlite
+, stdenv
+, toml11
+, util-linuxMinimal
+, xz
+, enableDocumentation ? stdenv.hostPlatform == stdenv.buildPlatform
+, enableStatic ? stdenv.hostPlatform.isStatic
+, withAWS ? !enableStatic && (stdenv.hostPlatform.isLinux || stdenv.hostPlatform.isDarwin)
+, withLibseccomp ? lib.meta.availableOn stdenv.hostPlatform libseccomp
+, libseccomp
+}:
+
+let
+  aws-sdk-cpp-nix =
+    if aws-sdk-cpp == null then
+      null
+    else
+      aws-sdk-cpp.override {
+        apis = [
+          "s3"
+          "transfer"
+        ];
+        customMemoryManagement = false;
+      };
+
+  boehmgc-nix = boehmgc.override { enableLargeConfig = true; };
+
+in mkMesonPackage {
+  name = "example-lix";
+  src = inputs.lix;
+  patches = [
+    ./disable-meson-clang-tidy.patch
+  ];
+
+  target = "src/nix/nix";
+
+  # nixNinjaExtraInputs = [];
+
+  dontInstall = true;
+  dontFixup = true;
+
+  nativeBuildInputs = [
+    pkg-config
+    flex
+    jq
+    cmake
+    python3
+
+    # Tests
+    git
+    mercurial
+    jq
+    lsof
+  ]
+  ++ lib.optionals enableDocumentation [
+    (lib.getBin lowdown-unsandboxed)
+    mdbook
+    mdbook-linkcheck
+    doxygen
+  ]
+  ++ lib.optionals stdenv.hostPlatform.isLinux [ util-linuxMinimal ];
+
+  buildInputs =
+  [
+    boost
+    brotli
+    bzip2
+    curl
+    editline
+    libsodium
+    openssl
+    sqlite
+    xz
+    gtest
+    libarchive
+    lowdown
+    rapidcheck
+    toml11
+    pegtl
+  ]
+  ++ lib.optionals (stdenv.hostPlatform.isx86_64) [ libcpuid ]
+  ++ lib.optionals withLibseccomp [ libseccomp ]
+  ++ lib.optionals withAWS [ aws-sdk-cpp-nix ];
+
+  propagatedBuildInputs = [
+    boehmgc-nix
+    nlohmann_json
+  ];
+
+  postPatch = ''
+    patchShebangs --build tests doc/manual
+  '';
+
+  preConfigure =
+  # Copy libboost_context so we don't get all of Boost in our closure.
+  # https://github.com/NixOS/nixpkgs/issues/45462
+  lib.optionalString (!enableStatic) ''
+    mkdir -p $out/lib
+    cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
+    rm -f $out/lib/*.a
+    ${lib.optionalString stdenv.hostPlatform.isLinux ''
+      chmod u+w $out/lib/*.so.*
+      patchelf --set-rpath $out/lib:${lib.getLib stdenv.cc.cc}/lib $out/lib/libboost_thread.so.*
+    ''}
+    ${lib.optionalString stdenv.hostPlatform.isDarwin ''
+      for LIB in $out/lib/*.dylib; do
+        chmod u+w $LIB
+        install_name_tool -id $LIB $LIB
+        install_name_tool -delete_rpath ${boost}/lib/ $LIB || true
+      done
+      install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib
+    ''}
+  '';
+
+  # -O3 seems to anger a gcc bug and provide no performance benefit.
+  # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=114360
+  # We use -O2 upstream https://gerrit.lix.systems/c/lix/+/554
+  mesonBuildType = "debugoptimized";
+
+  mesonFlags =
+  [
+    # Enable LTO, since it improves eval performance a fair amount
+    # LTO is disabled on static due to strange linking errors
+    (lib.mesonBool "b_lto" (!stdenv.hostPlatform.isStatic && stdenv.cc.isGNU))
+    (lib.mesonEnable "gc" true)
+    (lib.mesonBool "enable-tests" true)
+    (lib.mesonBool "enable-docs" enableDocumentation)
+    (lib.mesonEnable "internal-api-docs" enableDocumentation)
+    (lib.mesonBool "enable-embedded-sandbox-shell" (
+      stdenv.hostPlatform.isLinux && stdenv.hostPlatform.isStatic
+    ))
+    (lib.mesonEnable "seccomp-sandboxing" withLibseccomp)
+  ]
+  ++ lib.optionals stdenv.hostPlatform.isLinux [
+    (lib.mesonOption "sandbox-shell" "${busybox-sandbox-shell}/bin/busybox")
+  ];
+}

examples/lix/disable-meson-clang-tidy.patch

@@ -0,0 +1,10 @@
+diff --git a/meson.build b/meson.build
+index cf1b877f9..b2578ab94 100644
+--- a/meson.build
++++ b/meson.build
+@@ -565,4 +565,4 @@ if enable_tests
+   subdir('tests/functional')
+ endif
+
+-subdir('meson/clang-tidy')
++# subdir('meson/clang-tidy')