diff --git a/crates/nix-ninja/src/task.rs b/crates/nix-ninja/src/task.rs index b847dbc..60af811 100644 --- a/crates/nix-ninja/src/task.rs +++ b/crates/nix-ninja/src/task.rs @@ -125,13 +125,9 @@ impl Runner { } let path = entry.into_path(); - let relative_path = match relative_from(&path, &self.config.build_dir) { - Some(p) => p, - None => path, - }; - - let derived_file = new_opaque_file(&self.tools.nix, relative_path.clone())?; - let fid = self.add_derived_file(files, derived_file.clone(), &relative_path); + let derived_file = + new_opaque_file(&self.tools.nix, &self.config.build_dir, path.clone())?; + let fid = self.add_derived_file(files, derived_file.clone()); self.build_dir_inputs.insert(fid, derived_file); } Ok(()) @@ -170,8 +166,12 @@ impl Runner { None => Vec::new(), }; - let derived_file = new_opaque_file(&self.tools.nix, extra_input_path.clone())?; - self.add_derived_file(files, derived_file.clone(), &extra_input_path); + let derived_file = new_opaque_file( + &self.tools.nix, + &self.config.build_dir, + extra_input_path.clone(), + )?; + self.add_derived_file(files, derived_file.clone()); extra_inputs.push(derived_file); self.extra_inputs.insert(bid, extra_inputs); @@ -227,7 +227,7 @@ impl Runner { } for derived_file in result.derived_files { - self.add_derived_file(files, derived_file.clone(), &derived_file.source); + self.add_derived_file(files, derived_file.clone()); } Ok(result.bid) @@ -237,11 +237,8 @@ impl Runner { &mut self, files: &mut graph::GraphFiles, derived_file: DerivedFile, - path: &PathBuf, ) -> FileId { - let mut path_str = path.to_string_lossy().into_owned(); - canon::canonicalize_path(&mut path_str); - + let path_str = derived_file.source.to_string_lossy().into_owned(); let fid = match files.lookup(&path_str) { Some(fid) => fid, None => files.id_from_canonical(path_str), @@ -290,12 +287,12 @@ impl Runner { continue; } - let input = new_opaque_file(&self.tools.nix, file.name.clone().into())?; - self.add_derived_file( - files, - input.clone().to_owned(), - &file.name.clone().into(), - ); + let input = new_opaque_file( + &self.tools.nix, + &self.config.build_dir, + file.name.clone().into(), + )?; + self.add_derived_file(files, input.clone().to_owned()); input.to_owned() } }; @@ -430,14 +427,14 @@ fn build_task_derivation(tools: Tools, task: Task) -> Result> { .add_input_src(&tools.nix_ninja_task.to_string()); // Add all ninja build inputs. - let mut inputs: Vec = Vec::new(); + let mut input_set: HashSet = HashSet::new(); for input in &task.inputs { // Declare input for derivation. add_derived_path(&mut drv, input); // Encode input for nix-ninja-task. let encoded = &input.to_encoded(); - inputs.push(encoded.clone()); + input_set.insert(encoded.clone()); } // Handle when rule's dep = gcc, which means we need to find all the @@ -469,23 +466,15 @@ fn build_task_derivation(tools: Tools, task: Task) -> Result> { } } - // Make it relative to the build directory. - let relative_include = match relative_from(&include, &task.build_dir) { - Some(p) => p, - None => include, - }; - let mut path = relative_include.to_string_lossy().into_owned(); - canon::canonicalize_path(&mut path); - + let derived_file = new_opaque_file(&tools.nix, &task.build_dir, include)?; // Skip paths that are already in the task inputs. - if file_set.contains(&PathBuf::from(path.clone())) { + if file_set.contains(&derived_file.source) { continue; } - let derived_file = new_opaque_file(&tools.nix, path.into())?; let encoded = &derived_file.to_encoded(); // Should be source-linked. - inputs.push(encoded.clone()); + input_set.insert(encoded.clone()); // Should be included as an input to derivation. add_derived_path(&mut drv, &derived_file); // Should be returned back to the Runner as a discovered input. @@ -493,6 +482,8 @@ fn build_task_derivation(tools: Tools, task: Task) -> Result> { } } } + + let inputs: Vec = input_set.into_iter().collect(); drv.add_env("NIX_NINJA_INPUTS", &inputs.join(" ")); // Add all ninja build outputs. @@ -554,7 +545,11 @@ fn build_task_derivation(tools: Tools, task: Task) -> Result> { } fn process_phony(_: Tools, _: Task) -> Result> { - Err(anyhow!("Unimplemented")) + Ok(Vec::new()) + // Err(anyhow!( + // "Phony targets not yet supported for {}", + // &task.name + // )) } pub fn which_store_path(binary_name: &str) -> Result { @@ -587,12 +582,19 @@ fn extract_store_paths(store_regex: &Regex, s: &str) -> Result> { Ok(store_paths) } -fn new_opaque_file(nix: &NixTool, path: PathBuf) -> Result { +fn new_opaque_file(nix: &NixTool, build_dir: &PathBuf, path: PathBuf) -> Result { + let relative_path = match relative_from(&path, build_dir) { + Some(p) => p, + None => path, + }; + let mut path = relative_path.to_string_lossy().into_owned(); + canon::canonicalize_path(&mut path); + let canonical_path = fs::canonicalize(&path)?; let store_path = nix.store_add(&canonical_path)?; Ok(DerivedFile { path: SingleDerivedPath::Opaque(store_path.clone()), - source: path, + source: relative_path, }) } diff --git a/examples/lix/default.nix b/examples/lix/default.nix new file mode 100644 index 0000000..28b7713 --- /dev/null +++ b/examples/lix/default.nix @@ -0,0 +1,169 @@ +{ inputs +, aws-sdk-cpp +, boehmgc +, boost +, brotli +, busybox-sandbox-shell +, bzip2 +, cmake +, curl +, doxygen +, editline +, flex +, git +, gtest +, jq +, lib +, libarchive +, libcpuid +, libsodium +, lowdown +, lowdown-unsandboxed +, lsof +, mdbook +, mdbook-linkcheck +, mercurial +, mkMesonPackage +, nlohmann_json +, openssl +, pegtl +, pkg-config +, python3 +, rapidcheck +, sqlite +, stdenv +, toml11 +, util-linuxMinimal +, xz +, enableDocumentation ? stdenv.hostPlatform == stdenv.buildPlatform +, enableStatic ? stdenv.hostPlatform.isStatic +, withAWS ? !enableStatic && (stdenv.hostPlatform.isLinux || stdenv.hostPlatform.isDarwin) +, withLibseccomp ? lib.meta.availableOn stdenv.hostPlatform libseccomp +, libseccomp +}: + +let + aws-sdk-cpp-nix = + if aws-sdk-cpp == null then + null + else + aws-sdk-cpp.override { + apis = [ + "s3" + "transfer" + ]; + customMemoryManagement = false; + }; + + boehmgc-nix = boehmgc.override { enableLargeConfig = true; }; + +in mkMesonPackage { + name = "example-lix"; + src = inputs.lix; + patches = [ + ./disable-meson-clang-tidy.patch + ]; + + target = "src/nix/nix"; + + # nixNinjaExtraInputs = []; + + dontInstall = true; + dontFixup = true; + + nativeBuildInputs = [ + pkg-config + flex + jq + cmake + python3 + + # Tests + git + mercurial + jq + lsof + ] + ++ lib.optionals enableDocumentation [ + (lib.getBin lowdown-unsandboxed) + mdbook + mdbook-linkcheck + doxygen + ] + ++ lib.optionals stdenv.hostPlatform.isLinux [ util-linuxMinimal ]; + + buildInputs = + [ + boost + brotli + bzip2 + curl + editline + libsodium + openssl + sqlite + xz + gtest + libarchive + lowdown + rapidcheck + toml11 + pegtl + ] + ++ lib.optionals (stdenv.hostPlatform.isx86_64) [ libcpuid ] + ++ lib.optionals withLibseccomp [ libseccomp ] + ++ lib.optionals withAWS [ aws-sdk-cpp-nix ]; + + propagatedBuildInputs = [ + boehmgc-nix + nlohmann_json + ]; + + postPatch = '' + patchShebangs --build tests doc/manual + ''; + + preConfigure = + # Copy libboost_context so we don't get all of Boost in our closure. + # https://github.com/NixOS/nixpkgs/issues/45462 + lib.optionalString (!enableStatic) '' + mkdir -p $out/lib + cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib + rm -f $out/lib/*.a + ${lib.optionalString stdenv.hostPlatform.isLinux '' + chmod u+w $out/lib/*.so.* + patchelf --set-rpath $out/lib:${lib.getLib stdenv.cc.cc}/lib $out/lib/libboost_thread.so.* + ''} + ${lib.optionalString stdenv.hostPlatform.isDarwin '' + for LIB in $out/lib/*.dylib; do + chmod u+w $LIB + install_name_tool -id $LIB $LIB + install_name_tool -delete_rpath ${boost}/lib/ $LIB || true + done + install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib + ''} + ''; + + # -O3 seems to anger a gcc bug and provide no performance benefit. + # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=114360 + # We use -O2 upstream https://gerrit.lix.systems/c/lix/+/554 + mesonBuildType = "debugoptimized"; + + mesonFlags = + [ + # Enable LTO, since it improves eval performance a fair amount + # LTO is disabled on static due to strange linking errors + (lib.mesonBool "b_lto" (!stdenv.hostPlatform.isStatic && stdenv.cc.isGNU)) + (lib.mesonEnable "gc" true) + (lib.mesonBool "enable-tests" true) + (lib.mesonBool "enable-docs" enableDocumentation) + (lib.mesonEnable "internal-api-docs" enableDocumentation) + (lib.mesonBool "enable-embedded-sandbox-shell" ( + stdenv.hostPlatform.isLinux && stdenv.hostPlatform.isStatic + )) + (lib.mesonEnable "seccomp-sandboxing" withLibseccomp) + ] + ++ lib.optionals stdenv.hostPlatform.isLinux [ + (lib.mesonOption "sandbox-shell" "${busybox-sandbox-shell}/bin/busybox") + ]; +} diff --git a/examples/lix/disable-meson-clang-tidy.patch b/examples/lix/disable-meson-clang-tidy.patch new file mode 100644 index 0000000..6448c71 --- /dev/null +++ b/examples/lix/disable-meson-clang-tidy.patch @@ -0,0 +1,10 @@ +diff --git a/meson.build b/meson.build +index cf1b877f9..b2578ab94 100644 +--- a/meson.build ++++ b/meson.build +@@ -565,4 +565,4 @@ if enable_tests + subdir('tests/functional') + endif + +-subdir('meson/clang-tidy') ++# subdir('meson/clang-tidy') diff --git a/examples/nix/default.nix b/examples/nix/default.nix new file mode 100644 index 0000000..8dfa73e --- /dev/null +++ b/examples/nix/default.nix @@ -0,0 +1,98 @@ +{ inputs +, aws-sdk-cpp +, bison +, boehmgc +, boost +, brotli +, busybox-sandbox-shell +, bzip2 +, cmake +, curl +, doxygen +, editline +, flex +, gtest +, lib +, libarchive +, libblake3 +, libcpuid +, libgit2 +, libseccomp +, libsodium +, lowdown +, mkMesonPackage +, nlohmann_json +, openssl +, perl +, perlPackages +, pkg-config +, rapidcheck +, readline +, sqlite +, toml11 +}: + +mkMesonPackage { + name = "example-nix"; + src = inputs.nix; + target = "src/nix/nix"; + + nixNinjaExtraInputs = [ + "src/libexpr/libnixexpr.so.p/meson-generated_.._parser-tab.cc.o:../src/libexpr/parser.y" + "src/libexpr/libnixexpr.so.p/meson-generated_.._lexer-tab.cc.o:../src/libexpr/parser.y" + "src/libexpr/libnixexpr.so.p/meson-generated_.._lexer-tab.cc.o:../src/libexpr/lexer.l" + "src/libexpr/libnixexpr.so.p/eval.cc.o:../src/libexpr/parser.y" + "src/libexpr/libnixexpr.so.p/lexer-helpers.cc.o:../src/libexpr/parser.y" + ]; + + nativeBuildInputs = [ + aws-sdk-cpp + bison + boehmgc + boost + brotli + busybox-sandbox-shell + bzip2 + cmake + curl + doxygen + editline + flex + libarchive + libblake3 + libcpuid + libgit2 + libseccomp + libsodium + lowdown + nlohmann_json + openssl + perl + pkg-config + readline + sqlite + toml11 + ]; + + buildInputs = [ + rapidcheck + gtest + ]; + + # dontAddPrefix = true; + + mesonFlags = [ + "--prefix=/build/tmp" + "--bindir=/build/tmp/bin" + "--mandir=/build/tmp/man" + (lib.mesonOption "perl:dbi_path" "${perlPackages.DBI}/${perl.libPrefix}") + (lib.mesonOption "perl:dbd_sqlite_path" "${perlPackages.DBDSQLite}/${perl.libPrefix}") + ]; + + env = { + # Needed for Meson to find Boost. + # https://github.com/NixOS/nixpkgs/issues/86131. + BOOST_INCLUDEDIR = "${lib.getDev boost}/include"; + BOOST_LIBRARYDIR = "${lib.getLib boost}/lib"; + }; +} diff --git a/flake.lock b/flake.lock index 46a95b9..2c75cd7 100644 --- a/flake.lock +++ b/flake.lock @@ -69,6 +69,22 @@ } }, "flake-compat_2": { + "flake": false, + "locked": { + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-compat_3": { "flake": false, "locked": { "lastModified": 1733328505, @@ -156,14 +172,37 @@ "type": "github" } }, - "nix": { + "lix": { "inputs": { "flake-compat": "flake-compat_2", + "nix2container": "nix2container", + "nixpkgs": "nixpkgs", + "nixpkgs-regression": "nixpkgs-regression", + "pre-commit-hooks": "pre-commit-hooks" + }, + "locked": { + "lastModified": 1729298361, + "narHash": "sha256-hiGtfzxFkDc9TSYsb96Whg0vnqBVV7CUxyscZNhed0U=", + "owner": "lix-project", + "repo": "lix", + "rev": "2667fb70c10433dd1cbfbdc0815de43e72298c4d", + "type": "github" + }, + "original": { + "owner": "lix-project", + "ref": "2.91.1", + "repo": "lix", + "type": "github" + } + }, + "nix": { + "inputs": { + "flake-compat": "flake-compat_3", "flake-parts": "flake-parts", "git-hooks-nix": "git-hooks-nix", - "nixpkgs": "nixpkgs", + "nixpkgs": "nixpkgs_2", "nixpkgs-23-11": "nixpkgs-23-11", - "nixpkgs-regression": "nixpkgs-regression" + "nixpkgs-regression": "nixpkgs-regression_2" }, "locked": { "lastModified": 1743180261, @@ -180,18 +219,34 @@ "type": "github" } }, + "nix2container": { + "flake": false, + "locked": { + "lastModified": 1724996935, + "narHash": "sha256-njRK9vvZ1JJsP8oV2OgkBrpJhgQezI03S7gzskCcHos=", + "owner": "nlewo", + "repo": "nix2container", + "rev": "fa6bb0a1159f55d071ba99331355955ae30b3401", + "type": "github" + }, + "original": { + "owner": "nlewo", + "repo": "nix2container", + "type": "github" + } + }, "nixpkgs": { "locked": { - "lastModified": 1734359947, - "narHash": "sha256-1Noao/H+N8nFB4Beoy8fgwrcOQLVm9o4zKW1ODaqK9E=", + "lastModified": 1735651292, + "narHash": "sha256-YLbzcBtYo1/FEzFsB3AnM16qFc6fWPMIoOuSoDwvg9g=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "48d12d5e70ee91fe8481378e540433a7303dbf6a", + "rev": "0da3c44a9460a26d2025ec3ed2ec60a895eb1114", "type": "github" }, "original": { "owner": "NixOS", - "ref": "release-24.11", + "ref": "nixos-24.05-small", "repo": "nixpkgs", "type": "github" } @@ -228,7 +283,39 @@ "type": "github" } }, + "nixpkgs-regression_2": { + "locked": { + "lastModified": 1643052045, + "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + } + }, "nixpkgs_2": { + "locked": { + "lastModified": 1734359947, + "narHash": "sha256-1Noao/H+N8nFB4Beoy8fgwrcOQLVm9o4zKW1ODaqK9E=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "48d12d5e70ee91fe8481378e540433a7303dbf6a", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "release-24.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_3": { "locked": { "lastModified": 1741310760, "narHash": "sha256-aizILFrPgq/W53Jw8i0a1h1GZAAKtlYOrG/A5r46gVM=", @@ -244,6 +331,22 @@ "type": "github" } }, + "pre-commit-hooks": { + "flake": false, + "locked": { + "lastModified": 1733318908, + "narHash": "sha256-SVQVsbafSM1dJ4fpgyBqLZ+Lft+jcQuMtEL3lQWx2Sk=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "6f4e2a2112050951a314d2733a994fbab94864c6", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, "root": { "inputs": { "advisory-db": "advisory-db", @@ -251,8 +354,9 @@ "fenix": "fenix", "flake-compat": "flake-compat", "globset": "globset", + "lix": "lix", "nix": "nix", - "nixpkgs": "nixpkgs_2" + "nixpkgs": "nixpkgs_3" } } }, diff --git a/flake.nix b/flake.nix index d168fe8..fa3b04a 100644 --- a/flake.nix +++ b/flake.nix @@ -6,6 +6,8 @@ nix.url = "github:hinshun/nix/2.27.1-fix-nix-missing-includes"; + lix.url = "github:lix-project/lix/2.91.1"; + globset = { url = "github:pdtpartners/globset"; inputs.nixpkgs-lib.follows = "nixpkgs"; @@ -113,69 +115,12 @@ target = "hello"; }; - example-nix = mkMesonPackage { - name = "example-nix"; - src = inputs.nix; - target = "src/nix/nix"; - - nixNinjaExtraInputs = [ - "src/libexpr/libnixexpr.so.p/meson-generated_.._parser-tab.cc.o:../src/libexpr/parser.y" - "src/libexpr/libnixexpr.so.p/meson-generated_.._lexer-tab.cc.o:../src/libexpr/parser.y" - "src/libexpr/libnixexpr.so.p/meson-generated_.._lexer-tab.cc.o:../src/libexpr/lexer.l" - "src/libexpr/libnixexpr.so.p/eval.cc.o:../src/libexpr/parser.y" - "src/libexpr/libnixexpr.so.p/lexer-helpers.cc.o:../src/libexpr/parser.y" - ]; - - nativeBuildInputs = with pkgs; [ - aws-sdk-cpp - bison - boehmgc - boost - brotli - busybox-sandbox-shell - bzip2 - cmake - curl - doxygen - editline - flex - libarchive - libblake3 - libcpuid - libgit2 - libseccomp - libsodium - lowdown - nlohmann_json - openssl - perl - pkg-config - readline - sqlite - toml11 - ]; - - buildInputs = with pkgs; [ - rapidcheck - gtest - ]; - - # dontAddPrefix = true; - - mesonFlags = with pkgs; [ - "--prefix=/build/tmp" - "--bindir=/build/tmp/bin" - "--mandir=/build/tmp/man" - (lib.mesonOption "perl:dbi_path" "${perlPackages.DBI}/${perl.libPrefix}") - (lib.mesonOption "perl:dbd_sqlite_path" "${perlPackages.DBDSQLite}/${perl.libPrefix}") - ]; + example-nix = pkgs.callPackage ./examples/nix { + inherit inputs mkMesonPackage; + }; - env = with pkgs; { - # Needed for Meson to find Boost. - # https://github.com/NixOS/nixpkgs/issues/86131. - BOOST_INCLUDEDIR = "${lib.getDev boost}/include"; - BOOST_LIBRARYDIR = "${lib.getLib boost}/lib"; - }; + example-lix = pkgs.callPackage ./examples/lix { + inherit inputs mkMesonPackage; }; in @@ -255,6 +200,7 @@ example-hello example-header example-nix + example-lix ; };