From 9071d83400f182915760e01c80bb8953b1a65c60 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 1 May 2025 10:53:47 +0200 Subject: [PATCH 01/23] Allow dynamic registration of builtin builders --- src/libstore/builtins/buildenv.cc | 5 ++++- src/libstore/builtins/unpack-channel.cc | 4 +++- src/libstore/include/nix/store/builtins.hh | 17 ++++++++++++++--- .../include/nix/store/builtins/buildenv.hh | 4 ---- .../unix/build/local-derivation-goal.cc | 16 ++++++++++------ 5 files changed, 31 insertions(+), 15 deletions(-) diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index c3b80bb0b9b..497bf1dd267 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -1,4 +1,5 @@ #include "nix/store/builtins/buildenv.hh" +#include "nix/store/builtins.hh" #include "nix/store/derivations.hh" #include "nix/util/signals.hh" @@ -166,7 +167,7 @@ void buildProfile(const Path & out, Packages && pkgs) debug("created %d symlinks in user environment", state.symlinks); } -void builtinBuildenv( +static void builtinBuildenv( const BasicDerivation & drv, const std::map & outputs) { @@ -203,4 +204,6 @@ void builtinBuildenv( createSymlink(getAttr("manifest"), out + "/manifest.nix"); } +static RegisterBuiltinBuilder registerBuildenv("buildenv", builtinBuildenv); + } diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc index f6be21e356b..af6c4386083 100644 --- a/src/libstore/builtins/unpack-channel.cc +++ b/src/libstore/builtins/unpack-channel.cc @@ -5,7 +5,7 @@ namespace nix { namespace fs { using namespace std::filesystem; } -void builtinUnpackChannel( +static void builtinUnpackChannel( const BasicDerivation & drv, const std::map & outputs) { @@ -48,4 +48,6 @@ void builtinUnpackChannel( } } +static RegisterBuiltinBuilder registerUnpackChannel("unpack-channel", builtinUnpackChannel); + } diff --git a/src/libstore/include/nix/store/builtins.hh b/src/libstore/include/nix/store/builtins.hh index 004e9ef64a2..6d54c2a2206 100644 --- a/src/libstore/include/nix/store/builtins.hh +++ b/src/libstore/include/nix/store/builtins.hh @@ -12,8 +12,19 @@ void builtinFetchurl( const std::string & netrcData, const std::string & caFileData); -void builtinUnpackChannel( - const BasicDerivation & drv, - const std::map & outputs); +using BuiltinBuilder = + std::function & outputs)>; + +struct RegisterBuiltinBuilder +{ + typedef std::map BuiltinBuilders; + static BuiltinBuilders * builtinBuilders; + + RegisterBuiltinBuilder(const std::string & name, BuiltinBuilder && fun) + { + if (!builtinBuilders) builtinBuilders = new BuiltinBuilders; + builtinBuilders->insert_or_assign(name, std::move(fun)); + } +}; } diff --git a/src/libstore/include/nix/store/builtins/buildenv.hh b/src/libstore/include/nix/store/builtins/buildenv.hh index a0a26203716..163666c0bd4 100644 --- a/src/libstore/include/nix/store/builtins/buildenv.hh +++ b/src/libstore/include/nix/store/builtins/buildenv.hh @@ -45,8 +45,4 @@ typedef std::vector Packages; void buildProfile(const Path & out, Packages && pkgs); -void builtinBuildenv( - const BasicDerivation & drv, - const std::map & outputs); - } diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 3ba1e823f06..1b853cd231f 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -80,6 +80,8 @@ extern "C" int sandbox_init_with_parameters(const char *profile, uint64_t flags, namespace nix { +RegisterBuiltinBuilder::BuiltinBuilders * RegisterBuiltinBuilder::builtinBuilders = nullptr; + void handleDiffHook( uid_t uid, uid_t gid, const Path & tryA, const Path & tryB, @@ -2239,12 +2241,14 @@ void LocalDerivationGoal::runChild() if (drv->builder == "builtin:fetchurl") builtinFetchurl(*drv, outputs, netrcData, caFileData); - else if (drv->builder == "builtin:buildenv") - builtinBuildenv(*drv, outputs); - else if (drv->builder == "builtin:unpack-channel") - builtinUnpackChannel(*drv, outputs); - else - throw Error("unsupported builtin builder '%1%'", drv->builder.substr(8)); + else { + std::string builtinName = drv->builder.substr(8); + assert(RegisterBuiltinBuilder::builtinBuilders); + if (auto builtin = get(*RegisterBuiltinBuilder::builtinBuilders, builtinName)) + (*builtin)(*drv, outputs); + else + throw Error("unsupported builtin builder '%1%'", builtinName); + } _exit(0); } catch (std::exception & e) { writeFull(STDERR_FILENO, e.what() + std::string("\n")); From 7762dd23bb7347c96340f8a2487adfe8bd1d8573 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 1 May 2025 12:46:40 +0200 Subject: [PATCH 02/23] Put the builder context in a struct --- src/libstore/builtins/buildenv.cc | 10 +++---- src/libstore/builtins/fetchurl.cc | 28 +++++++++---------- src/libstore/builtins/unpack-channel.cc | 10 +++---- src/libstore/include/nix/store/builtins.hh | 16 +++++------ .../unix/build/local-derivation-goal.cc | 27 ++++++++---------- 5 files changed, 40 insertions(+), 51 deletions(-) diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 497bf1dd267..bd079f5cb75 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -167,17 +167,15 @@ void buildProfile(const Path & out, Packages && pkgs) debug("created %d symlinks in user environment", state.symlinks); } -static void builtinBuildenv( - const BasicDerivation & drv, - const std::map & outputs) +static void builtinBuildenv(const BuiltinBuilderContext & ctx) { auto getAttr = [&](const std::string & name) { - auto i = drv.env.find(name); - if (i == drv.env.end()) throw Error("attribute '%s' missing", name); + auto i = ctx.drv.env.find(name); + if (i == ctx.drv.env.end()) throw Error("attribute '%s' missing", name); return i->second; }; - auto out = outputs.at("out"); + auto out = ctx.outputs.at("out"); createDirs(out); /* Convert the stuff we get from the environment back into a diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 82f268d807d..18fa755580f 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -6,33 +6,29 @@ namespace nix { -void builtinFetchurl( - const BasicDerivation & drv, - const std::map & outputs, - const std::string & netrcData, - const std::string & caFileData) +static void builtinFetchurl(const BuiltinBuilderContext & ctx) { /* Make the host's netrc data available. Too bad curl requires this to be stored in a file. It would be nice if we could just pass a pointer to the data. */ - if (netrcData != "") { + if (ctx.netrcData != "") { settings.netrcFile = "netrc"; - writeFile(settings.netrcFile, netrcData, 0600); + writeFile(settings.netrcFile, ctx.netrcData, 0600); } settings.caFile = "ca-certificates.crt"; - writeFile(settings.caFile, caFileData, 0600); + writeFile(settings.caFile, ctx.caFileData, 0600); - auto out = get(drv.outputs, "out"); + auto out = get(ctx.drv.outputs, "out"); if (!out) throw Error("'builtin:fetchurl' requires an 'out' output"); - if (!(drv.type().isFixed() || drv.type().isImpure())) + if (!(ctx.drv.type().isFixed() || ctx.drv.type().isImpure())) throw Error("'builtin:fetchurl' must be a fixed-output or impure derivation"); - auto storePath = outputs.at("out"); - auto mainUrl = drv.env.at("url"); - bool unpack = getOr(drv.env, "unpack", "") == "1"; + auto storePath = ctx.outputs.at("out"); + auto mainUrl = ctx.drv.env.at("url"); + bool unpack = getOr(ctx.drv.env, "unpack", "") == "1"; /* Note: have to use a fresh fileTransfer here because we're in a forked process. */ @@ -56,8 +52,8 @@ void builtinFetchurl( else writeFile(storePath, *source); - auto executable = drv.env.find("executable"); - if (executable != drv.env.end() && executable->second == "1") { + auto executable = ctx.drv.env.find("executable"); + if (executable != ctx.drv.env.end() && executable->second == "1") { if (chmod(storePath.c_str(), 0755) == -1) throw SysError("making '%1%' executable", storePath); } @@ -79,4 +75,6 @@ void builtinFetchurl( fetch(mainUrl); } +static RegisterBuiltinBuilder registerFetchurl("fetchurl", builtinFetchurl); + } diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc index af6c4386083..6247d0a505d 100644 --- a/src/libstore/builtins/unpack-channel.cc +++ b/src/libstore/builtins/unpack-channel.cc @@ -5,17 +5,15 @@ namespace nix { namespace fs { using namespace std::filesystem; } -static void builtinUnpackChannel( - const BasicDerivation & drv, - const std::map & outputs) +static void builtinUnpackChannel(const BuiltinBuilderContext & ctx) { auto getAttr = [&](const std::string & name) -> const std::string & { - auto i = drv.env.find(name); - if (i == drv.env.end()) throw Error("attribute '%s' missing", name); + auto i = ctx.drv.env.find(name); + if (i == ctx.drv.env.end()) throw Error("attribute '%s' missing", name); return i->second; }; - fs::path out{outputs.at("out")}; + fs::path out{ctx.outputs.at("out")}; auto & channelName = getAttr("channelName"); auto & src = getAttr("src"); diff --git a/src/libstore/include/nix/store/builtins.hh b/src/libstore/include/nix/store/builtins.hh index 6d54c2a2206..6c63f4b9ee1 100644 --- a/src/libstore/include/nix/store/builtins.hh +++ b/src/libstore/include/nix/store/builtins.hh @@ -5,15 +5,15 @@ namespace nix { -// TODO: make pluggable. -void builtinFetchurl( - const BasicDerivation & drv, - const std::map & outputs, - const std::string & netrcData, - const std::string & caFileData); +struct BuiltinBuilderContext +{ + BasicDerivation & drv; + std::map outputs; + std::string netrcData; + std::string caFileData; +}; -using BuiltinBuilder = - std::function & outputs)>; +using BuiltinBuilder = std::function; struct RegisterBuiltinBuilder { diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 1b853cd231f..9d45c11453d 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -1806,15 +1806,15 @@ void LocalDerivationGoal::runChild() /* Make the contents of netrc and the CA certificate bundle available to builtin:fetchurl (which may run under a different uid and/or in a sandbox). */ - std::string netrcData; - std::string caFileData; + BuiltinBuilderContext ctx{.drv = *drv}; + if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") { try { - netrcData = readFile(settings.netrcFile); + ctx.netrcData = readFile(settings.netrcFile); } catch (SystemError &) { } try { - caFileData = readFile(settings.caFile); + ctx.caFileData = readFile(settings.caFile); } catch (SystemError &) { } } @@ -2234,21 +2234,16 @@ void LocalDerivationGoal::runChild() try { logger = makeJSONLogger(getStandardError()); - std::map outputs; for (auto & e : drv->outputs) - outputs.insert_or_assign(e.first, + ctx.outputs.insert_or_assign(e.first, worker.store.printStorePath(scratchOutputs.at(e.first))); - if (drv->builder == "builtin:fetchurl") - builtinFetchurl(*drv, outputs, netrcData, caFileData); - else { - std::string builtinName = drv->builder.substr(8); - assert(RegisterBuiltinBuilder::builtinBuilders); - if (auto builtin = get(*RegisterBuiltinBuilder::builtinBuilders, builtinName)) - (*builtin)(*drv, outputs); - else - throw Error("unsupported builtin builder '%1%'", builtinName); - } + std::string builtinName = drv->builder.substr(8); + assert(RegisterBuiltinBuilder::builtinBuilders); + if (auto builtin = get(*RegisterBuiltinBuilder::builtinBuilders, builtinName)) + (*builtin)(ctx); + else + throw Error("unsupported builtin builder '%1%'", builtinName); _exit(0); } catch (std::exception & e) { writeFull(STDERR_FILENO, e.what() + std::string("\n")); From 4d485f33df06be5f1a7c8946974b3706a912433f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 1 May 2025 13:40:51 +0200 Subject: [PATCH 03/23] Add builtin:fetch-tree This builtin builder is similar to `builtins.fetchTree` but works at build time. --- src/libfetchers/builtin.cc | 44 +++++++++++++++++++ src/libfetchers/meson.build | 1 + src/libstore/include/nix/store/builtins.hh | 3 ++ .../include/nix/store/parsed-derivations.hh | 5 +++ .../unix/build/local-derivation-goal.cc | 5 ++- 5 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 src/libfetchers/builtin.cc diff --git a/src/libfetchers/builtin.cc b/src/libfetchers/builtin.cc new file mode 100644 index 00000000000..d291d35ab22 --- /dev/null +++ b/src/libfetchers/builtin.cc @@ -0,0 +1,44 @@ +#include "nix/store/builtins.hh" +#include "nix/store/parsed-derivations.hh" +#include "nix/fetchers/fetchers.hh" +#include "nix/fetchers/fetch-settings.hh" +#include "nix/util/archive.hh" + +#include + +namespace nix { + +static void builtinFetchTree(const BuiltinBuilderContext & ctx) +{ + auto out = get(ctx.drv.outputs, "out"); + if (!out) + throw Error("'builtin:fetch-tree' requires an 'out' output"); + + if (!(ctx.drv.type().isFixed() || ctx.drv.type().isImpure())) + throw Error("'builtin:fetch-tree' must be a fixed-output or impure derivation"); + + if (!ctx.structuredAttrs) + throw Error("'builtin:fetch-tree' must have '__structuredAttrs = true'"); + + using namespace fetchers; + + fetchers::Settings fetchSettings; + + auto input = Input::fromAttrs(fetchSettings, jsonToAttrs((*ctx.structuredAttrs)["input"])); + + /* Make sure we don't use the real store because we're in a forked + process. */ + auto dummyStore = openStore("dummy://"); + + auto [accessor, lockedInput] = input.getAccessor(dummyStore); + + auto source = sinkToSource([&](Sink & sink) { + accessor->dumpPath(CanonPath::root, sink); + }); + + restorePath(ctx.outputs.at("out"), *source); +} + +static RegisterBuiltinBuilder registerUnpackChannel("fetch-tree", builtinFetchTree); + +} diff --git a/src/libfetchers/meson.build b/src/libfetchers/meson.build index 321146ca4ed..cacb2e4a03f 100644 --- a/src/libfetchers/meson.build +++ b/src/libfetchers/meson.build @@ -34,6 +34,7 @@ subdir('nix-meson-build-support/common') sources = files( 'attrs.cc', + 'builtin.cc', 'cache.cc', 'fetch-settings.cc', 'fetch-to-store.cc', diff --git a/src/libstore/include/nix/store/builtins.hh b/src/libstore/include/nix/store/builtins.hh index 6c63f4b9ee1..1c068169bc7 100644 --- a/src/libstore/include/nix/store/builtins.hh +++ b/src/libstore/include/nix/store/builtins.hh @@ -3,11 +3,14 @@ #include "nix/store/derivations.hh" +#include + namespace nix { struct BuiltinBuilderContext { BasicDerivation & drv; + nlohmann::json * structuredAttrs; std::map outputs; std::string netrcData; std::string caFileData; diff --git a/src/libstore/include/nix/store/parsed-derivations.hh b/src/libstore/include/nix/store/parsed-derivations.hh index d65db6133ba..cfa6d852bff 100644 --- a/src/libstore/include/nix/store/parsed-derivations.hh +++ b/src/libstore/include/nix/store/parsed-derivations.hh @@ -44,6 +44,11 @@ public: } std::optional prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths); + + nlohmann::json * getStructuredAttrs() + { + return structuredAttrs.get(); + } }; std::string writeStructuredAttrsShell(const nlohmann::json & json); diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 9d45c11453d..30ee261ed92 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -1806,7 +1806,10 @@ void LocalDerivationGoal::runChild() /* Make the contents of netrc and the CA certificate bundle available to builtin:fetchurl (which may run under a different uid and/or in a sandbox). */ - BuiltinBuilderContext ctx{.drv = *drv}; + BuiltinBuilderContext ctx{ + .drv = *drv, + .structuredAttrs = parsedDrv->getStructuredAttrs(), + }; if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") { try { From 464f408a61c779265e3915a25b5865748346eddd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 1 May 2025 17:49:58 +0200 Subject: [PATCH 04/23] Pass tmpDirInSandbox to the builtin builders --- src/libstore/include/nix/store/builtins.hh | 1 + src/libstore/unix/build/local-derivation-goal.cc | 1 + 2 files changed, 2 insertions(+) diff --git a/src/libstore/include/nix/store/builtins.hh b/src/libstore/include/nix/store/builtins.hh index 1c068169bc7..974c582c92b 100644 --- a/src/libstore/include/nix/store/builtins.hh +++ b/src/libstore/include/nix/store/builtins.hh @@ -14,6 +14,7 @@ struct BuiltinBuilderContext std::map outputs; std::string netrcData; std::string caFileData; + Path tmpDirInSandbox; }; using BuiltinBuilder = std::function; diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 30ee261ed92..e79410c6ac7 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -1809,6 +1809,7 @@ void LocalDerivationGoal::runChild() BuiltinBuilderContext ctx{ .drv = *drv, .structuredAttrs = parsedDrv->getStructuredAttrs(), + .tmpDirInSandbox = tmpDirInSandbox, }; if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") { From d3ff470b9a7ad05bee09dfa20ede8190345bc0c8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 1 May 2025 17:52:02 +0200 Subject: [PATCH 05/23] Move fetchSettings back to libfetchers This is needed for builtin:fetch-tree to get access to it. --- src/libcmd/common-eval-args.cc | 4 ---- src/libcmd/include/nix/cmd/common-eval-args.hh | 3 --- src/libfetchers/fetch-settings.cc | 9 +++++++++ src/libfetchers/include/nix/fetchers/fetch-settings.hh | 9 +++++++++ 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 21584b74aff..844038056b8 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -20,10 +20,6 @@ namespace nix { namespace fs { using namespace std::filesystem; } -fetchers::Settings fetchSettings; - -static GlobalConfig::Register rFetchSettings(&fetchSettings); - EvalSettings evalSettings { settings.readOnlyMode, { diff --git a/src/libcmd/include/nix/cmd/common-eval-args.hh b/src/libcmd/include/nix/cmd/common-eval-args.hh index 6f3367e58e9..aefc3cc31e6 100644 --- a/src/libcmd/include/nix/cmd/common-eval-args.hh +++ b/src/libcmd/include/nix/cmd/common-eval-args.hh @@ -22,9 +22,6 @@ struct SourcePath; namespace flake { struct Settings; } -/** - * @todo Get rid of global setttings variables - */ extern fetchers::Settings fetchSettings; /** diff --git a/src/libfetchers/fetch-settings.cc b/src/libfetchers/fetch-settings.cc index 4b4e4e29d98..27526a79705 100644 --- a/src/libfetchers/fetch-settings.cc +++ b/src/libfetchers/fetch-settings.cc @@ -1,4 +1,5 @@ #include "nix/fetchers/fetch-settings.hh" +#include "nix/util/config-global.hh" namespace nix::fetchers { @@ -7,3 +8,11 @@ Settings::Settings() } } + +namespace nix { + +fetchers::Settings fetchSettings; + +static GlobalConfig::Register rFetchSettings(&fetchSettings); + +} diff --git a/src/libfetchers/include/nix/fetchers/fetch-settings.hh b/src/libfetchers/include/nix/fetchers/fetch-settings.hh index 831a18bf0cd..2dd213d0c4a 100644 --- a/src/libfetchers/include/nix/fetchers/fetch-settings.hh +++ b/src/libfetchers/include/nix/fetchers/fetch-settings.hh @@ -109,3 +109,12 @@ struct Settings : public Config }; } + +namespace nix { + +/** + * @todo Get rid of global setttings variables + */ +extern fetchers::Settings fetchSettings; + +} From 94facc9b32928ccb56bbf34abab5569ef63ce88f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 1 May 2025 19:32:02 +0200 Subject: [PATCH 06/23] Hack to disable the fetcher cache in forked processes --- src/libfetchers/cache.cc | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index d369d213f51..7d3013e416e 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -54,6 +54,8 @@ struct CacheImpl : Cache const Key & key, const Attrs & value) override { + if (disabled()) return; + _state.lock()->upsert.use() (key.first) (attrsToJSON(key.second).dump()) @@ -81,9 +83,20 @@ struct CacheImpl : Cache return {}; } + pid_t originalPid = getpid(); + + bool disabled() + { + // FIXME: Temporary hack to disable the cache in + // builtin:fetch-tree builders. + return getpid() != originalPid; + } + std::optional lookupExpired( const Key & key) override { + if (disabled()) return {}; + auto state(_state.lock()); auto keyJSON = attrsToJSON(key.second).dump(); @@ -111,6 +124,8 @@ struct CacheImpl : Cache Attrs value, const StorePath & storePath) override { + if (disabled()) return; + /* Add the store prefix to the cache key to handle multiple store prefixes. */ key.second.insert_or_assign("store", store.storeDir); From 961b3a1c2fade9649c258cbd60cc9105e82f1454 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 1 May 2025 19:32:43 +0200 Subject: [PATCH 07/23] builtin:fetch-tree: Propagate access tokens, set cache directory --- src/libfetchers/builtin.cc | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/libfetchers/builtin.cc b/src/libfetchers/builtin.cc index d291d35ab22..8d151f4ba98 100644 --- a/src/libfetchers/builtin.cc +++ b/src/libfetchers/builtin.cc @@ -20,11 +20,18 @@ static void builtinFetchTree(const BuiltinBuilderContext & ctx) if (!ctx.structuredAttrs) throw Error("'builtin:fetch-tree' must have '__structuredAttrs = true'"); + setenv("NIX_CACHE_HOME", ctx.tmpDirInSandbox.c_str(), 1); + using namespace fetchers; - fetchers::Settings fetchSettings; + fetchers::Settings myFetchSettings; + myFetchSettings.accessTokens = fetchSettings.accessTokens.get(); + + // FIXME: disable use of the git/tarball cache + + auto input = Input::fromAttrs(myFetchSettings, jsonToAttrs((*ctx.structuredAttrs)["input"])); - auto input = Input::fromAttrs(fetchSettings, jsonToAttrs((*ctx.structuredAttrs)["input"])); + std::cerr << fmt("fetching '%s'...\n", input.to_string()); /* Make sure we don't use the real store because we're in a forked process. */ From 99f35e15f6f878b52f1357f0cf70bb318a7dd587 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 1 May 2025 19:33:14 +0200 Subject: [PATCH 08/23] Allow flake inputs to be fetched at build time This only works for non-flake inputs. Example: inputs.repo1 = { type = "github"; owner = "DeterminateSystems"; repo = "blabla"; flake = false; buildTime = true; }; `call-flake.nix` maps this to a builtin:fetch-tree derivation. Thus you can pass it to other derivations, and it will be fetched at build time rather than eval time. (It will still be fetched at eval time to create/update locks.) Importing from such an input triggers IFD. --- src/libflake/call-flake.nix | 13 ++++++++++++- src/libflake/flake.cc | 8 +++++++- src/libflake/include/nix/flake/flake.hh | 12 +++++++++--- src/libflake/include/nix/flake/lockfile.hh | 3 +++ src/libflake/lockfile.cc | 3 +++ 5 files changed, 34 insertions(+), 5 deletions(-) diff --git a/src/libflake/call-flake.nix b/src/libflake/call-flake.nix index fe326291f1f..4dbf454801c 100644 --- a/src/libflake/call-flake.nix +++ b/src/libflake/call-flake.nix @@ -50,7 +50,17 @@ let if node ? parent then parentDir + ("/" + dir) else dir; sourceInfo = - if overrides ? ${key} then + if node.buildTime or false then + derivation { + name = "source"; + builder = "builtin:fetch-tree"; + system = "builtin"; + __structuredAttrs = true; + input = node.locked; + outputHashMode = "recursive"; + outputHash = node.locked.narHash; + } + else if overrides ? ${key} then overrides.${key}.sourceInfo else if node.locked.type == "path" && builtins.substring 0 1 node.locked.path != "/" then parentNode.sourceInfo @@ -97,6 +107,7 @@ let result = if node.flake or true then assert builtins.isFunction flake.outputs; + assert !(node.buildTime or false); result else sourceInfo; diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index a85acf4b282..57af37b2fd5 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -99,6 +99,7 @@ static FlakeInput parseFlakeInput( auto sUrl = state.symbols.create("url"); auto sFlake = state.symbols.create("flake"); auto sFollows = state.symbols.create("follows"); + auto sBuildTime = state.symbols.create("buildTime"); fetchers::Attrs attrs; std::optional url; @@ -123,6 +124,9 @@ static FlakeInput parseFlakeInput( } else if (attr.name == sFlake) { expectType(state, nBool, *attr.value, attr.pos); input.isFlake = attr.value->boolean(); + } else if (attr.name == sBuildTime) { + expectType(state, nBool, *attr.value, attr.pos); + input.buildTime = attr.value->boolean(); } else if (attr.name == sInputs) { input.overrides = parseFlakeInputs(state, attr.value, attr.pos, lockRootAttrPath, flakeDir, false).first; } else if (attr.name == sFollows) { @@ -593,6 +597,7 @@ LockedFlake lockFlake( oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake, + oldLock->buildTime, oldLock->parentInputAttrPath); node->inputs.insert_or_assign(id, childNode); @@ -702,6 +707,7 @@ LockedFlake lockFlake( inputFlake.lockedRef, ref, true, + input.buildTime, overridenParentPath); node->inputs.insert_or_assign(id, childNode); @@ -751,7 +757,7 @@ LockedFlake lockFlake( } }(); - auto childNode = make_ref(lockedRef, ref, false, overridenParentPath); + auto childNode = make_ref(lockedRef, ref, false, input.buildTime, overridenParentPath); nodePaths.emplace(childNode, path); diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index fdac4397f65..577176f0b9f 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -43,12 +43,18 @@ typedef std::map FlakeInputs; struct FlakeInput { std::optional ref; + /** - * true = process flake to get outputs - * - * false = (fetched) static source path + * Whether to call the `flake.nix` file in this input to get its outputs. */ bool isFlake = true; + + /** + * Whether to fetch this input at evaluation time or at build + * time. + */ + bool buildTime = false; + std::optional follows; FlakeInputs overrides; }; diff --git a/src/libflake/include/nix/flake/lockfile.hh b/src/libflake/include/nix/flake/lockfile.hh index 97bd7a49538..a1a95780706 100644 --- a/src/libflake/include/nix/flake/lockfile.hh +++ b/src/libflake/include/nix/flake/lockfile.hh @@ -37,6 +37,7 @@ struct LockedNode : Node { FlakeRef lockedRef, originalRef; bool isFlake = true; + bool buildTime = false; /* The node relative to which relative source paths (e.g. 'path:../foo') are interpreted. */ @@ -46,10 +47,12 @@ struct LockedNode : Node const FlakeRef & lockedRef, const FlakeRef & originalRef, bool isFlake = true, + bool buildTime = false, std::optional parentInputAttrPath = {}) : lockedRef(std::move(lockedRef)) , originalRef(std::move(originalRef)) , isFlake(isFlake) + , buildTime(buildTime) , parentInputAttrPath(std::move(parentInputAttrPath)) { } diff --git a/src/libflake/lockfile.cc b/src/libflake/lockfile.cc index 646516caf2a..690c1a49468 100644 --- a/src/libflake/lockfile.cc +++ b/src/libflake/lockfile.cc @@ -44,6 +44,7 @@ LockedNode::LockedNode( : lockedRef(getFlakeRef(fetchSettings, json, "locked", "info")) // FIXME: remove "info" , originalRef(getFlakeRef(fetchSettings, json, "original", nullptr)) , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) + , buildTime(json.find("buildTime") != json.end() ? (bool) json["buildTime"] : false) , parentInputAttrPath(json.find("parent") != json.end() ? (std::optional) json["parent"] : std::nullopt) { if (!lockedRef.input.isLocked() && !lockedRef.input.isRelative()) { @@ -216,6 +217,8 @@ std::pair LockFile::toJSON() const n["locked"].erase("__final"); if (!lockedNode->isFlake) n["flake"] = false; + if (lockedNode->buildTime) + n["buildTime"] = true; if (lockedNode->parentInputAttrPath) n["parent"] = *lockedNode->parentInputAttrPath; } From febe4de1002184414987be5f71fb08ed3cb3842d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 1 May 2025 19:48:14 +0200 Subject: [PATCH 09/23] Formatting --- src/libfetchers/builtin.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/libfetchers/builtin.cc b/src/libfetchers/builtin.cc index 8d151f4ba98..cdef20461c5 100644 --- a/src/libfetchers/builtin.cc +++ b/src/libfetchers/builtin.cc @@ -39,9 +39,7 @@ static void builtinFetchTree(const BuiltinBuilderContext & ctx) auto [accessor, lockedInput] = input.getAccessor(dummyStore); - auto source = sinkToSource([&](Sink & sink) { - accessor->dumpPath(CanonPath::root, sink); - }); + auto source = sinkToSource([&](Sink & sink) { accessor->dumpPath(CanonPath::root, sink); }); restorePath(ctx.outputs.at("out"), *source); } From c3270b9026e2ee06b829dab8b36d594f942e045a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 14 May 2025 13:29:25 +0200 Subject: [PATCH 10/23] Always add a NAR hash for build-time inputs --- src/libexpr/include/nix/expr/eval.hh | 3 ++- src/libexpr/paths.cc | 4 ++-- src/libflake/flake.cc | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index d82baddb153..9e83a90f72f 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -458,7 +458,8 @@ public: fetchers::Input & input, const fetchers::Input & originalInput, ref accessor, - bool requireLockable); + bool requireLockable, + bool forceNarHash = false); /** * Parse a Nix expression from the specified file. diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index 3aaca232829..cce870bba4b 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -68,7 +68,7 @@ std::string EvalState::computeBaseName(const SourcePath & path) } StorePath EvalState::mountInput( - fetchers::Input & input, const fetchers::Input & originalInput, ref accessor, bool requireLockable) + fetchers::Input & input, const fetchers::Input & originalInput, ref accessor, bool requireLockable, bool forceNarHash) { auto storePath = settings.lazyTrees ? StorePath::random(input.getName()) : fetchToStore(*store, accessor, FetchMode::Copy, input.getName()); @@ -77,7 +77,7 @@ StorePath EvalState::mountInput( storeFS->mount(CanonPath(store->printStorePath(storePath)), accessor); - if (requireLockable && (!settings.lazyTrees || !input.isLocked()) && !input.getNarHash()) { + if ((forceNarHash || (requireLockable && (!settings.lazyTrees || !input.isLocked()))) && !input.getNarHash()) { auto narHash = accessor->hashPath(CanonPath::root); input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); } diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 57af37b2fd5..1e5ac588a52 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -751,7 +751,7 @@ LockedFlake lockFlake( warnRegistry(resolvedRef); return { - state.storePath(state.mountInput(lockedRef.input, input.ref->input, cachedInput.accessor, true)), + state.storePath(state.mountInput(lockedRef.input, input.ref->input, cachedInput.accessor, true, true)), lockedRef }; } From 655b26c6a83907b5c921addea6f1ac9b839e3419 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 14 May 2025 13:33:41 +0200 Subject: [PATCH 11/23] Revert "Hack to disable the fetcher cache in forked processes" This reverts commit 94facc9b32928ccb56bbf34abab5569ef63ce88f. --- src/libfetchers/cache.cc | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index 7d3013e416e..d369d213f51 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -54,8 +54,6 @@ struct CacheImpl : Cache const Key & key, const Attrs & value) override { - if (disabled()) return; - _state.lock()->upsert.use() (key.first) (attrsToJSON(key.second).dump()) @@ -83,20 +81,9 @@ struct CacheImpl : Cache return {}; } - pid_t originalPid = getpid(); - - bool disabled() - { - // FIXME: Temporary hack to disable the cache in - // builtin:fetch-tree builders. - return getpid() != originalPid; - } - std::optional lookupExpired( const Key & key) override { - if (disabled()) return {}; - auto state(_state.lock()); auto keyJSON = attrsToJSON(key.second).dump(); @@ -124,8 +111,6 @@ struct CacheImpl : Cache Attrs value, const StorePath & storePath) override { - if (disabled()) return; - /* Add the store prefix to the cache key to handle multiple store prefixes. */ key.second.insert_or_assign("store", store.storeDir); From 38b45aa049c88f48be0b1e9dee61b8dd5b9dc9c6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 14 May 2025 14:24:57 +0200 Subject: [PATCH 12/23] Sync: Support moving out of another Sync --- src/libutil/include/nix/util/sync.hh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libutil/include/nix/util/sync.hh b/src/libutil/include/nix/util/sync.hh index 0c3e1f52836..4b9d546d2b7 100644 --- a/src/libutil/include/nix/util/sync.hh +++ b/src/libutil/include/nix/util/sync.hh @@ -39,6 +39,7 @@ public: SyncBase() { } SyncBase(const T & data) : data(data) { } SyncBase(T && data) noexcept : data(std::move(data)) { } + SyncBase(SyncBase && other) noexcept : data(std::move(*other.lock())) { } template class Lock From 0d440c97a63ac4b3a3cb2d5d194821871cfa563b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 14 May 2025 14:17:57 +0200 Subject: [PATCH 13/23] Remove global fetcher cache The cache is now part of fetchers::Settings. --- src/libcmd/common-eval-args.cc | 20 ++++++++++++++++--- src/libcmd/installable-value.cc | 2 +- src/libexpr/eval.cc | 3 ++- src/libexpr/paths.cc | 9 +++++---- src/libexpr/primops.cc | 1 + src/libexpr/primops/fetchTree.cc | 3 ++- src/libfetchers/cache.cc | 9 ++++++--- src/libfetchers/fetch-to-store.cc | 12 ++++++----- src/libfetchers/fetchers.cc | 2 +- src/libfetchers/git-utils.cc | 7 ++++--- src/libfetchers/git.cc | 16 +++++++-------- src/libfetchers/github.cc | 10 +++++----- src/libfetchers/include/nix/fetchers/cache.hh | 2 -- .../include/nix/fetchers/fetch-settings.hh | 9 +++++++++ .../include/nix/fetchers/fetch-to-store.hh | 1 + .../include/nix/fetchers/git-utils.hh | 4 ++-- .../include/nix/fetchers/tarball.hh | 1 + src/libfetchers/mercurial.cc | 10 +++++----- src/libfetchers/path.cc | 3 ++- src/libfetchers/registry.cc | 2 +- src/libfetchers/tarball.cc | 16 +++++++++------ src/libflake/flake.cc | 2 +- src/nix-channel/nix-channel.cc | 7 ++++--- src/nix/flake.cc | 2 +- 24 files changed, 96 insertions(+), 57 deletions(-) diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 844038056b8..376dfd6a571 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -30,7 +30,12 @@ EvalSettings evalSettings { auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false); debug("fetching flake search path element '%s''", rest); auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store); - auto storePath = nix::fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName()); + auto storePath = nix::fetchToStore( + state.fetchSettings, + *state.store, + SourcePath(accessor), + FetchMode::Copy, + lockedRef.input.getName()); state.allowPath(storePath); return state.storePath(storePath); }, @@ -173,14 +178,23 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas state.store, state.fetchSettings, EvalSettings::resolvePseudoUrl(s)); - auto storePath = fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy); + auto storePath = fetchToStore( + state.fetchSettings, + *state.store, + SourcePath(accessor), + FetchMode::Copy); return state.storePath(storePath); } else if (hasPrefix(s, "flake:")) { auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false); auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store); - auto storePath = nix::fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName()); + auto storePath = nix::fetchToStore( + state.fetchSettings, + *state.store, + SourcePath(accessor), + FetchMode::Copy, + lockedRef.input.getName()); state.allowPath(storePath); return state.storePath(storePath); } diff --git a/src/libcmd/installable-value.cc b/src/libcmd/installable-value.cc index 4eb4993b14e..f5a129205c8 100644 --- a/src/libcmd/installable-value.cc +++ b/src/libcmd/installable-value.cc @@ -45,7 +45,7 @@ ref InstallableValue::require(ref installable) std::optional InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx) { if (v.type() == nPath) { - auto storePath = fetchToStore(*state->store, v.path(), FetchMode::Copy); + auto storePath = fetchToStore(state->fetchSettings, *state->store, v.path(), FetchMode::Copy); return {{ .path = DerivedPath::Opaque { .path = std::move(storePath), diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 85c044c2fa9..ad7191cb6de 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2427,6 +2427,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat ? *dstPathCached : [&]() { auto dstPath = fetchToStore( + fetchSettings, *store, path.resolveSymlinks(SymlinkResolution::Ancestors), settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, @@ -3139,7 +3140,7 @@ std::optional EvalState::resolveLookupPathPath(const LookupPath::Pat store, fetchSettings, EvalSettings::resolvePseudoUrl(value)); - auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy); + auto storePath = fetchToStore(fetchSettings, *store, SourcePath(accessor), FetchMode::Copy); return finish(this->storePath(storePath)); } catch (Error & e) { logWarning({ diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index cce870bba4b..dab31c663b4 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -24,7 +24,7 @@ StorePath EvalState::devirtualize(const StorePath & path, StringMap * rewrites) { if (auto mount = storeFS->getMount(CanonPath(store->printStorePath(path)))) { auto storePath = fetchToStore( - *store, SourcePath{ref(mount)}, settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, path.name()); + fetchSettings, *store, SourcePath{ref(mount)}, settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, path.name()); assert(storePath.name() == path.name()); if (rewrites) rewrites->emplace(path.hashPart(), storePath.hashPart()); @@ -61,7 +61,7 @@ std::string EvalState::computeBaseName(const SourcePath & path) "This can typically be avoided by rewriting an attribute like `src = ./.` " "to `src = builtins.path { path = ./.; name = \"source\"; }`.", path); - return std::string(fetchToStore(*store, path, FetchMode::DryRun, storePath->name()).to_string()); + return std::string(fetchToStore(fetchSettings, *store, path, FetchMode::DryRun, storePath->name()).to_string()); } } return std::string(path.baseName()); @@ -70,8 +70,9 @@ std::string EvalState::computeBaseName(const SourcePath & path) StorePath EvalState::mountInput( fetchers::Input & input, const fetchers::Input & originalInput, ref accessor, bool requireLockable, bool forceNarHash) { - auto storePath = settings.lazyTrees ? StorePath::random(input.getName()) - : fetchToStore(*store, accessor, FetchMode::Copy, input.getName()); + auto storePath = settings.lazyTrees + ? StorePath::random(input.getName()) + : fetchToStore(fetchSettings, *store, accessor, FetchMode::Copy, input.getName()); allowPath(storePath); // FIXME: should just whitelist the entire virtual store diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index c6a97fdaee0..665a3a815ab 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2539,6 +2539,7 @@ static void addPath( if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { // FIXME: make this lazy? auto dstPath = fetchToStore( + state.fetchSettings, *state.store, path.resolveSymlinks(), settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index c82fb82c5f7..61704b7e6cd 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -533,11 +533,12 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v auto storePath = unpack ? fetchToStore( + state.fetchSettings, *state.store, fetchers::downloadTarball(state.store, state.fetchSettings, *url), FetchMode::Copy, name) - : fetchers::downloadFile(state.store, *url, name).storePath; + : fetchers::downloadFile(state.store, state.fetchSettings, *url, name).storePath; if (expectedHash) { auto hash = unpack diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index d369d213f51..9a2531ba526 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -1,4 +1,5 @@ #include "nix/fetchers/cache.hh" +#include "nix/fetchers/fetch-settings.hh" #include "nix/util/users.hh" #include "nix/store/sqlite.hh" #include "nix/util/sync.hh" @@ -162,10 +163,12 @@ struct CacheImpl : Cache } }; -ref getCache() +ref Settings::getCache() const { - static auto cache = std::make_shared(); - return ref(cache); + auto cache(_cache.lock()); + if (!*cache) + *cache = std::make_shared(); + return ref(*cache); } } diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc index f1b02f4e0a8..f7ab32322ef 100644 --- a/src/libfetchers/fetch-to-store.cc +++ b/src/libfetchers/fetch-to-store.cc @@ -1,13 +1,14 @@ #include "nix/fetchers/fetch-to-store.hh" #include "nix/fetchers/fetchers.hh" +#include "nix/fetchers/fetch-settings.hh" namespace nix { fetchers::Cache::Key makeFetchToStoreCacheKey( - const std::string &name, - const std::string &fingerprint, + const std::string & name, + const std::string & fingerprint, ContentAddressMethod method, - const std::string &path) + const std::string & path) { return fetchers::Cache::Key{"fetchToStore", { {"name", name}, @@ -19,6 +20,7 @@ fetchers::Cache::Key makeFetchToStoreCacheKey( } StorePath fetchToStore( + const fetchers::Settings & settings, Store & store, const SourcePath & path, FetchMode mode, @@ -34,7 +36,7 @@ StorePath fetchToStore( if (!filter && path.accessor->fingerprint) { cacheKey = makeFetchToStoreCacheKey(std::string{name}, *path.accessor->fingerprint, method, path.path.abs()); - if (auto res = fetchers::getCache()->lookupStorePath(*cacheKey, store)) { + if (auto res = settings.getCache()->lookupStorePath(*cacheKey, store)) { debug("store path cache hit for '%s'", path); return res->storePath; } @@ -56,7 +58,7 @@ StorePath fetchToStore( debug(mode == FetchMode::DryRun ? "hashed '%s'" : "copied '%s' to '%s'", path, store.printStorePath(storePath)); if (cacheKey && mode == FetchMode::Copy) - fetchers::getCache()->upsert(*cacheKey, store, {}, storePath); + settings.getCache()->upsert(*cacheKey, store, {}, storePath); return storePath; } diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 33301933ca5..91c809f8e70 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -195,7 +195,7 @@ std::tuple, Input> Input::fetchToStore(ref try { auto [accessor, result] = getAccessorUnchecked(store); - auto storePath = nix::fetchToStore(*store, SourcePath(accessor), FetchMode::Copy, result.getName()); + auto storePath = nix::fetchToStore(*settings, *store, SourcePath(accessor), FetchMode::Copy, result.getName()); auto narHash = store->queryPathInfo(storePath)->narHash; result.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 7e1f085f599..d2c9e0c9b43 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -1,6 +1,7 @@ #include "nix/fetchers/git-utils.hh" #include "nix/fetchers/git-lfs-fetch.hh" #include "nix/fetchers/cache.hh" +#include "nix/fetchers/fetch-settings.hh" #include "nix/util/finally.hh" #include "nix/util/processes.hh" #include "nix/util/signals.hh" @@ -610,18 +611,18 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this throw Error("Commit signature verification on commit %s failed: %s", rev.gitRev(), output); } - Hash treeHashToNarHash(const Hash & treeHash) override + Hash treeHashToNarHash(const fetchers::Settings & settings, const Hash & treeHash) override { auto accessor = getAccessor(treeHash, false, ""); fetchers::Cache::Key cacheKey{"treeHashToNarHash", {{"treeHash", treeHash.gitRev()}}}; - if (auto res = fetchers::getCache()->lookup(cacheKey)) + if (auto res = settings.getCache()->lookup(cacheKey)) return Hash::parseAny(fetchers::getStrAttr(*res, "narHash"), HashAlgorithm::SHA256); auto narHash = accessor->hashPath(CanonPath::root); - fetchers::getCache()->upsert(cacheKey, fetchers::Attrs({{"narHash", narHash.to_string(HashFormat::SRI, true)}})); + settings.getCache()->upsert(cacheKey, fetchers::Attrs({{"narHash", narHash.to_string(HashFormat::SRI, true)}})); return narHash; } diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index ef74397ff90..a38d8fbe21b 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -483,11 +483,11 @@ struct GitInputScheme : InputScheme return repoInfo; } - uint64_t getLastModified(const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const + uint64_t getLastModified(const Settings & settings, const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const { Cache::Key key{"gitLastModified", {{"rev", rev.gitRev()}}}; - auto cache = getCache(); + auto cache = settings.getCache(); if (auto res = cache->lookup(key)) return getIntAttr(*res, "lastModified"); @@ -499,11 +499,11 @@ struct GitInputScheme : InputScheme return lastModified; } - uint64_t getRevCount(const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const + uint64_t getRevCount(const Settings & settings, const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const { Cache::Key key{"gitRevCount", {{"rev", rev.gitRev()}}}; - auto cache = getCache(); + auto cache = settings.getCache(); if (auto revCountAttrs = cache->lookup(key)) return getIntAttr(*revCountAttrs, "revCount"); @@ -679,12 +679,12 @@ struct GitInputScheme : InputScheme Attrs infoAttrs({ {"rev", rev.gitRev()}, - {"lastModified", getLastModified(repoInfo, repoDir, rev)}, + {"lastModified", getLastModified(*input.settings, repoInfo, repoDir, rev)}, }); if (!getShallowAttr(input)) infoAttrs.insert_or_assign("revCount", - getRevCount(repoInfo, repoDir, rev)); + getRevCount(*input.settings, repoInfo, repoDir, rev)); printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg()); @@ -800,7 +800,7 @@ struct GitInputScheme : InputScheme input.attrs.insert_or_assign("rev", rev.gitRev()); input.attrs.insert_or_assign("revCount", - rev == nullRev ? 0 : getRevCount(repoInfo, repoPath, rev)); + rev == nullRev ? 0 : getRevCount(*input.settings, repoInfo, repoPath, rev)); verifyCommit(input, repo); } else { @@ -819,7 +819,7 @@ struct GitInputScheme : InputScheme input.attrs.insert_or_assign( "lastModified", repoInfo.workdirInfo.headRev - ? getLastModified(repoInfo, repoPath, *repoInfo.workdirInfo.headRev) + ? getLastModified(*input.settings, repoInfo, repoPath, *repoInfo.workdirInfo.headRev) : 0); return {accessor, std::move(input)}; diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index bb82f751fd7..9af620e903b 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -262,7 +262,7 @@ struct GitArchiveInputScheme : InputScheme input.attrs.erase("ref"); input.attrs.insert_or_assign("rev", rev->gitRev()); - auto cache = getCache(); + auto cache = input.settings->getCache(); Cache::Key treeHashKey{"gitRevToTreeHash", {{"rev", rev->gitRev()}}}; Cache::Key lastModifiedKey{"gitRevToLastModified", {{"rev", rev->gitRev()}}}; @@ -406,7 +406,7 @@ struct GitHubInputScheme : GitArchiveInputScheme auto json = nlohmann::json::parse( readFile( store->toRealPath( - downloadFile(store, url, "source", headers).storePath))); + downloadFile(store, *input.settings, url, "source", headers).storePath))); return RefInfo { .rev = Hash::parseAny(std::string { json["sha"] }, HashAlgorithm::SHA1), @@ -480,7 +480,7 @@ struct GitLabInputScheme : GitArchiveInputScheme auto json = nlohmann::json::parse( readFile( store->toRealPath( - downloadFile(store, url, "source", headers).storePath))); + downloadFile(store, *input.settings, url, "source", headers).storePath))); if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) { return RefInfo { @@ -550,7 +550,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme std::string refUri; if (ref == "HEAD") { auto file = store->toRealPath( - downloadFile(store, fmt("%s/HEAD", base_url), "source", headers).storePath); + downloadFile(store, *input.settings, fmt("%s/HEAD", base_url), "source", headers).storePath); std::ifstream is(file); std::string line; getline(is, line); @@ -566,7 +566,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme std::regex refRegex(refUri); auto file = store->toRealPath( - downloadFile(store, fmt("%s/info/refs", base_url), "source", headers).storePath); + downloadFile(store, *input.settings, fmt("%s/info/refs", base_url), "source", headers).storePath); std::ifstream is(file); std::string line; diff --git a/src/libfetchers/include/nix/fetchers/cache.hh b/src/libfetchers/include/nix/fetchers/cache.hh index 5b9319d774b..6ac693183f9 100644 --- a/src/libfetchers/include/nix/fetchers/cache.hh +++ b/src/libfetchers/include/nix/fetchers/cache.hh @@ -91,6 +91,4 @@ struct Cache Store & store) = 0; }; -ref getCache(); - } diff --git a/src/libfetchers/include/nix/fetchers/fetch-settings.hh b/src/libfetchers/include/nix/fetchers/fetch-settings.hh index 2dd213d0c4a..7b2c5720074 100644 --- a/src/libfetchers/include/nix/fetchers/fetch-settings.hh +++ b/src/libfetchers/include/nix/fetchers/fetch-settings.hh @@ -3,6 +3,8 @@ #include "nix/util/types.hh" #include "nix/util/configuration.hh" +#include "nix/util/ref.hh" +#include "nix/util/sync.hh" #include #include @@ -11,6 +13,8 @@ namespace nix::fetchers { +struct Cache; + struct Settings : public Config { Settings(); @@ -106,6 +110,11 @@ struct Settings : public Config When empty, disables the global flake registry. )"}; + + ref getCache() const; + +private: + mutable Sync> _cache; }; } diff --git a/src/libfetchers/include/nix/fetchers/fetch-to-store.hh b/src/libfetchers/include/nix/fetchers/fetch-to-store.hh index 44c33c147ed..a52d567ecfb 100644 --- a/src/libfetchers/include/nix/fetchers/fetch-to-store.hh +++ b/src/libfetchers/include/nix/fetchers/fetch-to-store.hh @@ -15,6 +15,7 @@ enum struct FetchMode { DryRun, Copy }; * Copy the `path` to the Nix store. */ StorePath fetchToStore( + const fetchers::Settings & settings, Store & store, const SourcePath & path, FetchMode mode, diff --git a/src/libfetchers/include/nix/fetchers/git-utils.hh b/src/libfetchers/include/nix/fetchers/git-utils.hh index 1506f8509e4..2926deb4f44 100644 --- a/src/libfetchers/include/nix/fetchers/git-utils.hh +++ b/src/libfetchers/include/nix/fetchers/git-utils.hh @@ -5,7 +5,7 @@ namespace nix { -namespace fetchers { struct PublicKey; } +namespace fetchers { struct PublicKey; struct Settings; } /** * A sink that writes into a Git repository. Note that nothing may be written @@ -115,7 +115,7 @@ struct GitRepo * Given a Git tree hash, compute the hash of its NAR * serialisation. This is memoised on-disk. */ - virtual Hash treeHashToNarHash(const Hash & treeHash) = 0; + virtual Hash treeHashToNarHash(const fetchers::Settings & settings, const Hash & treeHash) = 0; /** * If the specified Git object is a directory with a single entry diff --git a/src/libfetchers/include/nix/fetchers/tarball.hh b/src/libfetchers/include/nix/fetchers/tarball.hh index 691142091fa..2c5ea209f01 100644 --- a/src/libfetchers/include/nix/fetchers/tarball.hh +++ b/src/libfetchers/include/nix/fetchers/tarball.hh @@ -26,6 +26,7 @@ struct DownloadFileResult DownloadFileResult downloadFile( ref store, + const Settings & settings, const std::string & url, const std::string & name, const Headers & headers = {}); diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index eb6bdd1ebdc..a35e7f47646 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -253,13 +253,13 @@ struct MercurialInputScheme : InputScheme }}; if (!input.getRev()) { - if (auto res = getCache()->lookupWithTTL(refToRevKey)) + if (auto res = input.settings->getCache()->lookupWithTTL(refToRevKey)) input.attrs.insert_or_assign("rev", getRevAttr(*res, "rev").gitRev()); } /* If we have a rev, check if we have a cached store path. */ if (auto rev = input.getRev()) { - if (auto res = getCache()->lookupStorePath(revInfoKey(*rev), *store)) + if (auto res = input.settings->getCache()->lookupStorePath(revInfoKey(*rev), *store)) return makeResult(res->value, res->storePath); } @@ -309,7 +309,7 @@ struct MercurialInputScheme : InputScheme /* Now that we have the rev, check the cache again for a cached store path. */ - if (auto res = getCache()->lookupStorePath(revInfoKey(rev), *store)) + if (auto res = input.settings->getCache()->lookupStorePath(revInfoKey(rev), *store)) return makeResult(res->value, res->storePath); Path tmpDir = createTempDir(); @@ -326,9 +326,9 @@ struct MercurialInputScheme : InputScheme }); if (!origRev) - getCache()->upsert(refToRevKey, {{"rev", rev.gitRev()}}); + input.settings->getCache()->upsert(refToRevKey, {{"rev", rev.gitRev()}}); - getCache()->upsert(revInfoKey(rev), *store, infoAttrs, storePath); + input.settings->getCache()->upsert(revInfoKey(rev), *store, infoAttrs, storePath); return makeResult(infoAttrs, std::move(storePath)); } diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index ff39cb02f9d..38b1918280c 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -4,6 +4,7 @@ #include "nix/fetchers/store-path-accessor.hh" #include "nix/fetchers/cache.hh" #include "nix/fetchers/fetch-to-store.hh" +#include "nix/fetchers/fetch-settings.hh" namespace nix::fetchers { @@ -149,7 +150,7 @@ struct PathInputScheme : InputScheme auto fp = getFingerprint(store, input); if (fp) { auto cacheKey = makeFetchToStoreCacheKey(input.getName(), *fp, method, "/"); - fetchers::getCache()->upsert(cacheKey, *store, {}, *storePath); + input.settings->getCache()->upsert(cacheKey, *store, {}, *storePath); } /* Trust the lastModified value supplied by the user, if diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index bfaf9569a4e..335935f53af 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -156,7 +156,7 @@ static std::shared_ptr getGlobalRegistry(const Settings & settings, re } if (!isAbsolute(path)) { - auto storePath = downloadFile(store, path, "flake-registry.json").storePath; + auto storePath = downloadFile(store, settings, path, "flake-registry.json").storePath; if (auto store2 = store.dynamic_pointer_cast()) store2->addPermRoot(storePath, getCacheDir() + "/flake-registry.json"); path = store->toRealPath(storePath); diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index ef91d6b2553..80f569f039c 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -9,11 +9,13 @@ #include "nix/fetchers/store-path-accessor.hh" #include "nix/store/store-api.hh" #include "nix/fetchers/git-utils.hh" +#include "nix/fetchers/fetch-settings.hh" namespace nix::fetchers { DownloadFileResult downloadFile( ref store, + const Settings & settings, const std::string & url, const std::string & name, const Headers & headers) @@ -25,7 +27,7 @@ DownloadFileResult downloadFile( {"name", name}, }}}; - auto cached = getCache()->lookupStorePath(key, *store); + auto cached = settings.getCache()->lookupStorePath(key, *store); auto useCached = [&]() -> DownloadFileResult { @@ -92,7 +94,7 @@ DownloadFileResult downloadFile( key.second.insert_or_assign("url", url); assert(!res.urls.empty()); infoAttrs.insert_or_assign("url", *res.urls.rbegin()); - getCache()->upsert(key, *store, infoAttrs, *storePath); + settings.getCache()->upsert(key, *store, infoAttrs, *storePath); } return { @@ -104,13 +106,14 @@ DownloadFileResult downloadFile( } static DownloadTarballResult downloadTarball_( + const Settings & settings, const std::string & url, const Headers & headers, const std::string & displayPrefix) { Cache::Key cacheKey{"tarball", {{"url", url}}}; - auto cached = getCache()->lookupExpired(cacheKey); + auto cached = settings.getCache()->lookupExpired(cacheKey); auto attrsToResult = [&](const Attrs & infoAttrs) { @@ -196,7 +199,7 @@ static DownloadTarballResult downloadTarball_( /* Insert a cache entry for every URL in the redirect chain. */ for (auto & url : res->urls) { cacheKey.second.insert_or_assign("url", url); - getCache()->upsert(cacheKey, infoAttrs); + settings.getCache()->upsert(cacheKey, infoAttrs); } // FIXME: add a cache entry for immutableUrl? That could allow @@ -341,7 +344,7 @@ struct FileInputScheme : CurlInputScheme the Nix store directly, since there is little deduplication benefit in using the Git cache for single big files like tarballs. */ - auto file = downloadFile(store, getStrAttr(input.attrs, "url"), input.getName()); + auto file = downloadFile(store, *input.settings, getStrAttr(input.attrs, "url"), input.getName()); auto narHash = store->queryPathInfo(file.storePath)->narHash; input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); @@ -373,6 +376,7 @@ struct TarballInputScheme : CurlInputScheme auto input(_input); auto result = downloadTarball_( + *input.settings, getStrAttr(input.attrs, "url"), {}, "«" + input.to_string() + "»"); @@ -390,7 +394,7 @@ struct TarballInputScheme : CurlInputScheme input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified)); input.attrs.insert_or_assign("narHash", - getTarballCache()->treeHashToNarHash(result.treeHash).to_string(HashFormat::SRI, true)); + getTarballCache()->treeHashToNarHash(*input.settings, result.treeHash).to_string(HashFormat::SRI, true)); return {result.accessor, input}; } diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 1e5ac588a52..59d1de58669 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -264,7 +264,7 @@ static Flake readFlake( state.symbols[setting.name], std::string(state.forceStringNoCtx(*setting.value, setting.pos, ""))); else if (setting.value->type() == nPath) { - auto storePath = fetchToStore(*state.store, setting.value->path(), FetchMode::Copy); + auto storePath = fetchToStore(state.fetchSettings, *state.store, setting.value->path(), FetchMode::Copy); flake.config.settings.emplace( state.symbols[setting.name], state.store->printStorePath(storePath)); diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 2ad88dbbea3..9ad87a76e84 100644 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -7,6 +7,7 @@ #include "nix/expr/eval-settings.hh" // for defexpr #include "nix/util/users.hh" #include "nix/fetchers/tarball.hh" +#include "nix/fetchers/fetch-settings.hh" #include "self-exe.hh" #include "man-pages.hh" @@ -114,7 +115,7 @@ static void update(const StringSet & channelNames) // We want to download the url to a file to see if it's a tarball while also checking if we // got redirected in the process, so that we can grab the various parts of a nix channel // definition from a consistent location if the redirect changes mid-download. - auto result = fetchers::downloadFile(store, url, std::string(baseNameOf(url))); + auto result = fetchers::downloadFile(store, fetchSettings, url, std::string(baseNameOf(url))); auto filename = store->toRealPath(result.storePath); url = result.effectiveUrl; @@ -128,9 +129,9 @@ static void update(const StringSet & channelNames) if (!unpacked) { // Download the channel tarball. try { - filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz").storePath); + filename = store->toRealPath(fetchers::downloadFile(store, fetchSettings, url + "/nixexprs.tar.xz", "nixexprs.tar.xz").storePath); } catch (FileTransferError & e) { - filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2").storePath); + filename = store->toRealPath(fetchers::downloadFile(store, fetchSettings, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2").storePath); } } // Regardless of where it came from, add the expression representing this channel to accumulated expression diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 4782cbb290f..fba4419651f 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1477,7 +1477,7 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON auto originalRef = getFlakeRef(); auto resolvedRef = originalRef.resolve(store); auto [accessor, lockedRef] = resolvedRef.lazyFetch(store); - auto storePath = fetchToStore(*store, accessor, FetchMode::Copy, lockedRef.input.getName()); + auto storePath = fetchToStore(getEvalState()->fetchSettings, *store, accessor, FetchMode::Copy, lockedRef.input.getName()); auto hash = store->queryPathInfo(storePath)->narHash; if (json) { From 06c44ce0cc2f06b647b7cb79c5eb8ebc63b9ea4d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 14 May 2025 15:32:08 +0200 Subject: [PATCH 14/23] builtin:fetch-tree: Hack to avoid touching the parent's FileTransfer object --- src/libfetchers/builtin.cc | 7 ++++++ src/libstore/filetransfer.cc | 23 +++++++++++-------- .../include/nix/store/filetransfer.hh | 2 ++ 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/libfetchers/builtin.cc b/src/libfetchers/builtin.cc index cdef20461c5..9f4e2ac35fa 100644 --- a/src/libfetchers/builtin.cc +++ b/src/libfetchers/builtin.cc @@ -3,6 +3,7 @@ #include "nix/fetchers/fetchers.hh" #include "nix/fetchers/fetch-settings.hh" #include "nix/util/archive.hh" +#include "nix/store/filetransfer.hh" #include @@ -27,6 +28,12 @@ static void builtinFetchTree(const BuiltinBuilderContext & ctx) fetchers::Settings myFetchSettings; myFetchSettings.accessTokens = fetchSettings.accessTokens.get(); + // Make sure we don't use the FileTransfer object of the parent + // since it's in a broken state after the fork. We also must not + // delete it, so hang on to the shared_ptr. + // FIXME: move FileTransfer into fetchers::Settings. + auto prevFileTransfer = resetFileTransfer(); + // FIXME: disable use of the git/tarball cache auto input = Input::fromAttrs(myFetchSettings, jsonToAttrs((*ctx.structuredAttrs)["input"])); diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index fb7c6c7a24a..b1d334e1a2f 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -798,24 +798,29 @@ struct curlFileTransfer : public FileTransfer } }; -ref makeCurlFileTransfer() -{ - return make_ref(); -} +static Sync> _fileTransfer; ref getFileTransfer() { - static ref fileTransfer = makeCurlFileTransfer(); + auto fileTransfer(_fileTransfer.lock()); - if (fileTransfer->state_.lock()->quit) - fileTransfer = makeCurlFileTransfer(); + if (!*fileTransfer || (*fileTransfer)->state_.lock()->quit) + *fileTransfer = std::make_shared(); - return fileTransfer; + return ref(*fileTransfer); } ref makeFileTransfer() { - return makeCurlFileTransfer(); + return make_ref(); +} + +std::shared_ptr resetFileTransfer() +{ + auto fileTransfer(_fileTransfer.lock()); + std::shared_ptr prev; + fileTransfer->swap(prev); + return prev; } std::future FileTransfer::enqueueFileTransfer(const FileTransferRequest & request) diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index f87f68e7fc8..9d4f8e7eb29 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -169,6 +169,8 @@ ref getFileTransfer(); */ ref makeFileTransfer(); +std::shared_ptr resetFileTransfer(); + class FileTransferError : public Error { public: From c75cab6807577593d4992c8b0f9d67fd3f22eada Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 14 May 2025 15:47:25 +0200 Subject: [PATCH 15/23] Move getTarballCache() into fetchers::Settings This keeps the tarball cache open across calls. --- src/libfetchers/git-utils.cc | 14 +++++++++++--- src/libfetchers/github.cc | 6 +++--- .../include/nix/fetchers/fetch-settings.hh | 10 ++++++++++ src/libfetchers/include/nix/fetchers/git-utils.hh | 2 -- src/libfetchers/tarball.cc | 8 ++++---- 5 files changed, 28 insertions(+), 12 deletions(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index d2c9e0c9b43..4553511bac4 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -1261,11 +1261,19 @@ std::vector> GitRepoImpl::getSubmodules return result; } -ref getTarballCache() +namespace fetchers { + +ref Settings::getTarballCache() const { - static auto repoDir = std::filesystem::path(getCacheDir()) / "tarball-cache"; + auto tarballCache(_tarballCache.lock()); + if (!*tarballCache) + *tarballCache = GitRepo::openRepo( + std::filesystem::path(getCacheDir()) / "tarball-cache", + true, + true); + return ref(*tarballCache); +} - return GitRepo::openRepo(repoDir, true, true); } GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path & path) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 9af620e903b..77549b84c21 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -271,7 +271,7 @@ struct GitArchiveInputScheme : InputScheme if (auto lastModifiedAttrs = cache->lookup(lastModifiedKey)) { auto treeHash = getRevAttr(*treeHashAttrs, "treeHash"); auto lastModified = getIntAttr(*lastModifiedAttrs, "lastModified"); - if (getTarballCache()->hasObject(treeHash)) + if (input.settings->getTarballCache()->hasObject(treeHash)) return {std::move(input), TarballInfo { .treeHash = treeHash, .lastModified = (time_t) lastModified }}; else debug("Git tree with hash '%s' has disappeared from the cache, refetching...", treeHash.gitRev()); @@ -291,7 +291,7 @@ struct GitArchiveInputScheme : InputScheme fmt("unpacking '%s' into the Git cache", input.to_string())); TarArchive archive { *source }; - auto tarballCache = getTarballCache(); + auto tarballCache = input.settings->getTarballCache(); auto parseSink = tarballCache->getFileSystemObjectSink(); auto lastModified = unpackTarfileToSink(archive, *parseSink); auto tree = parseSink->flush(); @@ -327,7 +327,7 @@ struct GitArchiveInputScheme : InputScheme #endif input.attrs.insert_or_assign("lastModified", uint64_t(tarballInfo.lastModified)); - auto accessor = getTarballCache()->getAccessor( + auto accessor = input.settings->getTarballCache()->getAccessor( tarballInfo.treeHash, false, "«" + input.to_string() + "»"); diff --git a/src/libfetchers/include/nix/fetchers/fetch-settings.hh b/src/libfetchers/include/nix/fetchers/fetch-settings.hh index 7b2c5720074..e1c7f70cff4 100644 --- a/src/libfetchers/include/nix/fetchers/fetch-settings.hh +++ b/src/libfetchers/include/nix/fetchers/fetch-settings.hh @@ -11,6 +11,12 @@ #include +namespace nix { + +struct GitRepo; + +} + namespace nix::fetchers { struct Cache; @@ -113,8 +119,12 @@ struct Settings : public Config ref getCache() const; + ref getTarballCache() const; + private: mutable Sync> _cache; + + mutable Sync> _tarballCache; }; } diff --git a/src/libfetchers/include/nix/fetchers/git-utils.hh b/src/libfetchers/include/nix/fetchers/git-utils.hh index 2926deb4f44..9cab72b7314 100644 --- a/src/libfetchers/include/nix/fetchers/git-utils.hh +++ b/src/libfetchers/include/nix/fetchers/git-utils.hh @@ -125,8 +125,6 @@ struct GitRepo virtual Hash dereferenceSingletonDirectory(const Hash & oid) = 0; }; -ref getTarballCache(); - // A helper to ensure that the `git_*_free` functions get called. template struct Deleter diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 80f569f039c..96b5318821d 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -122,11 +122,11 @@ static DownloadTarballResult downloadTarball_( .treeHash = treeHash, .lastModified = (time_t) getIntAttr(infoAttrs, "lastModified"), .immutableUrl = maybeGetStrAttr(infoAttrs, "immutableUrl"), - .accessor = getTarballCache()->getAccessor(treeHash, false, displayPrefix), + .accessor = settings.getTarballCache()->getAccessor(treeHash, false, displayPrefix), }; }; - if (cached && !getTarballCache()->hasObject(getRevAttr(cached->value, "treeHash"))) + if (cached && !settings.getTarballCache()->hasObject(getRevAttr(cached->value, "treeHash"))) cached.reset(); if (cached && !cached->expired) @@ -172,7 +172,7 @@ static DownloadTarballResult downloadTarball_( TarArchive{path}; }) : TarArchive{*source}; - auto tarballCache = getTarballCache(); + auto tarballCache = settings.getTarballCache(); auto parseSink = tarballCache->getFileSystemObjectSink(); auto lastModified = unpackTarfileToSink(archive, *parseSink); auto tree = parseSink->flush(); @@ -394,7 +394,7 @@ struct TarballInputScheme : CurlInputScheme input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified)); input.attrs.insert_or_assign("narHash", - getTarballCache()->treeHashToNarHash(*input.settings, result.treeHash).to_string(HashFormat::SRI, true)); + input.settings->getTarballCache()->treeHashToNarHash(*input.settings, result.treeHash).to_string(HashFormat::SRI, true)); return {result.accessor, input}; } From 16bd9a8bd291b038172805cfd04309a55e6e4ee4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 14 May 2025 15:50:39 +0200 Subject: [PATCH 16/23] Formatting --- src/libexpr/paths.cc | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index dab31c663b4..d782b7586ce 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -24,7 +24,11 @@ StorePath EvalState::devirtualize(const StorePath & path, StringMap * rewrites) { if (auto mount = storeFS->getMount(CanonPath(store->printStorePath(path)))) { auto storePath = fetchToStore( - fetchSettings, *store, SourcePath{ref(mount)}, settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, path.name()); + fetchSettings, + *store, + SourcePath{ref(mount)}, + settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, + path.name()); assert(storePath.name() == path.name()); if (rewrites) rewrites->emplace(path.hashPart(), storePath.hashPart()); @@ -61,18 +65,23 @@ std::string EvalState::computeBaseName(const SourcePath & path) "This can typically be avoided by rewriting an attribute like `src = ./.` " "to `src = builtins.path { path = ./.; name = \"source\"; }`.", path); - return std::string(fetchToStore(fetchSettings, *store, path, FetchMode::DryRun, storePath->name()).to_string()); + return std::string( + fetchToStore(fetchSettings, *store, path, FetchMode::DryRun, storePath->name()).to_string()); } } return std::string(path.baseName()); } StorePath EvalState::mountInput( - fetchers::Input & input, const fetchers::Input & originalInput, ref accessor, bool requireLockable, bool forceNarHash) + fetchers::Input & input, + const fetchers::Input & originalInput, + ref accessor, + bool requireLockable, + bool forceNarHash) { auto storePath = settings.lazyTrees - ? StorePath::random(input.getName()) - : fetchToStore(fetchSettings, *store, accessor, FetchMode::Copy, input.getName()); + ? StorePath::random(input.getName()) + : fetchToStore(fetchSettings, *store, accessor, FetchMode::Copy, input.getName()); allowPath(storePath); // FIXME: should just whitelist the entire virtual store From 3df518b12ab3cf47ebb214b568f2322e09f64c07 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 14 May 2025 17:23:58 +0200 Subject: [PATCH 17/23] Add test --- .../flakes/build-time-flake-inputs.sh | 60 +++++++++++++++++++ tests/functional/flakes/meson.build | 1 + 2 files changed, 61 insertions(+) create mode 100644 tests/functional/flakes/build-time-flake-inputs.sh diff --git a/tests/functional/flakes/build-time-flake-inputs.sh b/tests/functional/flakes/build-time-flake-inputs.sh new file mode 100644 index 00000000000..fd28c1d7818 --- /dev/null +++ b/tests/functional/flakes/build-time-flake-inputs.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +source ./common.sh + +requireGit + +lazy="$TEST_ROOT/lazy" +createGitRepo "$lazy" +echo world > "$lazy/who" +git -C "$lazy" add who +git -C "$lazy" commit -a -m foo + +repo="$TEST_ROOT/repo" + +createGitRepo "$repo" + +cat > "$repo/flake.nix" < "$lazy/who" +git -C "$lazy" commit -a -m foo + +nix flake update --flake "$repo" + +clearStore + +nix build --out-link "$TEST_ROOT/result" -L "$repo" +[[ $(cat "$TEST_ROOT/result") = utrecht ]] + +rm -rf "$lazy" + +clearStore + +expectStderr 1 nix build --out-link "$TEST_ROOT/result" -L "$repo" | grepQuiet "Cannot build.*source.drv" diff --git a/tests/functional/flakes/meson.build b/tests/functional/flakes/meson.build index 213c388a6d9..531d2ce79f7 100644 --- a/tests/functional/flakes/meson.build +++ b/tests/functional/flakes/meson.build @@ -33,6 +33,7 @@ suites += { 'debugger.sh', 'source-paths.sh', 'old-lockfiles.sh', + 'build-time-flake-inputs.sh', ], 'workdir': meson.current_source_dir(), } From 943aaa44718607e06b9f5b31d0e80a7c901916eb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 19 Aug 2025 16:05:16 +0200 Subject: [PATCH 18/23] Fix test --- tests/functional/flakes/build-time-flake-inputs.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/functional/flakes/build-time-flake-inputs.sh b/tests/functional/flakes/build-time-flake-inputs.sh index fd28c1d7818..82f30a4fc36 100644 --- a/tests/functional/flakes/build-time-flake-inputs.sh +++ b/tests/functional/flakes/build-time-flake-inputs.sh @@ -3,6 +3,7 @@ source ./common.sh requireGit +TODO_NixOS lazy="$TEST_ROOT/lazy" createGitRepo "$lazy" From 0159911d8d0565e169b2a21f917b2fceb73de55f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 19 Aug 2025 16:46:02 +0200 Subject: [PATCH 19/23] Add build-time-fetch-tree experimental feature --- src/libfetchers/builtin.cc | 2 ++ src/libflake/flake.cc | 2 ++ src/libutil/experimental-features.cc | 12 ++++++++++-- .../include/nix/util/experimental-features.hh | 1 + tests/functional/flakes/build-time-flake-inputs.sh | 10 +++++----- 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/libfetchers/builtin.cc b/src/libfetchers/builtin.cc index a04c400236f..9087e043500 100644 --- a/src/libfetchers/builtin.cc +++ b/src/libfetchers/builtin.cc @@ -12,6 +12,8 @@ namespace nix { static void builtinFetchTree(const BuiltinBuilderContext & ctx) { + experimentalFeatureSettings.require(Xp::BuildTimeFetchTree); + auto out = get(ctx.drv.outputs, "out"); if (!out) throw Error("'builtin:fetch-tree' requires an 'out' output"); diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 240c9eaa819..17eb2265969 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -133,6 +133,8 @@ static FlakeInput parseFlakeInput( } else if (attr.name == sBuildTime) { expectType(state, nBool, *attr.value, attr.pos); input.buildTime = attr.value->boolean(); + if (input.buildTime) + experimentalFeatureSettings.require(Xp::BuildTimeFetchTree); } else if (attr.name == sInputs) { input.overrides = parseFlakeInputs(state, attr.value, attr.pos, lockRootAttrPath, flakeDir, false).first; diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 56aa5880876..8de626b48cb 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -16,7 +16,7 @@ struct ExperimentalFeatureDetails /** * If two different PRs both add an experimental feature, and we just - * used a number for this, we *woudln't* get merge conflict and the + * used a number for this, we *wouldn't* get merge conflict and the * counter will be incremented once instead of twice, causing a build * failure. * @@ -24,7 +24,7 @@ struct ExperimentalFeatureDetails * feature, we either have no issue at all if few features are not added * at the end of the list, or a proper merge conflict if they are. */ -constexpr size_t numXpFeatures = 1 + static_cast(Xp::BLAKE3Hashes); +constexpr size_t numXpFeatures = 1 + static_cast(Xp::BuildTimeFetchTree); constexpr std::array xpFeatureDetails = {{ { @@ -304,6 +304,14 @@ constexpr std::array xpFeatureDetails )", .trackingUrl = "", }, + { + .tag = Xp::BuildTimeFetchTree, + .name = "build-time-fetch-tree", + .description = R"( + Enable the built-in derivation `builtin:fetch-tree`, as well as the flake input attribute `buildTime`. + )", + .trackingUrl = "", + }, }}; static_assert( diff --git a/src/libutil/include/nix/util/experimental-features.hh b/src/libutil/include/nix/util/experimental-features.hh index 2845f1e9b18..e44d4a2005d 100644 --- a/src/libutil/include/nix/util/experimental-features.hh +++ b/src/libutil/include/nix/util/experimental-features.hh @@ -36,6 +36,7 @@ enum struct ExperimentalFeature { PipeOperators, ExternalBuilders, BLAKE3Hashes, + BuildTimeFetchTree, }; extern std::set stabilizedFeatures; diff --git a/tests/functional/flakes/build-time-flake-inputs.sh b/tests/functional/flakes/build-time-flake-inputs.sh index 82f30a4fc36..467d59008bb 100644 --- a/tests/functional/flakes/build-time-flake-inputs.sh +++ b/tests/functional/flakes/build-time-flake-inputs.sh @@ -35,27 +35,27 @@ EOF cp "${config_nix}" "$repo/" git -C "$repo" add flake.nix config.nix -nix flake lock "$repo" +nix flake lock --extra-experimental-features build-time-fetch-tree "$repo" git -C "$repo" add flake.lock git -C "$repo" commit -a -m foo clearStore -nix build --out-link "$TEST_ROOT/result" -L "$repo" +nix build --extra-experimental-features build-time-fetch-tree --out-link "$TEST_ROOT/result" -L "$repo" [[ $(cat "$TEST_ROOT/result") = world ]] echo utrecht > "$lazy/who" git -C "$lazy" commit -a -m foo -nix flake update --flake "$repo" +nix flake update --extra-experimental-features build-time-fetch-tree --flake "$repo" clearStore -nix build --out-link "$TEST_ROOT/result" -L "$repo" +nix build --extra-experimental-features build-time-fetch-tree --out-link "$TEST_ROOT/result" -L "$repo" [[ $(cat "$TEST_ROOT/result") = utrecht ]] rm -rf "$lazy" clearStore -expectStderr 1 nix build --out-link "$TEST_ROOT/result" -L "$repo" | grepQuiet "Cannot build.*source.drv" +expectStderr 1 nix build --extra-experimental-features build-time-fetch-tree --out-link "$TEST_ROOT/result" -L "$repo" | grepQuiet "Cannot build.*source.drv" From 7e50ba7eeb50989e5ea5066f39e2eff9d466dbeb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 20 Aug 2025 14:42:37 +0200 Subject: [PATCH 20/23] Provide downloadFile() with a writable store --- src/libfetchers/builtin.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libfetchers/builtin.cc b/src/libfetchers/builtin.cc index 9087e043500..5f3c1f7e8af 100644 --- a/src/libfetchers/builtin.cc +++ b/src/libfetchers/builtin.cc @@ -43,11 +43,12 @@ static void builtinFetchTree(const BuiltinBuilderContext & ctx) std::cerr << fmt("fetching '%s'...\n", input.to_string()); - /* Make sure we don't use the real store because we're in a forked - process. */ - auto dummyStore = openStore("dummy://"); + /* Functions like downloadFile() expect a store. We can't use the + real one since we're in a forked process. FIXME: use recursive + Nix's daemon so we can use the real store? */ + auto tmpStore = openStore(ctx.tmpDirInSandbox + "/nix"); - auto [accessor, lockedInput] = input.getAccessor(dummyStore); + auto [accessor, lockedInput] = input.getAccessor(tmpStore); auto source = sinkToSource([&](Sink & sink) { accessor->dumpPath(CanonPath::root, sink); }); From 6f272c58e442d1b2113574359b2227bfd4185790 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 20 Aug 2025 14:45:23 +0200 Subject: [PATCH 21/23] Fix segfault destroying prevFileTransfer --- src/libfetchers/builtin.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/builtin.cc b/src/libfetchers/builtin.cc index 5f3c1f7e8af..c9210e9219b 100644 --- a/src/libfetchers/builtin.cc +++ b/src/libfetchers/builtin.cc @@ -35,7 +35,7 @@ static void builtinFetchTree(const BuiltinBuilderContext & ctx) // since it's in a broken state after the fork. We also must not // delete it, so hang on to the shared_ptr. // FIXME: move FileTransfer into fetchers::Settings. - auto prevFileTransfer = resetFileTransfer(); + static auto prevFileTransfer = resetFileTransfer(); // FIXME: disable use of the git/tarball cache From 486c48a26ae0d9bbd27f6d346c14164f1c6859a9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 20 Aug 2025 15:24:23 +0200 Subject: [PATCH 22/23] Add tests for build-time fetching of GitHub flakes --- tests/nixos/github-flakes.nix | 39 +++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/nixos/github-flakes.nix b/tests/nixos/github-flakes.nix index 67e2d41e906..6ea797cc623 100644 --- a/tests/nixos/github-flakes.nix +++ b/tests/nixos/github-flakes.nix @@ -203,6 +203,27 @@ in assert info["revision"] == "${nixpkgs.rev}", f"revision mismatch: {info['revision']} != ${nixpkgs.rev}" cat_log() + out = client.succeed("nix flake prefetch nixpkgs --json") + nar_hash = json.loads(out)['hash'] + + # Test build-time fetching of public flakes. + expr = f""" + derivation {{ + name = "source"; + builder = "builtin:fetch-tree"; + system = "builtin"; + __structuredAttrs = true; + input = {{ + type = "github"; + owner = "NixOS"; + repo = "nixpkgs"; + }}; + outputHashMode = "recursive"; + outputHash = "{nar_hash}"; + }} + """ + client.succeed(f"nix build --store /run/store --extra-experimental-features build-time-fetch-tree -L --expr '{expr}'") + # ... otherwise it should use the API out = client.succeed("nix flake metadata private-flake --json --access-tokens github.com=ghp_000000000000000000000000000000000000 --tarball-ttl 0 --no-trust-tarballs-from-git-forges") print(out) @@ -211,6 +232,24 @@ in assert info["fingerprint"] cat_log() + # Test build-time fetching of private flakes. + expr = f""" + derivation {{ + name = "source"; + builder = "builtin:fetch-tree"; + system = "builtin"; + __structuredAttrs = true; + input = {{ + type = "github"; + owner = "fancy-enterprise"; + repo = "private-flake"; + }}; + outputHashMode = "recursive"; + outputHash = "{info['locked']['narHash']}"; + }} + """ + client.succeed(f"nix build --store /run/store --extra-experimental-features build-time-fetch-tree --access-tokens github.com=ghp_000000000000000000000000000000000000 -L --expr '{expr}'") + # Fetching with the resolved URL should produce the same result. info2 = json.loads(client.succeed(f"nix flake metadata {info['url']} --json --access-tokens github.com=ghp_000000000000000000000000000000000000 --tarball-ttl 0")) print(info["fingerprint"], info2["fingerprint"]) From 1201c720dc28fb66ddacf773f8231567d79c0789 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 20 Aug 2025 16:37:28 +0200 Subject: [PATCH 23/23] GitRepo::fetch(): Fall back to using libgit2 for fetching In the builtin:fetch-tree sandbox, we don't have the `git` executable available, so let's use libgit2 instead. This generally won't work very well for SSH, but that currently doesn't work anyway because the sandbox doesn't have access to SSH keys. --- src/libfetchers/git-utils.cc | 54 +++++++++++++------ .../build-time-fetch-tree/default.nix | 49 +++++++++++++++++ 2 files changed, 88 insertions(+), 15 deletions(-) create mode 100644 tests/nixos/fetch-git/test-cases/build-time-fetch-tree/default.nix diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 4273991455a..446e94c0df8 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -10,6 +10,7 @@ #include "nix/util/sync.hh" #include "nix/util/thread-pool.hh" #include "nix/util/pool.hh" +#include "nix/util/executable-path.hh" #include #include @@ -549,21 +550,44 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this // that) // then use code that was removed in this commit (see blame) - auto dir = this->path; - Strings gitArgs{"-C", dir.string(), "--git-dir", ".", "fetch", "--quiet", "--force"}; - if (shallow) - append(gitArgs, {"--depth", "1"}); - append(gitArgs, {std::string("--"), url, refspec}); - - runProgram( - RunOptions{ - .program = "git", - .lookupPath = true, - // FIXME: git stderr messes up our progress indicator, so - // we're using --quiet for now. Should process its stderr. - .args = gitArgs, - .input = {}, - .isInteractive = true}); + if (ExecutablePath::load().findName("git")) { + auto dir = this->path; + Strings gitArgs{"-C", dir.string(), "--git-dir", ".", "fetch", "--quiet", "--force"}; + if (shallow) + append(gitArgs, {"--depth", "1"}); + append(gitArgs, {std::string("--"), url, refspec}); + + runProgram( + RunOptions{ + .program = "git", + .lookupPath = true, + // FIXME: git stderr messes up our progress indicator, so + // we're using --quiet for now. Should process its stderr. + .args = gitArgs, + .input = {}, + .isInteractive = true}); + } else { + // Fall back to using libgit2 for fetching. This does not + // support SSH very well. + Remote remote; + + if (git_remote_create_anonymous(Setter(remote), *this, url.c_str())) + throw Error("cannot create Git remote '%s': %s", url, git_error_last()->message); + + char * refspecs[] = {(char *) refspec.c_str()}; + git_strarray refspecs2{.strings = refspecs, .count = 1}; + + git_fetch_options opts = GIT_FETCH_OPTIONS_INIT; + // FIXME: for some reason, shallow fetching over ssh barfs + // with "could not read from remote repository". + opts.depth = shallow && parseURL(url).scheme != "ssh" ? 1 : GIT_FETCH_DEPTH_FULL; + opts.callbacks.payload = &act; + opts.callbacks.sideband_progress = sidebandProgressCallback; + opts.callbacks.transfer_progress = transferProgressCallback; + + if (git_remote_fetch(remote.get(), &refspecs2, &opts, nullptr)) + throw Error("fetching '%s' from '%s': %s", refspec, url, git_error_last()->message); + } } void verifyCommit(const Hash & rev, const std::vector & publicKeys) override diff --git a/tests/nixos/fetch-git/test-cases/build-time-fetch-tree/default.nix b/tests/nixos/fetch-git/test-cases/build-time-fetch-tree/default.nix new file mode 100644 index 00000000000..a241c877d21 --- /dev/null +++ b/tests/nixos/fetch-git/test-cases/build-time-fetch-tree/default.nix @@ -0,0 +1,49 @@ +{ config, ... }: +{ + description = "build-time fetching"; + script = '' + import json + + # add a file to the repo + client.succeed(f""" + echo ${config.name # to make the git tree and store path unique + } > {repo.path}/test-case \ + && echo chiang-mai > {repo.path}/thailand \ + && {repo.git} add test-case thailand \ + && {repo.git} commit -m 'commit1' \ + && {repo.git} push origin main + """) + + # get the NAR hash + nar_hash = json.loads(client.succeed(f""" + nix flake prefetch --flake-registry "" git+{repo.remote} --json + """))['hash'] + + # construct the derivation + expr = f""" + derivation {{ + name = "source"; + builder = "builtin:fetch-tree"; + system = "builtin"; + __structuredAttrs = true; + input = {{ + type = "git"; + url = "{repo.remote}"; + ref = "main"; + }}; + outputHashMode = "recursive"; + outputHash = "{nar_hash}"; + }} + """ + + # do the build-time fetch + out_path = client.succeed(f""" + nix build --print-out-paths --store /run/store --flake-registry "" --extra-experimental-features build-time-fetch-tree --expr '{expr}' + """).strip() + + # check if the committed file is there + client.succeed(f""" + test -f /run/store/{out_path}/thailand + """) + ''; +}