diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 9a4045029ec..ff0eef12031 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -18,10 +18,6 @@ namespace nix { -fetchers::Settings fetchSettings; - -static GlobalConfig::Register rFetchSettings(&fetchSettings); - EvalSettings evalSettings{ settings.readOnlyMode, { diff --git a/src/libcmd/include/nix/cmd/common-eval-args.hh b/src/libcmd/include/nix/cmd/common-eval-args.hh index 2a04994775f..e430ecc84b1 100644 --- a/src/libcmd/include/nix/cmd/common-eval-args.hh +++ b/src/libcmd/include/nix/cmd/common-eval-args.hh @@ -25,9 +25,6 @@ namespace flake { struct Settings; } -/** - * @todo Get rid of global settings variables - */ extern fetchers::Settings fetchSettings; /** diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index 5ef629b3fb9..d5911ac2750 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -484,7 +484,8 @@ public: fetchers::Input & input, const fetchers::Input & originalInput, ref accessor, - bool requireLockable); + bool requireLockable, + bool forceNarHash = false); /** * Parse a Nix expression from the specified file. diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index e31aff9ca48..d5dd9518ae2 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -73,7 +73,11 @@ std::string EvalState::computeBaseName(const SourcePath & path, PosIdx pos) } StorePath EvalState::mountInput( - fetchers::Input & input, const fetchers::Input & originalInput, ref accessor, bool requireLockable) + fetchers::Input & input, + const fetchers::Input & originalInput, + ref accessor, + bool requireLockable, + bool forceNarHash) { auto storePath = settings.lazyTrees ? StorePath::random(input.getName()) @@ -95,7 +99,9 @@ StorePath EvalState::mountInput( storeFS->mount(CanonPath(store->printStorePath(storePath)), accessor); - if (requireLockable && (!settings.lazyTrees || !settings.lazyLocks || !input.isLocked()) && !input.getNarHash()) + if (forceNarHash + || (requireLockable && (!settings.lazyTrees || !settings.lazyLocks || !input.isLocked()) + && !input.getNarHash())) input.attrs.insert_or_assign("narHash", getNarHash()->to_string(HashFormat::SRI, true)); if (originalInput.getNarHash() && *getNarHash() != *originalInput.getNarHash()) diff --git a/src/libfetchers/builtin.cc b/src/libfetchers/builtin.cc new file mode 100644 index 00000000000..c9210e9219b --- /dev/null +++ b/src/libfetchers/builtin.cc @@ -0,0 +1,60 @@ +#include "nix/store/builtins.hh" +#include "nix/store/parsed-derivations.hh" +#include "nix/fetchers/fetchers.hh" +#include "nix/fetchers/fetch-settings.hh" +#include "nix/util/archive.hh" +#include "nix/store/filetransfer.hh" +#include "nix/store/store-open.hh" + +#include + +namespace nix { + +static void builtinFetchTree(const BuiltinBuilderContext & ctx) +{ + experimentalFeatureSettings.require(Xp::BuildTimeFetchTree); + + auto out = get(ctx.drv.outputs, "out"); + if (!out) + throw Error("'builtin:fetch-tree' requires an 'out' output"); + + if (!(ctx.drv.type().isFixed() || ctx.drv.type().isImpure())) + throw Error("'builtin:fetch-tree' must be a fixed-output or impure derivation"); + + if (!ctx.parsedDrv) + throw Error("'builtin:fetch-tree' must have '__structuredAttrs = true'"); + + setenv("NIX_CACHE_HOME", ctx.tmpDirInSandbox.c_str(), 1); + + using namespace fetchers; + + fetchers::Settings myFetchSettings; + myFetchSettings.accessTokens = fetchSettings.accessTokens.get(); + + // Make sure we don't use the FileTransfer object of the parent + // since it's in a broken state after the fork. We also must not + // delete it, so hang on to the shared_ptr. + // FIXME: move FileTransfer into fetchers::Settings. + static auto prevFileTransfer = resetFileTransfer(); + + // FIXME: disable use of the git/tarball cache + + auto input = Input::fromAttrs(myFetchSettings, jsonToAttrs(ctx.parsedDrv->structuredAttrs["input"])); + + std::cerr << fmt("fetching '%s'...\n", input.to_string()); + + /* Functions like downloadFile() expect a store. We can't use the + real one since we're in a forked process. FIXME: use recursive + Nix's daemon so we can use the real store? */ + auto tmpStore = openStore(ctx.tmpDirInSandbox + "/nix"); + + auto [accessor, lockedInput] = input.getAccessor(tmpStore); + + auto source = sinkToSource([&](Sink & sink) { accessor->dumpPath(CanonPath::root, sink); }); + + restorePath(ctx.outputs.at("out"), *source); +} + +static RegisterBuiltinBuilder registerUnpackChannel("fetch-tree", builtinFetchTree); + +} // namespace nix diff --git a/src/libfetchers/fetch-settings.cc b/src/libfetchers/fetch-settings.cc index f92b94a0b3b..f50177f094e 100644 --- a/src/libfetchers/fetch-settings.cc +++ b/src/libfetchers/fetch-settings.cc @@ -1,7 +1,16 @@ #include "nix/fetchers/fetch-settings.hh" +#include "nix/util/config-global.hh" namespace nix::fetchers { Settings::Settings() {} } // namespace nix::fetchers + +namespace nix { + +fetchers::Settings fetchSettings; + +static GlobalConfig::Register rFetchSettings(&fetchSettings); + +} // namespace nix diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 5254fa9177b..446e94c0df8 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -10,6 +10,7 @@ #include "nix/util/sync.hh" #include "nix/util/thread-pool.hh" #include "nix/util/pool.hh" +#include "nix/util/executable-path.hh" #include #include @@ -549,21 +550,44 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this // that) // then use code that was removed in this commit (see blame) - auto dir = this->path; - Strings gitArgs{"-C", dir.string(), "--git-dir", ".", "fetch", "--quiet", "--force"}; - if (shallow) - append(gitArgs, {"--depth", "1"}); - append(gitArgs, {std::string("--"), url, refspec}); - - runProgram( - RunOptions{ - .program = "git", - .lookupPath = true, - // FIXME: git stderr messes up our progress indicator, so - // we're using --quiet for now. Should process its stderr. - .args = gitArgs, - .input = {}, - .isInteractive = true}); + if (ExecutablePath::load().findName("git")) { + auto dir = this->path; + Strings gitArgs{"-C", dir.string(), "--git-dir", ".", "fetch", "--quiet", "--force"}; + if (shallow) + append(gitArgs, {"--depth", "1"}); + append(gitArgs, {std::string("--"), url, refspec}); + + runProgram( + RunOptions{ + .program = "git", + .lookupPath = true, + // FIXME: git stderr messes up our progress indicator, so + // we're using --quiet for now. Should process its stderr. + .args = gitArgs, + .input = {}, + .isInteractive = true}); + } else { + // Fall back to using libgit2 for fetching. This does not + // support SSH very well. + Remote remote; + + if (git_remote_create_anonymous(Setter(remote), *this, url.c_str())) + throw Error("cannot create Git remote '%s': %s", url, git_error_last()->message); + + char * refspecs[] = {(char *) refspec.c_str()}; + git_strarray refspecs2{.strings = refspecs, .count = 1}; + + git_fetch_options opts = GIT_FETCH_OPTIONS_INIT; + // FIXME: for some reason, shallow fetching over ssh barfs + // with "could not read from remote repository". + opts.depth = shallow && parseURL(url).scheme != "ssh" ? 1 : GIT_FETCH_DEPTH_FULL; + opts.callbacks.payload = &act; + opts.callbacks.sideband_progress = sidebandProgressCallback; + opts.callbacks.transfer_progress = transferProgressCallback; + + if (git_remote_fetch(remote.get(), &refspecs2, &opts, nullptr)) + throw Error("fetching '%s' from '%s': %s", refspec, url, git_error_last()->message); + } } void verifyCommit(const Hash & rev, const std::vector & publicKeys) override @@ -1312,13 +1336,18 @@ std::vector> GitRepoImpl::getSubmodules return result; } -ref getTarballCache() -{ - static auto repoDir = std::filesystem::path(getCacheDir()) / "tarball-cache"; +namespace fetchers { - return GitRepo::openRepo(repoDir, true, true); +ref Settings::getTarballCache() const +{ + auto tarballCache(_tarballCache.lock()); + if (!*tarballCache) + *tarballCache = GitRepo::openRepo(std::filesystem::path(getCacheDir()) / "tarball-cache", true, true); + return ref(*tarballCache); } +} // namespace fetchers + GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path & path) { static Sync> _cache; diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 07715cd82d1..969469c8ac6 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -269,7 +269,7 @@ struct GitArchiveInputScheme : InputScheme if (auto lastModifiedAttrs = cache->lookup(lastModifiedKey)) { auto treeHash = getRevAttr(*treeHashAttrs, "treeHash"); auto lastModified = getIntAttr(*lastModifiedAttrs, "lastModified"); - if (getTarballCache()->hasObject(treeHash)) + if (input.settings->getTarballCache()->hasObject(treeHash)) return {std::move(input), TarballInfo{.treeHash = treeHash, .lastModified = (time_t) lastModified}}; else debug("Git tree with hash '%s' has disappeared from the cache, refetching...", treeHash.gitRev()); @@ -289,7 +289,7 @@ struct GitArchiveInputScheme : InputScheme *logger, lvlInfo, actUnknown, fmt("unpacking '%s' into the Git cache", input.to_string())); TarArchive archive{*source}; - auto tarballCache = getTarballCache(); + auto tarballCache = input.settings->getTarballCache(); auto parseSink = tarballCache->getFileSystemObjectSink(); auto lastModified = unpackTarfileToSink(archive, *parseSink); auto tree = parseSink->flush(); @@ -323,7 +323,8 @@ struct GitArchiveInputScheme : InputScheme #endif input.attrs.insert_or_assign("lastModified", uint64_t(tarballInfo.lastModified)); - auto accessor = getTarballCache()->getAccessor(tarballInfo.treeHash, false, "«" + input.to_string() + "»"); + auto accessor = + input.settings->getTarballCache()->getAccessor(tarballInfo.treeHash, false, "«" + input.to_string() + "»"); if (!input.settings->trustTarballsFromGitForges) // FIXME: computing the NAR hash here is wasteful if diff --git a/src/libfetchers/include/nix/fetchers/fetch-settings.hh b/src/libfetchers/include/nix/fetchers/fetch-settings.hh index 1746aa9402a..6c39bb2ea92 100644 --- a/src/libfetchers/include/nix/fetchers/fetch-settings.hh +++ b/src/libfetchers/include/nix/fetchers/fetch-settings.hh @@ -11,6 +11,12 @@ #include +namespace nix { + +struct GitRepo; + +} + namespace nix::fetchers { struct Cache; @@ -119,8 +125,21 @@ struct Settings : public Config ref getCache() const; + ref getTarballCache() const; + private: mutable Sync> _cache; + + mutable Sync> _tarballCache; }; } // namespace nix::fetchers + +namespace nix { + +/** + * @todo Get rid of global setttings variables + */ +extern fetchers::Settings fetchSettings; + +} // namespace nix diff --git a/src/libfetchers/include/nix/fetchers/git-utils.hh b/src/libfetchers/include/nix/fetchers/git-utils.hh index 2ea2acd02e9..42c32313cc5 100644 --- a/src/libfetchers/include/nix/fetchers/git-utils.hh +++ b/src/libfetchers/include/nix/fetchers/git-utils.hh @@ -120,8 +120,6 @@ struct GitRepo virtual Hash dereferenceSingletonDirectory(const Hash & oid) = 0; }; -ref getTarballCache(); - // A helper to ensure that the `git_*_free` functions get called. template struct Deleter diff --git a/src/libfetchers/meson.build b/src/libfetchers/meson.build index 321146ca4ed..cacb2e4a03f 100644 --- a/src/libfetchers/meson.build +++ b/src/libfetchers/meson.build @@ -34,6 +34,7 @@ subdir('nix-meson-build-support/common') sources = files( 'attrs.cc', + 'builtin.cc', 'cache.cc', 'fetch-settings.cc', 'fetch-to-store.cc', diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 4f2c70c126a..9ce79114204 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -119,11 +119,11 @@ static DownloadTarballResult downloadTarball_( .treeHash = treeHash, .lastModified = (time_t) getIntAttr(infoAttrs, "lastModified"), .immutableUrl = maybeGetStrAttr(infoAttrs, "immutableUrl"), - .accessor = getTarballCache()->getAccessor(treeHash, false, displayPrefix), + .accessor = settings.getTarballCache()->getAccessor(treeHash, false, displayPrefix), }; }; - if (cached && !getTarballCache()->hasObject(getRevAttr(cached->value, "treeHash"))) + if (cached && !settings.getTarballCache()->hasObject(getRevAttr(cached->value, "treeHash"))) cached.reset(); if (cached && !cached->expired) @@ -162,7 +162,7 @@ static DownloadTarballResult downloadTarball_( TarArchive{path}; }) : TarArchive{*source}; - auto tarballCache = getTarballCache(); + auto tarballCache = settings.getTarballCache(); auto parseSink = tarballCache->getFileSystemObjectSink(); auto lastModified = unpackTarfileToSink(archive, *parseSink); auto tree = parseSink->flush(); @@ -378,7 +378,9 @@ struct TarballInputScheme : CurlInputScheme input.attrs.insert_or_assign( "narHash", - getTarballCache()->treeHashToNarHash(*input.settings, result.treeHash).to_string(HashFormat::SRI, true)); + input.settings->getTarballCache() + ->treeHashToNarHash(*input.settings, result.treeHash) + .to_string(HashFormat::SRI, true)); return {result.accessor, input}; } diff --git a/src/libflake/call-flake.nix b/src/libflake/call-flake.nix index ed7947e0601..6db35269705 100644 --- a/src/libflake/call-flake.nix +++ b/src/libflake/call-flake.nix @@ -45,7 +45,17 @@ let parentNode = allNodes.${getInputByPath lockFile.root node.parent}; sourceInfo = - if hasOverride then + if node.buildTime or false then + derivation { + name = "source"; + builder = "builtin:fetch-tree"; + system = "builtin"; + __structuredAttrs = true; + input = node.locked; + outputHashMode = "recursive"; + outputHash = node.locked.narHash; + } + else if hasOverride then overrides.${key}.sourceInfo else if isRelative then parentNode.sourceInfo @@ -93,6 +103,7 @@ let result = if node.flake or true then assert builtins.isFunction flake.outputs; + assert !(node.buildTime or false); result else sourceInfo // { inherit sourceInfo outPath; }; diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index fcd804adb4f..17eb2265969 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -101,6 +101,7 @@ static FlakeInput parseFlakeInput( auto sUrl = state.symbols.create("url"); auto sFlake = state.symbols.create("flake"); auto sFollows = state.symbols.create("follows"); + auto sBuildTime = state.symbols.create("buildTime"); fetchers::Attrs attrs; std::optional url; @@ -129,6 +130,11 @@ static FlakeInput parseFlakeInput( } else if (attr.name == sFlake) { expectType(state, nBool, *attr.value, attr.pos); input.isFlake = attr.value->boolean(); + } else if (attr.name == sBuildTime) { + expectType(state, nBool, *attr.value, attr.pos); + input.buildTime = attr.value->boolean(); + if (input.buildTime) + experimentalFeatureSettings.require(Xp::BuildTimeFetchTree); } else if (attr.name == sInputs) { input.overrides = parseFlakeInputs(state, attr.value, attr.pos, lockRootAttrPath, flakeDir, false).first; @@ -581,7 +587,11 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, didn't change and there is no override from a higher level flake. */ auto childNode = make_ref( - oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake, oldLock->parentInputAttrPath); + oldLock->lockedRef, + oldLock->originalRef, + oldLock->isFlake, + oldLock->buildTime, + oldLock->parentInputAttrPath); node->inputs.insert_or_assign(id, childNode); @@ -695,8 +705,8 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, auto inputFlake = getInputFlake( *input.ref, inputIsOverride ? fetchers::UseRegistries::All : useRegistriesInputs); - auto childNode = - make_ref(inputFlake.lockedRef, ref, true, overriddenParentPath); + auto childNode = make_ref( + inputFlake.lockedRef, ref, true, input.buildTime, overriddenParentPath); node->inputs.insert_or_assign(id, childNode); @@ -739,12 +749,13 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, return { state.storePath(state.mountInput( - lockedRef.input, input.ref->input, cachedInput.accessor, true)), + lockedRef.input, input.ref->input, cachedInput.accessor, true, true)), lockedRef}; } }(); - auto childNode = make_ref(lockedRef, ref, false, overriddenParentPath); + auto childNode = + make_ref(lockedRef, ref, false, input.buildTime, overriddenParentPath); nodePaths.emplace(childNode, path); diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index e647dcda2d9..3c8acb2b72d 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -43,12 +43,18 @@ typedef std::map FlakeInputs; struct FlakeInput { std::optional ref; + /** - * true = process flake to get outputs - * - * false = (fetched) static source path + * Whether to call the `flake.nix` file in this input to get its outputs. */ bool isFlake = true; + + /** + * Whether to fetch this input at evaluation time or at build + * time. + */ + bool buildTime = false; + std::optional follows; FlakeInputs overrides; }; diff --git a/src/libflake/include/nix/flake/lockfile.hh b/src/libflake/include/nix/flake/lockfile.hh index c5740a2f114..1ca7cc3dd30 100644 --- a/src/libflake/include/nix/flake/lockfile.hh +++ b/src/libflake/include/nix/flake/lockfile.hh @@ -37,6 +37,7 @@ struct LockedNode : Node { FlakeRef lockedRef, originalRef; bool isFlake = true; + bool buildTime = false; /* The node relative to which relative source paths (e.g. 'path:../foo') are interpreted. */ @@ -46,10 +47,12 @@ struct LockedNode : Node const FlakeRef & lockedRef, const FlakeRef & originalRef, bool isFlake = true, + bool buildTime = false, std::optional parentInputAttrPath = {}) : lockedRef(std::move(lockedRef)) , originalRef(std::move(originalRef)) , isFlake(isFlake) + , buildTime(buildTime) , parentInputAttrPath(std::move(parentInputAttrPath)) { } diff --git a/src/libflake/lockfile.cc b/src/libflake/lockfile.cc index 94e7f11f1a6..edee264edae 100644 --- a/src/libflake/lockfile.cc +++ b/src/libflake/lockfile.cc @@ -38,6 +38,7 @@ LockedNode::LockedNode(const fetchers::Settings & fetchSettings, const nlohmann: : lockedRef(getFlakeRef(fetchSettings, json, "locked", "info")) // FIXME: remove "info" , originalRef(getFlakeRef(fetchSettings, json, "original", nullptr)) , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) + , buildTime(json.find("buildTime") != json.end() ? (bool) json["buildTime"] : false) , parentInputAttrPath( json.find("parent") != json.end() ? (std::optional) json["parent"] : std::nullopt) { @@ -210,6 +211,8 @@ std::pair LockFile::toJSON() const n["locked"].erase("__final"); if (!lockedNode->isFlake) n["flake"] = false; + if (lockedNode->buildTime) + n["buildTime"] = true; if (lockedNode->parentInputAttrPath) n["parent"] = *lockedNode->parentInputAttrPath; } diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index dbe0e789399..c45c6ece356 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -851,24 +851,29 @@ struct curlFileTransfer : public FileTransfer } }; -ref makeCurlFileTransfer() -{ - return make_ref(); -} +static Sync> _fileTransfer; ref getFileTransfer() { - static ref fileTransfer = makeCurlFileTransfer(); + auto fileTransfer(_fileTransfer.lock()); - if (fileTransfer->state_.lock()->quit) - fileTransfer = makeCurlFileTransfer(); + if (!*fileTransfer || (*fileTransfer)->state_.lock()->quit) + *fileTransfer = std::make_shared(); - return fileTransfer; + return ref(*fileTransfer); } ref makeFileTransfer() { - return makeCurlFileTransfer(); + return make_ref(); +} + +std::shared_ptr resetFileTransfer() +{ + auto fileTransfer(_fileTransfer.lock()); + std::shared_ptr prev; + fileTransfer->swap(prev); + return prev; } std::future FileTransfer::enqueueFileTransfer(const FileTransferRequest & request) diff --git a/src/libstore/include/nix/store/builtins.hh b/src/libstore/include/nix/store/builtins.hh index cc164fe8273..560f0cac956 100644 --- a/src/libstore/include/nix/store/builtins.hh +++ b/src/libstore/include/nix/store/builtins.hh @@ -3,8 +3,12 @@ #include "nix/store/derivations.hh" +#include + namespace nix { +struct StructuredAttrs; + struct BuiltinBuilderContext { const BasicDerivation & drv; @@ -12,6 +16,7 @@ struct BuiltinBuilderContext std::string netrcData; std::string caFileData; Path tmpDirInSandbox; + const StructuredAttrs * parsedDrv; }; using BuiltinBuilder = std::function; diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index 8ff0de5ef2b..64ac102d77a 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -179,6 +179,8 @@ ref getFileTransfer(); */ ref makeFileTransfer(); +std::shared_ptr resetFileTransfer(); + class FileTransferError : public Error { public: diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index e9c764be2c3..bbe74b8f827 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1340,6 +1340,7 @@ void DerivationBuilderImpl::runChild() BuiltinBuilderContext ctx{ .drv = drv, .tmpDirInSandbox = tmpDirInSandbox(), + .parsedDrv = parsedDrv, }; if (drv.isBuiltin() && drv.builder == "builtin:fetchurl") { diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 56aa5880876..8de626b48cb 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -16,7 +16,7 @@ struct ExperimentalFeatureDetails /** * If two different PRs both add an experimental feature, and we just - * used a number for this, we *woudln't* get merge conflict and the + * used a number for this, we *wouldn't* get merge conflict and the * counter will be incremented once instead of twice, causing a build * failure. * @@ -24,7 +24,7 @@ struct ExperimentalFeatureDetails * feature, we either have no issue at all if few features are not added * at the end of the list, or a proper merge conflict if they are. */ -constexpr size_t numXpFeatures = 1 + static_cast(Xp::BLAKE3Hashes); +constexpr size_t numXpFeatures = 1 + static_cast(Xp::BuildTimeFetchTree); constexpr std::array xpFeatureDetails = {{ { @@ -304,6 +304,14 @@ constexpr std::array xpFeatureDetails )", .trackingUrl = "", }, + { + .tag = Xp::BuildTimeFetchTree, + .name = "build-time-fetch-tree", + .description = R"( + Enable the built-in derivation `builtin:fetch-tree`, as well as the flake input attribute `buildTime`. + )", + .trackingUrl = "", + }, }}; static_assert( diff --git a/src/libutil/include/nix/util/experimental-features.hh b/src/libutil/include/nix/util/experimental-features.hh index 2845f1e9b18..e44d4a2005d 100644 --- a/src/libutil/include/nix/util/experimental-features.hh +++ b/src/libutil/include/nix/util/experimental-features.hh @@ -36,6 +36,7 @@ enum struct ExperimentalFeature { PipeOperators, ExternalBuilders, BLAKE3Hashes, + BuildTimeFetchTree, }; extern std::set stabilizedFeatures; diff --git a/tests/functional/flakes/build-time-flake-inputs.sh b/tests/functional/flakes/build-time-flake-inputs.sh new file mode 100644 index 00000000000..467d59008bb --- /dev/null +++ b/tests/functional/flakes/build-time-flake-inputs.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +source ./common.sh + +requireGit +TODO_NixOS + +lazy="$TEST_ROOT/lazy" +createGitRepo "$lazy" +echo world > "$lazy/who" +git -C "$lazy" add who +git -C "$lazy" commit -a -m foo + +repo="$TEST_ROOT/repo" + +createGitRepo "$repo" + +cat > "$repo/flake.nix" < "$lazy/who" +git -C "$lazy" commit -a -m foo + +nix flake update --extra-experimental-features build-time-fetch-tree --flake "$repo" + +clearStore + +nix build --extra-experimental-features build-time-fetch-tree --out-link "$TEST_ROOT/result" -L "$repo" +[[ $(cat "$TEST_ROOT/result") = utrecht ]] + +rm -rf "$lazy" + +clearStore + +expectStderr 1 nix build --extra-experimental-features build-time-fetch-tree --out-link "$TEST_ROOT/result" -L "$repo" | grepQuiet "Cannot build.*source.drv" diff --git a/tests/functional/flakes/meson.build b/tests/functional/flakes/meson.build index 801fefc6f9a..b6d9610805f 100644 --- a/tests/functional/flakes/meson.build +++ b/tests/functional/flakes/meson.build @@ -34,6 +34,7 @@ suites += { 'source-paths.sh', 'old-lockfiles.sh', 'trace-ifd.sh', + 'build-time-flake-inputs.sh', ], 'workdir': meson.current_source_dir(), } diff --git a/tests/nixos/fetch-git/test-cases/build-time-fetch-tree/default.nix b/tests/nixos/fetch-git/test-cases/build-time-fetch-tree/default.nix new file mode 100644 index 00000000000..a241c877d21 --- /dev/null +++ b/tests/nixos/fetch-git/test-cases/build-time-fetch-tree/default.nix @@ -0,0 +1,49 @@ +{ config, ... }: +{ + description = "build-time fetching"; + script = '' + import json + + # add a file to the repo + client.succeed(f""" + echo ${config.name # to make the git tree and store path unique + } > {repo.path}/test-case \ + && echo chiang-mai > {repo.path}/thailand \ + && {repo.git} add test-case thailand \ + && {repo.git} commit -m 'commit1' \ + && {repo.git} push origin main + """) + + # get the NAR hash + nar_hash = json.loads(client.succeed(f""" + nix flake prefetch --flake-registry "" git+{repo.remote} --json + """))['hash'] + + # construct the derivation + expr = f""" + derivation {{ + name = "source"; + builder = "builtin:fetch-tree"; + system = "builtin"; + __structuredAttrs = true; + input = {{ + type = "git"; + url = "{repo.remote}"; + ref = "main"; + }}; + outputHashMode = "recursive"; + outputHash = "{nar_hash}"; + }} + """ + + # do the build-time fetch + out_path = client.succeed(f""" + nix build --print-out-paths --store /run/store --flake-registry "" --extra-experimental-features build-time-fetch-tree --expr '{expr}' + """).strip() + + # check if the committed file is there + client.succeed(f""" + test -f /run/store/{out_path}/thailand + """) + ''; +} diff --git a/tests/nixos/github-flakes.nix b/tests/nixos/github-flakes.nix index 67e2d41e906..6ea797cc623 100644 --- a/tests/nixos/github-flakes.nix +++ b/tests/nixos/github-flakes.nix @@ -203,6 +203,27 @@ in assert info["revision"] == "${nixpkgs.rev}", f"revision mismatch: {info['revision']} != ${nixpkgs.rev}" cat_log() + out = client.succeed("nix flake prefetch nixpkgs --json") + nar_hash = json.loads(out)['hash'] + + # Test build-time fetching of public flakes. + expr = f""" + derivation {{ + name = "source"; + builder = "builtin:fetch-tree"; + system = "builtin"; + __structuredAttrs = true; + input = {{ + type = "github"; + owner = "NixOS"; + repo = "nixpkgs"; + }}; + outputHashMode = "recursive"; + outputHash = "{nar_hash}"; + }} + """ + client.succeed(f"nix build --store /run/store --extra-experimental-features build-time-fetch-tree -L --expr '{expr}'") + # ... otherwise it should use the API out = client.succeed("nix flake metadata private-flake --json --access-tokens github.com=ghp_000000000000000000000000000000000000 --tarball-ttl 0 --no-trust-tarballs-from-git-forges") print(out) @@ -211,6 +232,24 @@ in assert info["fingerprint"] cat_log() + # Test build-time fetching of private flakes. + expr = f""" + derivation {{ + name = "source"; + builder = "builtin:fetch-tree"; + system = "builtin"; + __structuredAttrs = true; + input = {{ + type = "github"; + owner = "fancy-enterprise"; + repo = "private-flake"; + }}; + outputHashMode = "recursive"; + outputHash = "{info['locked']['narHash']}"; + }} + """ + client.succeed(f"nix build --store /run/store --extra-experimental-features build-time-fetch-tree --access-tokens github.com=ghp_000000000000000000000000000000000000 -L --expr '{expr}'") + # Fetching with the resolved URL should produce the same result. info2 = json.loads(client.succeed(f"nix flake metadata {info['url']} --json --access-tokens github.com=ghp_000000000000000000000000000000000000 --tarball-ttl 0")) print(info["fingerprint"], info2["fingerprint"])