From e83285ef77612e61fe5589630f546b227ed9ac33 Mon Sep 17 00:00:00 2001 From: Yannik Sander Date: Sun, 15 May 2022 23:09:45 +0200 Subject: [PATCH 1/4] MVC: Locally register flakes --- src/libexpr/flake/flake.cc | 17 ++++++++++++++++- src/libexpr/flake/flake.hh | 1 + 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index cbf4f0a6f92..580afa9b578 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -7,6 +7,7 @@ #include "fetchers.hh" #include "finally.hh" #include "fetch-settings.hh" +#include "registry.hh" namespace nix { @@ -104,6 +105,7 @@ static FlakeInput parseFlakeInput(EvalState & state, auto sUrl = state.symbols.create("url"); auto sFlake = state.symbols.create("flake"); auto sFollows = state.symbols.create("follows"); + auto sRegister = state.symbols.create("register"); fetchers::Attrs attrs; std::optional url; @@ -124,7 +126,11 @@ static FlakeInput parseFlakeInput(EvalState & state, auto follows(parseInputPath(attr.value->string.s)); follows.insert(follows.begin(), lockRootPath.begin(), lockRootPath.end()); input.follows = follows; - } else { + } else if (attr.name == sRegister) { + expectType(state, nBool, *attr.value, attr.pos); + input.registered = attr.value; + } + else { switch (attr.value->type()) { case nString: attrs.emplace(state.symbols[attr.name], attr.value->string.s); @@ -166,6 +172,15 @@ static FlakeInput parseFlakeInput(EvalState & state, if (!input.follows && !input.ref) input.ref = FlakeRef::fromAttrs({{"type", "indirect"}, {"id", inputName}}); + if (input.registered) + { + debug("Registering url: %s", parseFlakeRef(*url, baseDir, true, input.isFlake).to_string()); + fetchers::overrideRegistry( + fetchers::Input::fromAttrs({{"type", "indirect"}, {"id", inputName}}), + parseFlakeRef(*url, baseDir, true, input.isFlake).input, + parseFlakeRef(*url, baseDir, true, input.isFlake).toAttrs()); + } + return input; } diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index 524b18af175..c6b49e71844 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -42,6 +42,7 @@ struct FlakeInput { std::optional ref; bool isFlake = true; // true = process flake to get outputs, false = (fetched) static source path + bool registered = false; std::optional follows; FlakeInputs overrides; }; From 0e1e84b59080f6233b527e8b63a6e0e0b5eb0acd Mon Sep 17 00:00:00 2001 From: Yannik Sander Date: Mon, 16 May 2022 13:26:10 +0200 Subject: [PATCH 2/4] New approach reusing actual lock nodes --- src/libexpr/flake/flake.cc | 46 +++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 580afa9b578..ce9876765f3 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -174,11 +174,6 @@ static FlakeInput parseFlakeInput(EvalState & state, if (input.registered) { - debug("Registering url: %s", parseFlakeRef(*url, baseDir, true, input.isFlake).to_string()); - fetchers::overrideRegistry( - fetchers::Input::fromAttrs({{"type", "indirect"}, {"id", inputName}}), - parseFlakeRef(*url, baseDir, true, input.isFlake).input, - parseFlakeRef(*url, baseDir, true, input.isFlake).toAttrs()); } return input; @@ -368,13 +363,14 @@ LockedFlake lockFlake( std::vector parents; std::function node, - const InputPath & inputPathPrefix, + const InputPath &inputPathPrefix, std::shared_ptr oldNode, - const InputPath & lockRootPath, - const Path & parentPath, - bool trustLock)> + const InputPath &lockRootPath, + const Path &parentPath, + bool trustLock, + std::map> registered)> computeLocks; computeLocks = [&]( @@ -384,7 +380,8 @@ LockedFlake lockFlake( std::shared_ptr oldNode, const InputPath & lockRootPath, const Path & parentPath, - bool trustLock) + bool trustLock, + std::map> registered) { debug("computing lock file node '%s'", printInputPath(inputPathPrefix)); @@ -434,7 +431,16 @@ LockedFlake lockFlake( continue; } - assert(input.ref); + auto registeredInput = registered.find(id); + if (registeredInput != registered.end()) { + debug("Using already registered node for %s", id); + // InputPath target; + // target.insert(target.end(), input.registered->begin(), input.registered->end()); + node->inputs.insert_or_assign(id, registeredInput->second); + continue; + } + + assert(input.ref); /* Do we have an entry in the existing lock file? And we don't have a --update-input flag for this input? */ @@ -518,7 +524,7 @@ LockedFlake lockFlake( mustRefetch ? getFlake(state, oldLock->lockedRef, false, flakeCache, inputPath).inputs : fakeInputs, - childNode, inputPath, oldLock, lockRootPath, parentPath, !mustRefetch); + childNode, inputPath, oldLock, lockRootPath, parentPath, !mustRefetch, registered); } else { /* We need to create a new lock file entry. So fetch @@ -549,6 +555,15 @@ LockedFlake lockFlake( auto childNode = std::make_shared( inputFlake.lockedRef, input2.ref ? *input2.ref : *input.ref); + /* Get the registered flakes (e.g. 'inputs.nixpkgs.register = true;') */ + + if (input.registered) + { + debug("Registering: %s: %s", id.c_str(), input.ref->to_string()); + registered.insert_or_assign(id, childNode); + } + + node->inputs.insert_or_assign(id, childNode); /* Guard against circular flake imports. */ @@ -568,7 +583,7 @@ LockedFlake lockFlake( ? std::dynamic_pointer_cast(oldLock) : LockFile::read( inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root, - oldLock ? lockRootPath : inputPath, localPath, false); + oldLock ? lockRootPath : inputPath, localPath, false, registered); } else { @@ -588,10 +603,11 @@ LockedFlake lockFlake( // Bring in the current ref for relative path resolution if we have it auto parentPath = canonPath(flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir, true); + std::map> registered; computeLocks( flake.inputs, newLockFile.root, {}, - lockFlags.recreateLockFile ? nullptr : oldLockFile.root, {}, parentPath, false); + lockFlags.recreateLockFile ? nullptr : oldLockFile.root, {}, parentPath, false, registered); for (auto & i : lockFlags.inputOverrides) if (!overridesUsed.count(i.first)) From 26d4594cb18d924dda90efb03112c5b5c2281b06 Mon Sep 17 00:00:00 2001 From: Yannik Sander Date: Tue, 17 May 2022 00:47:10 +0200 Subject: [PATCH 3/4] Lock nodes facilitating following mechanism --- src/libexpr/flake/flake.cc | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index ce9876765f3..74802562386 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -370,7 +370,7 @@ LockedFlake lockFlake( const InputPath &lockRootPath, const Path &parentPath, bool trustLock, - std::map> registered)> + std::map registered)> computeLocks; computeLocks = [&]( @@ -381,10 +381,12 @@ LockedFlake lockFlake( const InputPath & lockRootPath, const Path & parentPath, bool trustLock, - std::map> registered) + std::map registered) { debug("computing lock file node '%s'", printInputPath(inputPathPrefix)); + std::set registeredChilds; + /* Get the overrides (i.e. attributes of the form 'inputs.nixops.inputs.nixpkgs.url = ...'). */ for (auto & [id, input] : flakeInputs) { @@ -394,6 +396,16 @@ LockedFlake lockFlake( inputPath.push_back(idOverride); overrides.insert_or_assign(inputPath, inputOverride); } + if (input.registered) { + // This is a stub node that is overriden in the input + // auto childNode = std::make_shared( + // *input.ref, *input.ref, input.isFlake); + auto inputPath(inputPathPrefix); + inputPath.push_back(id); + debug("preregistering %s", id.c_str()); + registeredChilds.insert(id); + registered.insert_or_assign(id, inputPath); + } } /* Go over the flake inputs, resolve/fetch them if @@ -431,12 +443,15 @@ LockedFlake lockFlake( continue; } - auto registeredInput = registered.find(id); - if (registeredInput != registered.end()) { + auto registeredInputID = registered.find(id); + if (registeredInputID != registered.end() + && (registeredChilds.find(id) == registeredChilds.end()) + && (input.ref->to_string() == "flake:" + id)) + { debug("Using already registered node for %s", id); - // InputPath target; - // target.insert(target.end(), input.registered->begin(), input.registered->end()); - node->inputs.insert_or_assign(id, registeredInput->second); + InputPath target; + target.insert(target.end(), registeredInputID->second.begin(), registeredInputID->second.end()); + node->inputs.insert_or_assign(id, target); continue; } @@ -559,8 +574,6 @@ LockedFlake lockFlake( if (input.registered) { - debug("Registering: %s: %s", id.c_str(), input.ref->to_string()); - registered.insert_or_assign(id, childNode); } @@ -603,7 +616,7 @@ LockedFlake lockFlake( // Bring in the current ref for relative path resolution if we have it auto parentPath = canonPath(flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir, true); - std::map> registered; + std::map registered; computeLocks( flake.inputs, newLockFile.root, {}, From f5bb9ff490ca754edf6625a983ea80105bc754d4 Mon Sep 17 00:00:00 2001 From: Yannik Sander Date: Tue, 17 May 2022 10:11:33 +0200 Subject: [PATCH 4/4] Cleanup Code Style and Comments --- src/libexpr/flake/flake.cc | 32 +++++++++++--------------------- 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 74802562386..10f99b8f0bf 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -172,10 +172,6 @@ static FlakeInput parseFlakeInput(EvalState & state, if (!input.follows && !input.ref) input.ref = FlakeRef::fromAttrs({{"type", "indirect"}, {"id", inputName}}); - if (input.registered) - { - } - return input; } @@ -363,12 +359,12 @@ LockedFlake lockFlake( std::vector parents; std::function node, - const InputPath &inputPathPrefix, + const InputPath & inputPathPrefix, std::shared_ptr oldNode, - const InputPath &lockRootPath, - const Path &parentPath, + const InputPath & lockRootPath, + const Path & parentPath, bool trustLock, std::map registered)> computeLocks; @@ -397,9 +393,9 @@ LockedFlake lockFlake( overrides.insert_or_assign(inputPath, inputOverride); } if (input.registered) { - // This is a stub node that is overriden in the input - // auto childNode = std::make_shared( - // *input.ref, *input.ref, input.isFlake); + /* Record the name and the eventual path of + registered inputs. Registered inputs are set to follow the recorded input path. + Only in the defining flake the registered input is handled explicitly to catch transistive inputs */ auto inputPath(inputPathPrefix); inputPath.push_back(id); debug("preregistering %s", id.c_str()); @@ -445,8 +441,9 @@ LockedFlake lockFlake( auto registeredInputID = registered.find(id); if (registeredInputID != registered.end() - && (registeredChilds.find(id) == registeredChilds.end()) - && (input.ref->to_string() == "flake:" + id)) + && (registeredChilds.find(id) == registeredChilds.end()) // do not substitute original input + && (input.ref->to_string() == "flake:" + id) // check that the input is not explicitly defined + ) { debug("Using already registered node for %s", id); InputPath target; @@ -455,7 +452,7 @@ LockedFlake lockFlake( continue; } - assert(input.ref); + assert(input.ref); /* Do we have an entry in the existing lock file? And we don't have a --update-input flag for this input? */ @@ -570,13 +567,6 @@ LockedFlake lockFlake( auto childNode = std::make_shared( inputFlake.lockedRef, input2.ref ? *input2.ref : *input.ref); - /* Get the registered flakes (e.g. 'inputs.nixpkgs.register = true;') */ - - if (input.registered) - { - } - - node->inputs.insert_or_assign(id, childNode); /* Guard against circular flake imports. */