Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion src/libexpr/include/nix/expr/eval.hh
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,7 @@ struct StaticEvalSymbols
line, column, functor, toString, right, wrong, structuredAttrs, json, allowedReferences, allowedRequisites,
disallowedReferences, disallowedRequisites, maxSize, maxClosureSize, builder, args, contentAddressed, impure,
outputHash, outputHashAlgo, outputHashMode, recurseForDerivations, description, self, epsilon, startSet,
operator_, key, path, prefix, outputSpecified;
operator_, key, path, prefix, outputSpecified, __meta, identifiers, license, licenses;

Expr::AstSymbols exprSymbols;

Expand Down Expand Up @@ -281,6 +281,10 @@ struct StaticEvalSymbols
.path = alloc.create("path"),
.prefix = alloc.create("prefix"),
.outputSpecified = alloc.create("outputSpecified"),
.__meta = alloc.create("__meta"),
.identifiers = alloc.create("identifier"),
.license = alloc.create("license"),
.licenses = alloc.create("licenses"),
.exprSymbols = {
.sub = alloc.create("__sub"),
.lessThan = alloc.create("__lessThan"),
Expand Down
1 change: 1 addition & 0 deletions src/libexpr/include/nix/expr/meson.build
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ headers = [ config_pub_h ] + files(
'print-ambiguous.hh',
'print-options.hh',
'print.hh',
'provenance.hh',
'repl-exit-status.hh',
'search-path.hh',
'static-string-data.hh',
Expand Down
23 changes: 23 additions & 0 deletions src/libexpr/include/nix/expr/provenance.hh
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
#pragma once

#include "nix/util/provenance.hh"

namespace nix {

/**
* Provenance indicating that this store path was instantiated by the `derivation` builtin function. Its main purpose is
* to record `meta` fields.
*/
struct DerivationProvenance : Provenance
{
std::shared_ptr<const Provenance> next;
ref<nlohmann::json> meta;

DerivationProvenance(std::shared_ptr<const Provenance> next, ref<nlohmann::json> meta)
: next(std::move(next))
, meta(std::move(meta)) {};

nlohmann::json to_json() const override;
};

} // namespace nix
1 change: 1 addition & 0 deletions src/libexpr/meson.build
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,7 @@ sources = files(
'primops.cc',
'print-ambiguous.cc',
'print.cc',
'provenance.cc',
'search-path.cc',
'symbol-table.cc',
'value-to-json.cc',
Expand Down
38 changes: 35 additions & 3 deletions src/libexpr/primops.cc
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
#include "nix/fetchers/fetch-to-store.hh"
#include "nix/util/sort.hh"
#include "nix/util/mounted-source-accessor.hh"
#include "nix/expr/provenance.hh"

#include <boost/container/small_vector.hpp>
#include <boost/unordered/concurrent_flat_map.hpp>
Expand Down Expand Up @@ -1504,6 +1505,8 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName
StringSet outputs;
outputs.insert("out");

auto provenance = state.evalContext.provenance;

for (auto & i : attrs->lexicographicOrder(state.symbols)) {
if (i->name == state.s.ignoreNulls)
continue;
Expand Down Expand Up @@ -1571,6 +1574,29 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName
experimentalFeatureSettings.require(Xp::ImpureDerivations);
}
break;
case EvalState::s.__meta.getId():
if (experimentalFeatureSettings.isEnabled(Xp::Provenance)) {
state.forceAttrs(*i->value, pos, "");
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

missing error message in the force

auto meta = i->value->attrs();
auto obj = nlohmann::json();

for (auto & i : meta->lexicographicOrder(state.symbols)) {
auto key = state.symbols[i->name];
switch (i->name.getId()) {
case EvalState::s.identifiers.getId():
case EvalState::s.license.getId():
case EvalState::s.licenses.getId():
obj.emplace(key, printValueAsJSON(state, true, *i->value, pos, context));
break;
default:
continue;
}
}

provenance =
std::make_shared<const DerivationProvenance>(provenance, make_ref<nlohmann::json>(obj));
}
break;
/* The `args' attribute is special: it supplies the
command-line arguments to the builder. */
case EvalState::s.args.getId():
Expand Down Expand Up @@ -1847,8 +1873,7 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName
}

/* Write the resulting term into the Nix store directory. */
auto drvPath =
writeDerivation(*state.store, *state.asyncPathWriter, drv, state.repair, false, state.evalContext.provenance);
auto drvPath = writeDerivation(*state.store, *state.asyncPathWriter, drv, state.repair, false, provenance);
auto drvPathS = state.store->printStorePath(drvPath);

printMsg(lvlChatty, "instantiated '%1%' -> '%2%'", drvName, drvPathS);
Expand Down Expand Up @@ -5420,7 +5445,14 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings)
language feature gets added. It's not necessary to increase it
when primops get added, because you can just use `builtins ?
primOp' to check. */
v.mkInt(6);
if (experimentalFeatureSettings.isEnabled(Xp::Provenance)) {
/* Provenance allows for meta to be inside of derivations.
We increment the version to 7 so Nixpkgs will know when
provenance is available. */
v.mkInt(7);
} else {
v.mkInt(6);
}
Comment on lines +5448 to +5455
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Bumping the language version in an experimental feature seems pretty wrong to me... Especially because it means that if upstream bumps its version for an unrelated reason, we now have two diverging language versions.

Why do we need to do this? What are the alternatives?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We need some sort of way to communicate that __meta is accepted inside of a derivation. We could split it into its own builtins, call it something like derivationWithMeta, but when I was looking at the code that would complicate how __meta is handled since it doesn't get serialized into the derivation and there would need to be a way to drop it.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm more interested in what happens if upstream happens to bump the language version to v7 as well.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Then people not using DetNix and pass __meta, things would break. If they are using DetNix, things will be fine.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Alternatively, we could add a field builtins.derivationSupportsMeta that Nixpkgs could check. Or my original idea, which was to have a builtins.derivationWithMeta, so then Nixpkgs could do

if builtins ? derivationWithMeta then
  builtins.derivationWithMeta (attrs // { meta = ... })
else
  builtins.derivation attrs

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

One of the problems with having it as meta was, I got some weird flakiness with evaluation where in certain cases it would say attributes didn't exist. I'd run nix repl and check that attribute and it would exist. The other problem is figuring out how to make meta behave the same with the regular derivation primop.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

addConstant(
"__langVersion",
v,
Expand Down
25 changes: 25 additions & 0 deletions src/libexpr/provenance.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
#include "nix/expr/provenance.hh"
#include "nix/util/json-utils.hh"

#include <nlohmann/json.hpp>

namespace nix {

nlohmann::json DerivationProvenance::to_json() const
{
return nlohmann::json{
{"type", "derivation"},
{"meta", *meta},
{"next", next ? next->to_json() : nlohmann::json(nullptr)},
};
}

Provenance::Register registerDerivationProvenance("derivation", [](nlohmann::json json) {
auto & obj = getObject(json);
std::shared_ptr<const Provenance> next;
if (auto p = optionalValueAt(obj, "next"); p && !p->is_null())
next = Provenance::from_json(*p);
return make_ref<DerivationProvenance>(next, make_ref<nlohmann::json>(valueAt(obj, "meta")));
});

} // namespace nix
1 change: 1 addition & 0 deletions src/nix/provenance-show.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ The provenance chain shows the history of how the store path came to exist, incl
- **Built**: The path was built from a derivation.
- **Flake evaluation**: The derivation was instantiated during the evaluation of a flake output.
- **Fetched**: The path was obtained by fetching a source tree.
- **Meta**: Metadata associated with the derivation.

Note: if you want provenance in JSON format, use the `provenance` field returned by `nix path-info --json`.

Expand Down
56 changes: 56 additions & 0 deletions src/nix/provenance.cc
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
#include "nix/cmd/command.hh"
#include "nix/store/store-api.hh"
#include "nix/expr/provenance.hh"
#include "nix/store/provenance.hh"
#include "nix/flake/provenance.hh"
#include "nix/fetchers/provenance.hh"
#include "nix/util/provenance.hh"
#include "nix/util/json-utils.hh"

#include <memory>
#include <nlohmann/json.hpp>
Expand Down Expand Up @@ -92,6 +94,60 @@ struct CmdProvenanceShow : StorePathsCommand
} else if (auto subpath = std::dynamic_pointer_cast<const SubpathProvenance>(provenance)) {
logger->cout("← from file " ANSI_BOLD "%s" ANSI_NORMAL, subpath->subpath.abs());
provenance = subpath->next;
} else if (auto drv = std::dynamic_pointer_cast<const DerivationProvenance>(provenance)) {
logger->cout("← with derivation metadata");
#define TAB " "
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there a reason not to define this closer to the top, and not undefine it?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, this was just how I did it heh. We could move it to the top, that's fine by me.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

auto json = getObject(*(drv->meta));
if (auto identifiers = optionalValueAt(json, "identifiers")) {
auto ident = getObject(*identifiers);
if (auto cpeParts = optionalValueAt(ident, "cpeParts")) {
auto parts = getObject(*cpeParts);

auto vendor = parts["vendor"];
auto product = parts["product"];
auto version = parts["version"];
auto update = parts["update"];

logger->cout(
TAB "" ANSI_BOLD "CPE:" ANSI_NORMAL " cpe:2.3:a:%s:%s:%s:%s:*:*:*:*:*:*",
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For my own knowledge:

https://en.wikipedia.org/wiki/Common_Platform_Enumeration

The a stands for "application"

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes because there's no type part.

vendor.is_null() ? "*" : vendor.get<std::string>(),
product.is_null() ? "*" : product.get<std::string>(),
version.is_null() ? "*" : version.get<std::string>(),
update.is_null() ? "*" : update.get<std::string>());
}
}
if (auto license = optionalValueAt(json, "license")) {
if (license->is_array()) {
logger->cout(TAB "" ANSI_BOLD "Licenses:" ANSI_NORMAL);
auto licenses = getArray(*license);
for (auto it = licenses.begin(); it != licenses.end(); it++) {
auto license = getObject(*it);
auto shortName = license["shortName"];
logger->cout(TAB "" TAB "- %s", shortName.get<std::string>());
}
} else {
auto obj = getObject(*license);
auto shortName = obj["shortName"];
logger->cout(TAB "" ANSI_BOLD "License:" ANSI_NORMAL " %s", shortName.get<std::string>());
}
}
if (auto licenses = optionalValueAt(json, "licenses")) {
if (licenses->is_array()) {
logger->cout(TAB "" ANSI_BOLD "Licenses:" ANSI_NORMAL);
auto licensesArray = getArray(*licenses);
for (auto it = licensesArray.begin(); it != licensesArray.end(); it++) {
auto license = getObject(*it);
auto shortName = license["shortName"];
logger->cout(TAB "" TAB "- %s", shortName.get<std::string>());
}
} else {
auto license = getObject(*licenses);
auto shortName = license["shortName"];
logger->cout(TAB "" ANSI_BOLD "License:" ANSI_NORMAL " %s", shortName.get<std::string>());
}
}
#undef TAB
provenance = drv->next;
} else {
// Unknown or unhandled provenance type
auto json = provenance->to_json();
Expand Down
7 changes: 6 additions & 1 deletion tests/functional/config.nix.in
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@ let
outputHashMode = "recursive";
outputHashAlgo = "sha256";
} else {};

optional = cond: elem: if cond then [ elem ] else [];
optionalAttrs = cond: attrs: if cond then attrs else {};
in

rec {
Expand All @@ -25,6 +28,8 @@ rec {
eval "$buildCommand"
'')];
PATH = path;
} // caArgs // removeAttrs args ["builder" "meta"])
} // caArgs // optionalAttrs (builtins.langVersion >= 7) {
__meta = args.meta or {};
} // removeAttrs args ["builder" "meta"])
// { meta = args.meta or {}; };
}
Loading