Provenance

doc/manual/source/protocols/json/schema/store-object-info-v2.yaml

@@ -179,6 +179,15 @@ $defs:
           The total size of this store object and every other object in its [closure](@docroot@/glossary.md#gloss-closure).
 
           > This field is not stored at all, but computed by traversing the other fields across all the store objects in a closure.
+
+      provenance:
+        oneOf:
+          - type: "null"
+          - type: object # FIXME
+        title: Provenance
+        description: |
+          An arbitrary JSON object containing provenance information about the store object, or `null` if not available.
+
     additionalProperties: false
 
   narInfo:
@@ -262,4 +271,13 @@ $defs:
           > This is an impure "`.narinfo`" field that may not be included in certain contexts.
 
           > This field is not stored at all, but computed by traversing the other fields across all the store objects in a closure.
+
+      provenance:
+        oneOf:
+          - type: "null"
+          - type: object # FIXME
+        title: Provenance
+        description: |
+          An arbitrary JSON object containing provenance information about the store object, or `null` if not available.
+
     additionalProperties: false

src/libcmd/include/nix/cmd/installable-flake.hh

@@ -72,6 +72,8 @@ struct InstallableFlake : InstallableValue
     ref<flake::LockedFlake> getLockedFlake() const;
 
     FlakeRef nixpkgsFlakeRef() const;
+
+    std::shared_ptr<const Provenance> makeProvenance(std::string_view attrPath) const;
 };
 
 /**

src/libcmd/installable-flake.cc

@@ -17,6 +17,7 @@
 #include "nix/util/url.hh"
 #include "nix/fetchers/registry.hh"
 #include "nix/store/build-result.hh"
+#include "nix/flake/provenance.hh"
 
 #include <regex>
 #include <queue>
@@ -84,6 +85,8 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths()
 
     auto attrPath = attr->getAttrPathStr();
 
+    PushProvenance pushedProvenance(*state, makeProvenance(attrPath));
+
     if (!attr->isDerivation()) {
 
         // FIXME: use eval cache?
@@ -172,6 +175,8 @@ std::vector<ref<eval_cache::AttrCursor>> InstallableFlake::getCursors(EvalState
     for (auto & attrPath : attrPaths) {
         debug("trying flake output attribute '%s'", attrPath);
 
+        PushProvenance pushedProvenance(state, makeProvenance(attrPath));
+
         auto attr = root->findAlongAttrPath(AttrPath::parse(state, attrPath));
         if (attr) {
             res.push_back(ref(*attr));
@@ -212,4 +217,14 @@ FlakeRef InstallableFlake::nixpkgsFlakeRef() const
     return defaultNixpkgsFlakeRef();
 }
 
+std::shared_ptr<const Provenance> InstallableFlake::makeProvenance(std::string_view attrPath) const
+{
+    if (!evalSettings.pureEval)
+        return nullptr;
+    auto provenance = getLockedFlake()->flake.provenance;
+    if (!provenance)
+        return nullptr;
+    return std::make_shared<const FlakeProvenance>(provenance, std::string(attrPath));
+}
+
 } // namespace nix

src/libexpr/eval.cc

@@ -257,6 +257,8 @@ EvalMemory::EvalMemory()
     assertGCInitialized();
 }
 
+thread_local EvalState::EvalContext EvalState::evalContext;
+
 EvalState::EvalState(
     const LookupPath & lookupPathFromArguments,
     ref<Store> store,

src/libexpr/include/nix/expr/eval.hh

@@ -52,11 +52,12 @@ enum RepairFlag : bool;
 struct MemorySourceAccessor;
 struct MountedSourceAccessor;
 struct AsyncPathWriter;
+struct Provenance;
+struct Executor;
 
 namespace eval_cache {
 class EvalCache;
 }
-struct Executor;
 
 /**
  * Increments a count on construction and decrements on destruction.
@@ -1128,6 +1129,45 @@ private:
     friend class ListBuilder;
 
 public:
+
+    /**
+     * Per-thread evaluation context. This context is propagated to worker threads when a value is evaluated
+     * asynchronously.
+     */
+    struct EvalContext
+    {
+        std::shared_ptr<const Provenance> provenance;
+    };
+
+    thread_local static EvalContext evalContext;
+
+    /**
+     * Create a work item that propagates the current evaluation context.
+     */
+    template<typename T>
+    auto makeWork(T && t)
+    {
+        return [this, t{std::move(t)}, evalContext(evalContext)]() {
+            this->evalContext = evalContext;
+            t();
+        };
+    }
+
+    /**
+     * Add a work item to the given work vector that propagates the current evaluation context.
+     */
+    template<typename WorkItems, typename T>
+    void addWork(WorkItems & work, uint8_t priority, T && t)
+    {
+        work.emplace_back(makeWork(std::move(t)), priority);
+    }
+
+    template<typename FuturesVector, typename T>
+    void spawn(FuturesVector & futures, uint8_t priority, T && t)
+    {
+        futures.spawn(priority, makeWork(std::move(t)));
+    }
+
     /**
      * Worker threads manager.
      *
@@ -1173,6 +1213,24 @@ SourcePath resolveExprPath(SourcePath path, bool addDefaultNix = true);
  */
 bool isAllowedURI(std::string_view uri, const Strings & allowedPaths);
 
+struct PushProvenance
+{
+    EvalState & state;
+    std::shared_ptr<const Provenance> prev;
+
+    PushProvenance(EvalState & state, std::shared_ptr<const Provenance> prov)
+        : state(state)
+    {
+        state.evalContext.provenance.swap(prev);
+        state.evalContext.provenance.swap(prov);
+    }
+
+    ~PushProvenance()
+    {
+        state.evalContext.provenance.swap(prev);
+    }
+};
+
 } // namespace nix
 
 #include "nix/expr/eval-inline.hh"

src/libexpr/include/nix/expr/parallel-eval.hh

@@ -57,7 +57,9 @@ struct Executor
 
     void worker();
 
-    std::vector<std::future<void>> spawn(std::vector<std::pair<work_t, uint8_t>> && items);
+    using WorkItems = std::vector<std::pair<Executor::work_t, uint8_t>>;
+
+    std::vector<std::future<void>> spawn(WorkItems && items);
 
     static thread_local bool amWorkerThread;
 };
@@ -77,7 +79,7 @@ struct FutureVector
 
     // FIXME: add a destructor that cancels/waits for all futures.
 
-    void spawn(std::vector<std::pair<Executor::work_t, uint8_t>> && work);
+    void spawn(Executor::WorkItems && work);
 
     void spawn(uint8_t prioPrefix, Executor::work_t && work)
     {

src/libexpr/parallel-eval.cc

@@ -110,7 +110,7 @@ void Executor::worker()
     }
 }
 
-std::vector<std::future<void>> Executor::spawn(std::vector<std::pair<work_t, uint8_t>> && items)
+std::vector<std::future<void>> Executor::spawn(WorkItems && items)
 {
     if (items.empty())
         return {};
@@ -146,7 +146,7 @@ FutureVector::~FutureVector()
     }
 }
 
-void FutureVector::spawn(std::vector<std::pair<Executor::work_t, uint8_t>> && work)
+void FutureVector::spawn(Executor::WorkItems && work)
 {
     auto futures = executor.spawn(std::move(work));
     auto state(state_.lock());
@@ -273,10 +273,11 @@ static void prim_parallel(EvalState & state, const PosIdx pos, Value ** args, Va
     state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.parallel");
 
     if (state.executor->evalCores > 1) {
-        std::vector<std::pair<Executor::work_t, uint8_t>> work;
+        Executor::WorkItems work;
         for (auto value : args[0]->listView())
             if (!value->isFinished())
-                work.emplace_back([value(allocRootValue(value)), &state, pos]() { state.forceValue(**value, pos); }, 0);
+                state.addWork(
+                    work, 0, [value(allocRootValue(value)), &state, pos]() { state.forceValue(**value, pos); });
         state.executor->spawn(std::move(work));
     }
 

src/libexpr/primops.cc

@@ -1847,7 +1847,8 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName
     }
 
     /* Write the resulting term into the Nix store directory. */
-    auto drvPath = writeDerivation(*state.store, *state.asyncPathWriter, drv, state.repair);
+    auto drvPath =
+        writeDerivation(*state.store, *state.asyncPathWriter, drv, state.repair, false, state.evalContext.provenance);
     auto drvPathS = state.store->printStorePath(drvPath);
 
     printMsg(lvlChatty, "instantiated '%1%' -> '%2%'", drvName, drvPathS);
@@ -2750,7 +2751,8 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value ** args, Valu
                                                      ContentAddressMethod::Raw::Text,
                                                      HashAlgorithm::SHA256,
                                                      refs,
-                                                     state.repair);
+                                                     state.repair,
+                                                     state.evalContext.provenance);
                                              });
 
     /* Note: we don't need to add `context' to the context of the

src/libexpr/value-to-json.cc

@@ -18,7 +18,7 @@ static void parallelForceDeep(EvalState & state, Value & v, PosIdx pos)
 {
     state.forceValue(v, pos);
 
-    std::vector<std::pair<Executor::work_t, uint8_t>> work;
+    Executor::WorkItems work;
 
     switch (v.type()) {
 
@@ -29,8 +29,9 @@ static void parallelForceDeep(EvalState & state, Value & v, PosIdx pos)
         if (v.attrs()->get(state.s.outPath))
             return;
         for (auto & a : *v.attrs())
-            work.emplace_back(
-                [value(allocRootValue(a.value)), pos(a.pos), &state]() { parallelForceDeep(state, **value, pos); }, 0);
+            state.addWork(work, 0, [value(allocRootValue(a.value)), pos(a.pos), &state]() {
+                parallelForceDeep(state, **value, pos);
+            });
         break;
     }
 

src/libfetchers/attrs.cc

@@ -27,6 +27,9 @@ nlohmann::json attrsToJSON(const Attrs & attrs)
 {
     nlohmann::json json;
     for (auto & attr : attrs) {
+        /* The __final attribute is purely internal, so never serialize it. */
+        if (attr.first == "__final")
+            continue;
         if (auto v = std::get_if<uint64_t>(&attr.second)) {
             json[attr.first] = *v;
         } else if (auto v = std::get_if<std::string>(&attr.second)) {

src/libfetchers/fetchers.cc

@@ -4,7 +4,7 @@
 #include "nix/fetchers/fetch-to-store.hh"
 #include "nix/util/json-utils.hh"
 #include "nix/fetchers/fetch-settings.hh"
-#include "nix/fetchers/fetch-to-store.hh"
+#include "nix/fetchers/provenance.hh"
 #include "nix/util/url.hh"
 #include "nix/util/forwarding-source-accessor.hh"
 #include "nix/util/archive.hh"
@@ -196,7 +196,6 @@ bool Input::contains(const Input & other) const
     return false;
 }
 
-// FIXME: remove
 std::tuple<StorePath, ref<SourceAccessor>, Input> Input::fetchToStore(const Settings & settings, Store & store) const
 {
     if (!scheme)
@@ -339,6 +338,9 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(const Settings
                 {{"hash", store.queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true)}});
         }
 
+        if (isLocked(settings))
+            accessor->provenance = std::make_shared<TreeProvenance>(*this);
+
         // FIXME: ideally we would use the `showPath()` of the
         // "real" accessor for this fetcher type.
         accessor->setPathDisplay("«" + to_string(true) + "»");
@@ -363,6 +365,9 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(const Settings
         else
             accessor->fingerprint = result.getFingerprint(store);
 
+        if (result.isLocked(settings))
+            accessor->provenance = std::make_shared<TreeProvenance>(result);
+
         return {accessor, std::move(result)};
     } catch (Error & e) {
         if (storePath) {

src/libfetchers/filtering-source-accessor.cc

@@ -68,6 +68,13 @@ std::pair<CanonPath, std::optional<std::string>> FilteringSourceAccessor::getFin
     return next->getFingerprint(prefix / path);
 }
 
+std::shared_ptr<const Provenance> FilteringSourceAccessor::getProvenance(const CanonPath & path)
+{
+    if (provenance)
+        return SourceAccessor::getProvenance(path);
+    return next->getProvenance(prefix / path);
+}
+
 void FilteringSourceAccessor::invalidateCache(const CanonPath & path)
 {
     next->invalidateCache(prefix / path);

src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh

@@ -52,6 +52,8 @@ struct FilteringSourceAccessor : SourceAccessor
 
     std::pair<CanonPath, std::optional<std::string>> getFingerprint(const CanonPath & path) override;
 
+    std::shared_ptr<const Provenance> getProvenance(const CanonPath & path) override;
+
     void invalidateCache(const CanonPath & path) override;
 
     /**

src/libfetchers/include/nix/fetchers/meson.build

@@ -10,6 +10,7 @@ headers = files(
   'git-lfs-fetch.hh',
   'git-utils.hh',
   'input-cache.hh',
+  'provenance.hh',
   'registry.hh',
   'tarball.hh',
 )

src/libfetchers/include/nix/fetchers/provenance.hh

@@ -0,0 +1,17 @@
+#pragma once
+
+#include "nix/util/provenance.hh"
+#include "nix/fetchers/fetchers.hh"
+
+namespace nix {
+
+struct TreeProvenance : Provenance
+{
+    ref<nlohmann::json> attrs;
+
+    TreeProvenance(const fetchers::Input & input);
+
+    nlohmann::json to_json() const override;
+};
+
+} // namespace nix

src/libfetchers/meson.build

@@ -51,6 +51,7 @@ sources = files(
   'input-cache.cc',
   'mercurial.cc',
   'path.cc',
+  'provenance.cc',
   'registry.cc',
   'tarball.cc',
 )

src/libfetchers/provenance.cc

@@ -0,0 +1,27 @@
+#include "nix/fetchers/provenance.hh"
+#include "nix/fetchers/attrs.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix {
+
+TreeProvenance::TreeProvenance(const fetchers::Input & input)
+    : attrs(make_ref<nlohmann::json>([&]() {
+        // Remove the narHash attribute from the provenance info, as it's redundant (it's already recorded in the store
+        // path info).
+        auto attrs2 = input.attrs;
+        attrs2.erase("narHash");
+        return fetchers::attrsToJSON(attrs2);
+    }()))
+{
+}
+
+nlohmann::json TreeProvenance::to_json() const
+{
+    return nlohmann::json{
+        {"type", "tree"},
+        {"attrs", *attrs},
+    };
+}
+
+} // namespace nix