Introduce StoreReferences and ContentAddressWithReferences

perl/lib/Nix/Store.xs

@@ -111,7 +111,7 @@ SV * queryPathInfo(char * path, int base32)
             mXPUSHi(info->registrationTime);
             mXPUSHi(info->narSize);
             AV * arr = newAV();
-            for (auto & i : info->references)
+            for (auto & i : info->referencesPossiblyToSelf())
                 av_push(arr, newSVpv(store()->printStorePath(i).c_str(), 0));
             XPUSHs(sv_2mortal(newRV((SV *) arr)));
         } catch (Error & e) {
@@ -287,7 +287,13 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name)
         try {
             auto h = Hash::parseAny(hash, parseHashType(algo));
             auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
-            auto path = store()->makeFixedOutputPath(method, h, name);
+            auto path = store()->makeFixedOutputPath(name, FixedOutputInfo {
+                {
+                    .method = method,
+                    .hash = h,
+                },
+                {},
+            });
             XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());

src/libexpr/primops.cc

@@ -1045,7 +1045,13 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo);
         Hash h = newHashAllowEmpty(*outputHash, ht);
 
-        auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName);
+        auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputInfo {
+            {
+                .method = ingestionMethod,
+                .hash = h,
+            },
+            {},
+        });
         drv.env["out"] = state.store->printStorePath(outPath);
         drv.outputs.insert_or_assign("out", DerivationOutput {
                 .output = DerivationOutputCAFixed {
@@ -1764,7 +1770,13 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con
 
     std::optional<StorePath> expectedStorePath;
     if (expectedHash)
-        expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name);
+        expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo {
+            {
+                .method = method,
+                .hash = *expectedHash,
+            },
+            {},
+        });
     Path dstPath;
     if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
         dstPath = state.store->printStorePath(settings.readOnlyMode

src/libexpr/value-to-xml.hh

@@ -10,5 +10,5 @@ namespace nix {
 
 void printValueAsXML(EvalState & state, bool strict, bool location,
     Value & v, std::ostream & out, PathSet & context);
-    
+
 }

src/libfetchers/fetchers.cc

@@ -198,7 +198,13 @@ StorePath Input::computeStorePath(Store & store) const
     auto narHash = getNarHash();
     if (!narHash)
         throw Error("cannot compute store path for mutable input '%s'", to_string());
-    return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, "source");
+    return store.makeFixedOutputPath("source", FixedOutputInfo {
+        {
+            .method = FileIngestionMethod::Recursive,
+            .hash = *narHash,
+        },
+        {},
+    });
 }
 
 std::string Input::getType() const

src/libfetchers/tarball.cc

@@ -71,14 +71,20 @@ DownloadFileResult downloadFile(
         dumpString(*res.data, sink);
         auto hash = hashString(htSHA256, *res.data);
         ValidPathInfo info {
-            store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name),
+            *store,
+            {
+                .name = name,
+                .info = FixedOutputInfo {
+                    {
+                        .method = FileIngestionMethod::Flat,
+                        .hash = hash,
+                    },
+                    {},
+                },
+            },
             hashString(htSHA256, *sink.s),
         };
         info.narSize = sink.s->size();
-        info.ca = FixedOutputHash {
-            .method = FileIngestionMethod::Flat,
-            .hash = hash,
-        };
         auto source = StringSource { *sink.s };
         store->addToStore(info, source, NoRepair, NoCheckSigs);
         storePath = std::move(info.path);

src/libstore/binary-cache-store.cc

@@ -193,8 +193,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
        reads, but typically they'll already be cached. */
     for (auto & ref : info.references)
         try {
-            if (ref != info.path)
-                queryPathInfo(ref);
+            queryPathInfo(ref);
         } catch (InvalidPath &) {
             throw Error("cannot add '%s' to the binary cache because the reference '%s' is not valid",
                 printStorePath(info.path), printStorePath(ref));
@@ -323,7 +322,17 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & nam
         unsupported("addToStoreFromDump");
     return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) {
         ValidPathInfo info {
-            makeFixedOutputPath(method, nar.first, name),
+            *this,
+            {
+                .name = name,
+                .info = FixedOutputInfo {
+                    {
+                        .method = method,
+                        .hash = nar.first,
+                    },
+                    {},
+                },
+            },
             nar.first,
         };
         info.narSize = nar.second;
@@ -413,14 +422,20 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
     });
     return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) {
         ValidPathInfo info {
-            makeFixedOutputPath(method, h, name),
+            *this,
+            {
+                .name = name,
+                .info = FixedOutputInfo {
+                    {
+                        .method = method,
+                        .hash = h,
+                    },
+                    {},
+                },
+            },
             nar.first,
         };
         info.narSize = nar.second;
-        info.ca = FixedOutputHash {
-            .method = method,
-            .hash = h,
-        };
         return info;
     })->path;
 }
@@ -429,17 +444,26 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
     const StorePathSet & references, RepairFlag repair)
 {
     auto textHash = hashString(htSHA256, s);
-    auto path = makeTextPath(name, textHash, references);
+    auto path = makeTextPath(name, TextInfo { textHash, references });
 
     if (!repair && isValidPath(path))
         return path;
 
     auto source = StringSource { s };
     return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
-        ValidPathInfo info { path, nar.first };
+        ValidPathInfo info {
+            *this,
+            {
+                .name = name,
+                .info = TextInfo {
+                    { .hash = textHash },
+                    references,
+                },
+            },
+            nar.first,
+        };
         info.narSize = nar.second;
         info.ca = TextHash { textHash };
-        info.references = references;
         return info;
     })->path;
 }

src/libstore/build.cc

@@ -4008,27 +4008,26 @@ void DerivationGoal::registerOutputs()
             }
         };
 
-        auto rewriteRefs = [&]() -> std::pair<bool, StorePathSet> {
+        auto rewriteRefs = [&]() -> PathReferences<StorePath> {
             /* In the CA case, we need the rewritten refs to calculate the
                final path, therefore we look for a *non-rewritten
                self-reference, and use a bool rather try to solve the
                computationally intractable fixed point. */
-            std::pair<bool, StorePathSet> res {
-                false,
-                {},
+            PathReferences<StorePath> res {
+                .hasSelfReference = false,
             };
             for (auto & r : references) {
                 auto name = r.name();
                 auto origHash = std::string { r.hashPart() };
                 if (r == scratchPath)
-                    res.first = true;
+                    res.hasSelfReference = true;
                 else if (outputRewrites.count(origHash) == 0)
-                    res.second.insert(r);
+                    res.references.insert(r);
                 else {
                     std::string newRef = outputRewrites.at(origHash);
                     newRef += '-';
                     newRef += name;
-                    res.second.insert(StorePath { newRef });
+                    res.references.insert(StorePath { newRef });
                 }
             }
             return res;
@@ -4057,27 +4056,24 @@ void DerivationGoal::registerOutputs()
                 break;
             }
             auto got = caSink.finish().first;
-            auto refs = rewriteRefs();
             HashModuloSink narSink { htSHA256, oldHashPart };
             dumpPath(actualPath, narSink);
             auto narHashAndSize = narSink.finish();
             ValidPathInfo newInfo0 {
-                worker.store.makeFixedOutputPath(
-                    outputHash.method,
-                    got,
-                    outputPathName(drv->name, outputName),
-                    refs.second,
-                    refs.first),
+                worker.store,
+                {
+                    .name = outputPathName(drv->name, outputName),
+                    .info = FixedOutputInfo {
+                        {
+                            .method = outputHash.method,
+                            .hash = got,
+                        },
+                        rewriteRefs(),
+                    },
+                },
                 narHashAndSize.first,
             };
             newInfo0.narSize = narHashAndSize.second;
-            newInfo0.ca = FixedOutputHash {
-                .method = outputHash.method,
-                .hash = got,
-            };
-            newInfo0.references = refs.second;
-            if (refs.first)
-                newInfo0.references.insert(newInfo0.path);
 
             assert(newInfo0.ca);
             return newInfo0;
@@ -4097,10 +4093,7 @@ void DerivationGoal::registerOutputs()
                 auto narHashAndSize = hashPath(htSHA256, actualPath);
                 ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first };
                 newInfo0.narSize = narHashAndSize.second;
-                auto refs = rewriteRefs();
-                newInfo0.references = refs.second;
-                if (refs.first)
-                    newInfo0.references.insert(newInfo0.path);
+                static_cast<PathReferences<StorePath> &>(newInfo0) = rewriteRefs();
                 return newInfo0;
             },
             [&](DerivationOutputCAFixed dof) {
@@ -4366,12 +4359,12 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
                 auto i = outputsByPath.find(worker.store.printStorePath(path));
                 if (i != outputsByPath.end()) {
                     closureSize += i->second.narSize;
-                    for (auto & ref : i->second.references)
+                    for (auto & ref : i->second.referencesPossiblyToSelf())
                         pathsLeft.push(ref);
                 } else {
                     auto info = worker.store.queryPathInfo(path);
                     closureSize += info->narSize;
-                    for (auto & ref : info->references)
+                    for (auto & ref : info->referencesPossiblyToSelf())
                         pathsLeft.push(ref);
                 }
             }
@@ -4410,7 +4403,7 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
 
                 auto used = recursive
                     ? getClosure(info.path).first
-                    : info.references;
+                    : info.referencesPossiblyToSelf();
 
                 if (recursive && checks.ignoreSelfRefs)
                     used.erase(info.path);
@@ -4867,7 +4860,10 @@ void SubstitutionGoal::tryNext()
     subs.pop_front();
 
     if (ca) {
-        subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca);
+        subPath = sub->makeFixedOutputPathFromCA({
+            .name = std::string { storePath.name() },
+            .info = caWithoutRefs(*ca),
+        });
         if (sub->storeDir == worker.store.storeDir)
             assert(subPath == storePath);
     } else if (sub->storeDir != worker.store.storeDir) {
@@ -4897,7 +4893,7 @@ void SubstitutionGoal::tryNext()
     }
 
     if (info->path != storePath) {
-        if (info->isContentAddressed(*sub) && info->references.empty()) {
+        if (info->isContentAddressed(*sub) && info->references.empty() && !info->hasSelfReference) {
             auto info2 = std::make_shared<ValidPathInfo>(*info);
             info2->path = storePath;
             info = info2;
@@ -4940,8 +4936,7 @@ void SubstitutionGoal::tryNext()
     /* To maintain the closure invariant, we first have to realise the
        paths referenced by this one. */
     for (auto & i : info->references)
-        if (i != storePath) /* ignore self-references */
-            addWaitee(worker.makeSubstitutionGoal(i));
+        addWaitee(worker.makeSubstitutionGoal(i));
 
     if (waitees.empty()) /* to prevent hang (no wake-up event) */
         referencesValid();
@@ -4961,8 +4956,7 @@ void SubstitutionGoal::referencesValid()
     }
 
     for (auto & i : info->references)
-        if (i != storePath) /* ignore self-references */
-            assert(worker.store.isValidPath(i));
+        assert(worker.store.isValidPath(i));
 
     state = &SubstitutionGoal::tryToRun;
     worker.wakeUp(shared_from_this());

src/libstore/content-address.cc

@@ -9,16 +9,16 @@ std::string FixedOutputHash::printMethodAlgo() const
     return makeFileIngestionPrefix(method) + printHashType(hash.type);
 }
 
+
 std::string makeFileIngestionPrefix(const FileIngestionMethod m)
 {
     switch (m) {
     case FileIngestionMethod::Flat:
         return "";
     case FileIngestionMethod::Recursive:
         return "r:";
-    default:
-        throw Error("impossible, caught both cases");
     }
+    assert(false);
 }
 
 std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
@@ -32,10 +32,13 @@ std::string renderContentAddress(ContentAddress ca)
 {
     return std::visit(overloaded {
         [](TextHash th) {
-            return "text:" + th.hash.to_string(Base32, true);
+            return "text:"
+                + th.hash.to_string(Base32, true);
         },
         [](FixedOutputHash fsh) {
-            return makeFixedOutputCA(fsh.method, fsh.hash);
+            return "fixed:"
+                + makeFileIngestionPrefix(fsh.method)
+                + fsh.hash.to_string(Base32, true);
         }
     }, ca);
 }
@@ -142,7 +145,18 @@ Hash getContentAddressHash(const ContentAddress & ca)
         },
         [](FixedOutputHash fsh) {
             return fsh.hash;
-        }
+        },
+    }, ca);
+}
+
+ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) {
+    return std::visit(overloaded {
+        [&](TextHash h) -> ContentAddressWithReferences {
+            return TextInfo { h, {}};
+        },
+        [&](FixedOutputHash h) -> ContentAddressWithReferences {
+            return FixedOutputInfo { h, {}};
+        },
     }, ca);
 }