From a24ebd21e89f8277a7a7467a70f5523bac0ae77a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 24 Jun 2020 22:18:40 +0000 Subject: [PATCH 01/31] Use `FixedOutputHash` in parameters for `makeFixedOutputPath` --- src/libexpr/primops.cc | 10 ++++++++-- src/libfetchers/tarball.cc | 9 +++++---- src/libfetchers/tree-info.cc | 5 ++++- src/libstore/binary-cache-store.cc | 5 ++++- src/libstore/build.cc | 5 ++++- src/libstore/content-address.cc | 15 +++++---------- src/libstore/content-address.hh | 4 ---- src/libstore/local-store.cc | 10 +++++----- src/libstore/store-api.cc | 20 +++++++++++--------- src/libstore/store-api.hh | 13 +++++++++---- src/nix-prefetch-url/nix-prefetch-url.cc | 16 +++++++++++----- src/nix-store/nix-store.cc | 5 ++++- src/nix/add-to-store.cc | 5 ++++- src/nix/make-content-addressable.cc | 5 ++++- 14 files changed, 78 insertions(+), 49 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 3830d8107f11..ea85ad2ba272 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -772,7 +772,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * std::optional ht = parseHashTypeOpt(outputHashAlgo); Hash h = newHashAllowEmpty(*outputHash, ht); - auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName); + auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputHash { + .method = ingestionMethod, + .hash = h + }); if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", DerivationOutput { .path = std::move(outPath), @@ -1153,7 +1156,10 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con std::optional expectedStorePath; if (expectedHash) - expectedStorePath = state.store->makeFixedOutputPath(method, expectedHash, name); + expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputHash { + .method = method, + .hash = expectedHash, + }); Path dstPath; if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { dstPath = state.store->printStorePath(settings.readOnlyMode diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index f5356f0af8b3..7f9b4d434ea9 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -67,13 +67,14 @@ DownloadFileResult downloadFile( StringSink sink; dumpString(*res.data, sink); auto hash = hashString(htSHA256, *res.data); - ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name)); - info.narHash = hashString(htSHA256, *sink.s); - info.narSize = sink.s->size(); - info.ca = FixedOutputHash { + FixedOutputHash foh { .method = FileIngestionMethod::Flat, .hash = hash, }; + ValidPathInfo info(store->makeFixedOutputPath(name, foh)); + info.narHash = hashString(htSHA256, *sink.s); + info.narSize = sink.s->size(); + info.ca = std::move(foh); auto source = StringSource { *sink.s }; store->addToStore(info, source, NoRepair, NoCheckSigs); storePath = std::move(info.path); diff --git a/src/libfetchers/tree-info.cc b/src/libfetchers/tree-info.cc index b2d8cfc8d03d..39642352969e 100644 --- a/src/libfetchers/tree-info.cc +++ b/src/libfetchers/tree-info.cc @@ -8,7 +8,10 @@ namespace nix::fetchers { StorePath TreeInfo::computeStorePath(Store & store) const { assert(narHash); - return store.makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "source"); + return store.makeFixedOutputPath("source", FixedOutputHash { + .method = FileIngestionMethod::Recursive, + .hash = narHash, + }); } } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 7167ec900760..56f7e1003242 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -347,7 +347,10 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath h = hashString(hashAlgo, s); } - ValidPathInfo info(makeFixedOutputPath(method, h, name)); + ValidPathInfo info(makeFixedOutputPath(name, FixedOutputHash { + .method = method, + .hash = h, + })); auto source = StringSource { *sink.s }; addToStore(info, source, repair, CheckSigs, nullptr); diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 1474f51745be..821921be90d4 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3733,7 +3733,10 @@ void DerivationGoal::registerOutputs() ? hashPath(*i.second.hash->hash.type, actualPath).first : hashFile(*i.second.hash->hash.type, actualPath); - auto dest = worker.store.makeFixedOutputPath(i.second.hash->method, h2, i.second.path.name()); + auto dest = worker.store.makeFixedOutputPath(i.second.path.name(), FixedOutputHash { + .method = i.second.hash->method, + .hash = h2, + }); if (i.second.hash->hash != h2) { diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 3d753836f8ac..ebd1763f522d 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -17,13 +17,6 @@ std::string makeFileIngestionPrefix(const FileIngestionMethod m) { } } -std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) -{ - return "fixed:" - + makeFileIngestionPrefix(method) - + hash.to_string(Base32, true); -} - // FIXME Put this somewhere? template struct overloaded : Ts... { using Ts::operator()...; }; template overloaded(Ts...) -> overloaded; @@ -31,10 +24,13 @@ template overloaded(Ts...) -> overloaded; std::string renderContentAddress(ContentAddress ca) { return std::visit(overloaded { [](TextHash th) { - return "text:" + th.hash.to_string(Base32, true); + return "text:" + + th.hash.to_string(Base32, true); }, [](FixedOutputHash fsh) { - return makeFixedOutputCA(fsh.method, fsh.hash); + return "fixed:" + + makeFileIngestionPrefix(fsh.method) + + fsh.hash.to_string(Base32, true); } }, ca); } @@ -51,7 +47,6 @@ ContentAddress parseContentAddress(std::string_view rawCa) { } return TextHash { hash }; } else if (prefix == "fixed") { - // This has to be an inverse of makeFixedOutputCA auto methodAndHash = rawCa.substr(prefixSeparator+1, string::npos); if (methodAndHash.substr(0,2) == "r:") { std::string_view hashRaw = methodAndHash.substr(2,string::npos); diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index ba4797f5b00e..1c58fa7355c8 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -41,10 +41,6 @@ typedef std::variant< ingested. */ std::string makeFileIngestionPrefix(const FileIngestionMethod m); -/* Compute the content-addressability assertion (ValidPathInfo::ca) - for paths created by makeFixedOutputPath() / addToStore(). */ -std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash); - std::string renderContentAddress(ContentAddress ca); std::string renderContentAddress(std::optional ca); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c266ee5fd61f..05d1f88acfda 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -562,10 +562,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath)); check( - makeFixedOutputPath( - out->second.hash->method, - out->second.hash->hash, - drvName), + makeFixedOutputPath(drvName, *out->second.hash), out->second.path, "out"); } @@ -1050,7 +1047,10 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam { Hash h = hashString(hashAlgo, dump); - auto dstPath = makeFixedOutputPath(method, h, name); + auto dstPath = makeFixedOutputPath(name, FixedOutputHash { + .method = method, + .hash = h, + }); addTempRoot(dstPath); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index d8a03bc3b4fb..92fc02aa96b8 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -173,21 +173,20 @@ static std::string makeType( StorePath Store::makeFixedOutputPath( - FileIngestionMethod method, - const Hash & hash, std::string_view name, + const FixedOutputHash & foh, const StorePathSet & references, bool hasSelfReference) const { - if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) { - return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); + if (*foh.hash.type == htSHA256 && foh.method == FileIngestionMethod::Recursive) { + return makeStorePath(makeType(*this, "source", references, hasSelfReference), foh.hash, name); } else { assert(references.empty()); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" - + makeFileIngestionPrefix(method) - + hash.to_string(Base16, true) + ":"), + + makeFileIngestionPrefix(foh.method) + + foh.hash.to_string(Base16, true) + ":"), name); } } @@ -205,7 +204,7 @@ StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress return makeTextPath(name, th.hash, references); }, [&](FixedOutputHash fsh) { - return makeFixedOutputPath(fsh.method, fsh.hash, name, references, hasSelfReference); + return makeFixedOutputPath(name, fsh, references, hasSelfReference); } }, ca); } @@ -227,7 +226,10 @@ std::pair Store::computeStorePathForPath(std::string_view name, Hash h = method == FileIngestionMethod::Recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); - return std::make_pair(makeFixedOutputPath(method, h, name), h); + return std::make_pair(makeFixedOutputPath(name, FixedOutputHash { + .method = method, + .hash = h, + }), h); } @@ -835,7 +837,7 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const return store.makeTextPath(path.name(), th.hash, references); }, [&](FixedOutputHash fsh) { - return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), references, hasSelfReference); + return store.makeFixedOutputPath(path.name(), fsh, references, hasSelfReference); } }, *ca); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 33e607ed37ff..b3a9cff367e3 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -250,15 +250,20 @@ public: StorePath makeOutputPath(const string & id, const Hash & hash, std::string_view name) const; - StorePath makeFixedOutputPath(FileIngestionMethod method, - const Hash & hash, std::string_view name, + StorePath makeFixedOutputPath( + std::string_view name, + const FixedOutputHash & hash, const StorePathSet & references = {}, bool hasSelfReference = false) const; - StorePath makeTextPath(std::string_view name, const Hash & hash, + StorePath makeTextPath( + std::string_view name, + const Hash & hash, const StorePathSet & references = {}) const; - StorePath makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, + StorePath makeFixedOutputPathFromCA( + std::string_view name, + ContentAddress ca, const StorePathSet & references = {}, bool hasSelfReference = false) const; diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 40b05a2f39ef..64f2f956a8e1 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -157,8 +157,11 @@ static int _main(int argc, char * * argv) std::optional storePath; if (args.size() == 2) { expectedHash = Hash(args[1], ht); - const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - storePath = store->makeFixedOutputPath(recursive, expectedHash, name); + const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + storePath = store->makeFixedOutputPath(name, FixedOutputHash { + .method = method, + .hash = expectedHash, + }); if (store->isValidPath(*storePath)) hash = expectedHash; else @@ -207,15 +210,18 @@ static int _main(int argc, char * * argv) if (expectedHash != Hash(ht) && expectedHash != hash) throw Error("hash mismatch for '%1%'", uri); - const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; /* Copy the file to the Nix store. FIXME: if RemoteStore implemented addToStoreFromDump() and downloadFile() supported a sink, we could stream the download directly into the Nix store. */ - storePath = store->addToStore(name, tmpFile, recursive, ht); + storePath = store->addToStore(name, tmpFile, method, ht); - assert(*storePath == store->makeFixedOutputPath(recursive, hash, name)); + assert(*storePath == store->makeFixedOutputPath(name, FixedOutputHash { + .method = method, + .hash = expectedHash, + })); } stopProgressBar(); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index c4ca89c8558f..43f83b4d2a51 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -208,7 +208,10 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) string hash = *i++; string name = *i++; - cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash(hash, hashAlgo), name))); + cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputHash { + .method = recursive, + .hash = Hash { hash, hashAlgo }, + }))); } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 48a4722aec34..fce7d3510ff3 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -67,7 +67,10 @@ struct CmdAddToStore : MixDryRun, StoreCommand } } - ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, hash, *namePart)); + ValidPathInfo info(store->makeFixedOutputPath(*namePart, FixedOutputHash { + .method = ingestionMethod, + .hash = hash, + })); info.narHash = narHash; info.narSize = sink.s->size(); info.ca = std::optional { FixedOutputHash { diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 5267948eeb9b..47c6f1f3e77f 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -77,7 +77,10 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON auto narHash = hashModuloSink.finish().first; - ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference)); + ValidPathInfo info(store->makeFixedOutputPath(path.name(), FixedOutputHash { + .method = FileIngestionMethod::Recursive, + .hash = narHash, + }, references, hasSelfReference)); info.references = std::move(references); info.hasSelfReference = std::move(hasSelfReference); info.narHash = narHash; From 69fe58c90155cdbf1082aa0c96d1f47c750b925e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jun 2020 20:14:36 +0000 Subject: [PATCH 02/31] Use `PathReferences` in a few more places --- src/libstore/path-info.hh | 16 +++++++---- src/libstore/store-api.cc | 43 ++++++++++++++++++----------- src/libstore/store-api.hh | 6 ++-- src/nix/make-content-addressable.cc | 39 ++++++++++++-------------- 4 files changed, 56 insertions(+), 48 deletions(-) diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 27efe5ae93fa..f1d57c85325d 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -26,9 +26,9 @@ struct PathReferences std::set references; bool hasSelfReference = false; - /* Functions to view references + hasSelfReference as one set, mainly for - compatibility's sake. */ - StorePathSet referencesPossiblyToSelf(const Ref & self) const; + /* Functions to view references + hasSelfReference as one set, mainly for + compatibility's sake. */ + StorePathSet referencesPossiblyToSelf(const Ref & self) const; void insertReferencePossiblyToSelf(const Ref & self, Ref && ref); void setReferencesPossiblyToSelf(const Ref & self, std::set && refs); }; @@ -116,9 +116,9 @@ struct ValidPathInfo : PathReferences /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; - /* Functions to view references + hasSelfReference as one set, mainly for - compatibility's sake. */ - StorePathSet referencesPossiblyToSelf() const; + /* Functions to view references + hasSelfReference as one set, mainly for + compatibility's sake. */ + StorePathSet referencesPossiblyToSelf() const; void insertReferencePossiblyToSelf(StorePath && ref); void setReferencesPossiblyToSelf(StorePathSet && refs); @@ -139,6 +139,10 @@ struct ValidPathInfo : PathReferences ValidPathInfo(StorePath && path) : path(std::move(path)) { }; ValidPathInfo(const StorePath & path) : path(path) { }; + ValidPathInfo(const Store & store, + std::string_view name, + ContentAddress && ca, PathReferences && refs); + virtual ~ValidPathInfo() { } }; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 92fc02aa96b8..9a645ec7da7a 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -160,14 +160,13 @@ StorePath Store::makeOutputPath(const string & id, static std::string makeType( const Store & store, string && type, - const StorePathSet & references, - bool hasSelfReference = false) + const PathReferences & references) { - for (auto & i : references) { + for (auto & i : references.references) { type += ":"; type += store.printStorePath(i); } - if (hasSelfReference) type += ":self"; + if (references.hasSelfReference) type += ":self"; return std::move(type); } @@ -175,13 +174,13 @@ static std::string makeType( StorePath Store::makeFixedOutputPath( std::string_view name, const FixedOutputHash & foh, - const StorePathSet & references, - bool hasSelfReference) const + const PathReferences & references) const { if (*foh.hash.type == htSHA256 && foh.method == FileIngestionMethod::Recursive) { - return makeStorePath(makeType(*this, "source", references, hasSelfReference), foh.hash, name); + return makeStorePath(makeType(*this, "source", references), foh.hash, name); } else { - assert(references.empty()); + assert(references.references.empty()); + assert(!references.hasSelfReference); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" @@ -196,15 +195,16 @@ template struct overloaded : Ts... { using Ts::operator()...; }; template overloaded(Ts...) -> overloaded; StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, - const StorePathSet & references, bool hasSelfReference) const + const PathReferences & references) const { // New template return std::visit(overloaded { [&](TextHash th) { - return makeTextPath(name, th.hash, references); + assert(!references.hasSelfReference); + return makeTextPath(name, th.hash, references.references); }, [&](FixedOutputHash fsh) { - return makeFixedOutputPath(name, fsh, references, hasSelfReference); + return makeFixedOutputPath(name, fsh, references); } }, ca); } @@ -216,7 +216,7 @@ StorePath Store::makeTextPath(std::string_view name, const Hash & hash, /* Stuff the references (if any) into the type. This is a bit hacky, but we can't put them in `s' since that would be ambiguous. */ - return makeStorePath(makeType(*this, "text", references), hash, name); + return makeStorePath(makeType(*this, "text", PathReferences { references }), hash, name); } @@ -796,17 +796,17 @@ string showPaths(const PathSet & paths) StorePathSet ValidPathInfo::referencesPossiblyToSelf() const { - return PathReferences::referencesPossiblyToSelf(path); + return PathReferences::referencesPossiblyToSelf(path); } void ValidPathInfo::insertReferencePossiblyToSelf(StorePath && ref) { - return PathReferences::insertReferencePossiblyToSelf(path, std::move(ref)); + return PathReferences::insertReferencePossiblyToSelf(path, std::move(ref)); } void ValidPathInfo::setReferencesPossiblyToSelf(StorePathSet && refs) { - return PathReferences::setReferencesPossiblyToSelf(path, std::move(refs)); + return PathReferences::setReferencesPossiblyToSelf(path, std::move(refs)); } std::string ValidPathInfo::fingerprint(const Store & store) const @@ -837,7 +837,7 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const return store.makeTextPath(path.name(), th.hash, references); }, [&](FixedOutputHash fsh) { - return store.makeFixedOutputPath(path.name(), fsh, references, hasSelfReference); + return store.makeFixedOutputPath(path.name(), fsh, *this); } }, *ca); @@ -877,6 +877,17 @@ Strings ValidPathInfo::shortRefs() const } +ValidPathInfo::ValidPathInfo( + const Store & store, + std::string_view name, + ContentAddress && ca, + PathReferences && refs) + : PathReferences(std::move(refs)) + , path(store.makeFixedOutputPathFromCA(name, ca, *this)) + , ca(std::move(ca)) +{ +} + } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index b3a9cff367e3..eaff7477c8cf 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -253,8 +253,7 @@ public: StorePath makeFixedOutputPath( std::string_view name, const FixedOutputHash & hash, - const StorePathSet & references = {}, - bool hasSelfReference = false) const; + const PathReferences & references = {}) const; StorePath makeTextPath( std::string_view name, @@ -264,8 +263,7 @@ public: StorePath makeFixedOutputPathFromCA( std::string_view name, ContentAddress ca, - const StorePathSet & references = {}, - bool hasSelfReference = false) const; + const PathReferences & references = {}) const; /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 47c6f1f3e77f..f5fbca8e7249 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -55,19 +55,15 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON StringMap rewrites; - StorePathSet references; - bool hasSelfReference = false; + PathReferences refs; + refs.hasSelfReference = oldInfo->hasSelfReference; for (auto & ref : oldInfo->references) { - if (ref == path) - hasSelfReference = true; - else { - auto i = remappings.find(ref); - auto replacement = i != remappings.end() ? i->second : ref; - // FIXME: warn about unremapped paths? - if (replacement != ref) - rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement)); - references.insert(std::move(replacement)); - } + auto i = remappings.find(ref); + auto replacement = i != remappings.end() ? i->second : ref; + // FIXME: warn about unremapped paths? + if (replacement != ref) + rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement)); + refs.references.insert(std::move(replacement)); } *sink.s = rewriteStrings(*sink.s, rewrites); @@ -77,18 +73,17 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON auto narHash = hashModuloSink.finish().first; - ValidPathInfo info(store->makeFixedOutputPath(path.name(), FixedOutputHash { - .method = FileIngestionMethod::Recursive, - .hash = narHash, - }, references, hasSelfReference)); - info.references = std::move(references); - info.hasSelfReference = std::move(hasSelfReference); + ValidPathInfo info { + *store, + path.name(), + FixedOutputHash { + .method = FileIngestionMethod::Recursive, + .hash = narHash, + }, + std::move(refs), + }; info.narHash = narHash; info.narSize = sink.s->size(); - info.ca = FixedOutputHash { - .method = FileIngestionMethod::Recursive, - .hash = info.narHash, - }; if (!json) printInfo("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); From 783e262cecf4cb845e1f238b034805c8e4702ed3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jun 2020 22:50:57 +0000 Subject: [PATCH 03/31] CA: Distingish Full vs Mini content addresses --- src/libexpr/primops.cc | 14 +-- src/libexpr/value-to-xml.hh | 2 +- src/libfetchers/tarball.cc | 15 ++- src/libfetchers/tree-info.cc | 7 +- src/libstore/binary-cache-store.cc | 7 +- src/libstore/build.cc | 40 ++++--- src/libstore/content-address.cc | 16 +-- src/libstore/content-address.hh | 85 ++++++++++++++- src/libstore/daemon.cc | 19 ++-- src/libstore/legacy-ssh-store.cc | 4 +- src/libstore/local-store.cc | 99 ++++++++++-------- src/libstore/local-store.hh | 3 +- src/libstore/misc.cc | 18 +++- src/libstore/nar-info-disk-cache.cc | 4 +- src/libstore/nar-info.cc | 4 +- src/libstore/path-info.hh | 48 +-------- src/libstore/path.hh | 3 - src/libstore/remote-store.cc | 59 ++++++----- src/libstore/remote-store.hh | 3 +- src/libstore/store-api.cc | 126 +++++++++++++---------- src/libstore/store-api.hh | 20 ++-- src/libstore/worker-protocol.hh | 4 +- src/libutil/fmt.hh | 2 +- src/nix-prefetch-url/nix-prefetch-url.cc | 14 +-- src/nix-store/nix-store.cc | 11 +- src/nix/add-to-store.cc | 19 ++-- src/nix/make-content-addressable.cc | 18 ++-- src/nix/path-info.cc | 2 +- 28 files changed, 380 insertions(+), 286 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index ea85ad2ba272..3e220386b91f 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -772,9 +772,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * std::optional ht = parseHashTypeOpt(outputHashAlgo); Hash h = newHashAllowEmpty(*outputHash, ht); - auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputHash { - .method = ingestionMethod, - .hash = h + auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputInfo { + ingestionMethod, + h, + {}, }); if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", DerivationOutput { @@ -1156,9 +1157,10 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con std::optional expectedStorePath; if (expectedHash) - expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputHash { - .method = method, - .hash = expectedHash, + expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo { + method, + expectedHash, + {}, }); Path dstPath; if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { diff --git a/src/libexpr/value-to-xml.hh b/src/libexpr/value-to-xml.hh index 97657327edba..02ccd714a908 100644 --- a/src/libexpr/value-to-xml.hh +++ b/src/libexpr/value-to-xml.hh @@ -10,5 +10,5 @@ namespace nix { void printValueAsXML(EvalState & state, bool strict, bool location, Value & v, std::ostream & out, PathSet & context); - + } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 7f9b4d434ea9..197ad642853f 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -67,14 +67,19 @@ DownloadFileResult downloadFile( StringSink sink; dumpString(*res.data, sink); auto hash = hashString(htSHA256, *res.data); - FixedOutputHash foh { - .method = FileIngestionMethod::Flat, - .hash = hash, + ValidPathInfo info { + *store, + FullContentAddress { + .name = name, + .info = FixedOutputInfo { + FileIngestionMethod::Flat, + hash, + {}, + }, + }, }; - ValidPathInfo info(store->makeFixedOutputPath(name, foh)); info.narHash = hashString(htSHA256, *sink.s); info.narSize = sink.s->size(); - info.ca = std::move(foh); auto source = StringSource { *sink.s }; store->addToStore(info, source, NoRepair, NoCheckSigs); storePath = std::move(info.path); diff --git a/src/libfetchers/tree-info.cc b/src/libfetchers/tree-info.cc index 39642352969e..f77b95dea0e0 100644 --- a/src/libfetchers/tree-info.cc +++ b/src/libfetchers/tree-info.cc @@ -8,9 +8,10 @@ namespace nix::fetchers { StorePath TreeInfo::computeStorePath(Store & store) const { assert(narHash); - return store.makeFixedOutputPath("source", FixedOutputHash { - .method = FileIngestionMethod::Recursive, - .hash = narHash, + return store.makeFixedOutputPath("source", FixedOutputInfo { + FileIngestionMethod::Recursive, + narHash, + {}, }); } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 56f7e1003242..8c9bfccb7bfa 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -347,9 +347,10 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath h = hashString(hashAlgo, s); } - ValidPathInfo info(makeFixedOutputPath(name, FixedOutputHash { - .method = method, - .hash = h, + ValidPathInfo info(makeFixedOutputPath(name, FixedOutputInfo { + method, + h, + {}, })); auto source = StringSource { *sink.s }; diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 821921be90d4..f58724f3a10a 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -297,7 +297,7 @@ class Worker GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); std::shared_ptr makeBasicDerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal); - GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); /* Remove a dead goal. */ void removeGoal(GoalPtr goal); @@ -3714,7 +3714,7 @@ void DerivationGoal::registerOutputs() /* Check that fixed-output derivations produced the right outputs (i.e., the content hash should match the specified hash). */ - std::optional ca; + std::optional ca; if (fixedOutput) { @@ -3733,9 +3733,10 @@ void DerivationGoal::registerOutputs() ? hashPath(*i.second.hash->hash.type, actualPath).first : hashFile(*i.second.hash->hash.type, actualPath); - auto dest = worker.store.makeFixedOutputPath(i.second.path.name(), FixedOutputHash { - .method = i.second.hash->method, - .hash = h2, + auto dest = worker.store.makeFixedOutputPath(i.second.path.name(), FixedOutputInfo { + i.second.hash->method, + h2, + {}, // TODO references }); if (i.second.hash->hash != h2) { @@ -3767,9 +3768,13 @@ void DerivationGoal::registerOutputs() else assert(worker.store.parseStorePath(path) == dest); - ca = FixedOutputHash { - .method = i.second.hash->method, - .hash = h2, + ca = FullContentAddress { + .name = std::string { i.second.path.name() }, + .info = FixedOutputInfo { + i.second.hash->method, + h2, + {}, + }, }; } @@ -3834,13 +3839,14 @@ void DerivationGoal::registerOutputs() worker.markContentsGood(worker.store.parseStorePath(path)); } - ValidPathInfo info(worker.store.parseStorePath(path)); + auto info = ca + ? ValidPathInfo { worker.store, FullContentAddress { *ca } } + : ValidPathInfo { worker.store.parseStorePath(path) }; info.narHash = hash.first; info.narSize = hash.second; info.setReferencesPossiblyToSelf(std::move(references)); info.deriver = drvPath; info.ultimate = true; - info.ca = ca; worker.store.signPathInfo(info); if (!info.references.empty()) { @@ -4268,6 +4274,7 @@ class SubstitutionGoal : public Goal private: /* The store path that should be realised through a substitute. */ + // TODO std::variant storePath; StorePath storePath; /* The remaining substituters. */ @@ -4304,10 +4311,11 @@ class SubstitutionGoal : public Goal GoalState state; /* Content address for recomputing store path */ - std::optional ca; + // TODO delete once `storePath` is variant. + std::optional ca; public: - SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); ~SubstitutionGoal(); void timedOut(Error && ex) override { abort(); }; @@ -4337,7 +4345,7 @@ class SubstitutionGoal : public Goal }; -SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) +SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) : Goal(worker) , storePath(storePath) , repair(repair) @@ -4418,7 +4426,7 @@ void SubstitutionGoal::tryNext() auto subPath = storePath; if (ca) { - subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); + subPath = sub->makeFixedOutputPathFromCA(*ca); if (sub->storeDir == worker.store.storeDir) assert(subPath == storePath); } @@ -4545,7 +4553,7 @@ void SubstitutionGoal::tryToRun() auto subPath = storePath; if (ca) { - subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); + subPath = sub->makeFixedOutputPathFromCA(*ca); if (sub->storeDir == worker.store.storeDir) assert(subPath == storePath); } @@ -4684,7 +4692,7 @@ std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath } -GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) +GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) { GoalPtr goal = substitutionGoals[path].lock(); // FIXME if (!goal) { diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index ebd1763f522d..477b93c5bad6 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -21,7 +21,7 @@ std::string makeFileIngestionPrefix(const FileIngestionMethod m) { template struct overloaded : Ts... { using Ts::operator()...; }; template overloaded(Ts...) -> overloaded; -std::string renderContentAddress(ContentAddress ca) { +std::string renderMiniContentAddress(MiniContentAddress ca) { return std::visit(overloaded { [](TextHash th) { return "text:" @@ -35,7 +35,7 @@ std::string renderContentAddress(ContentAddress ca) { }, ca); } -ContentAddress parseContentAddress(std::string_view rawCa) { +MiniContentAddress parseMiniContentAddress(std::string_view rawCa) { auto prefixSeparator = rawCa.find(':'); if (prefixSeparator != string::npos) { auto prefix = string(rawCa, 0, prefixSeparator); @@ -43,7 +43,7 @@ ContentAddress parseContentAddress(std::string_view rawCa) { auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos); Hash hash = Hash(string(hashTypeAndHash)); if (*hash.type != htSHA256) { - throw Error("parseContentAddress: the text hash should have type SHA256"); + throw Error("parseMiniContentAddress: the text hash should have type SHA256"); } return TextHash { hash }; } else if (prefix == "fixed") { @@ -62,19 +62,19 @@ ContentAddress parseContentAddress(std::string_view rawCa) { }; } } else { - throw Error("parseContentAddress: format not recognized; has to be text or fixed"); + throw Error("parseMiniContentAddress: format not recognized; has to be text or fixed"); } } else { throw Error("Not a content address because it lacks an appropriate prefix"); } }; -std::optional parseContentAddressOpt(std::string_view rawCaOpt) { - return rawCaOpt == "" ? std::optional {} : parseContentAddress(rawCaOpt); +std::optional parseMiniContentAddressOpt(std::string_view rawCaOpt) { + return rawCaOpt == "" ? std::optional {} : parseMiniContentAddress(rawCaOpt); }; -std::string renderContentAddress(std::optional ca) { - return ca ? renderContentAddress(*ca) : ""; +std::string renderMiniContentAddress(std::optional ca) { + return ca ? renderMiniContentAddress(*ca) : ""; } } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 1c58fa7355c8..b10943da2282 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -2,9 +2,14 @@ #include #include "hash.hh" +#include "path.hh" namespace nix { +/* + * Mini content address + */ + enum struct FileIngestionMethod : uint8_t { Flat = false, Recursive = true @@ -35,18 +40,88 @@ struct FixedOutputHash { typedef std::variant< TextHash, // for paths computed by makeTextPath() / addTextToStore FixedOutputHash // for path computed by makeFixedOutputPath -> ContentAddress; +> MiniContentAddress; /* Compute the prefix to the hash algorithm which indicates how the files were ingested. */ std::string makeFileIngestionPrefix(const FileIngestionMethod m); -std::string renderContentAddress(ContentAddress ca); +std::string renderMiniContentAddress(MiniContentAddress ca); + +std::string renderMiniContentAddress(std::optional ca); + +MiniContentAddress parseMiniContentAddress(std::string_view rawCa); + +std::optional parseMiniContentAddressOpt(std::string_view rawCaOpt); + +/* + * References set + */ + +template +struct PathReferences +{ + std::set references; + bool hasSelfReference = false; + + /* Functions to view references + hasSelfReference as one set, mainly for + compatibility's sake. */ + StorePathSet referencesPossiblyToSelf(const Ref & self) const; + void insertReferencePossiblyToSelf(const Ref & self, Ref && ref); + void setReferencesPossiblyToSelf(const Ref & self, std::set && refs); +}; + +template +StorePathSet PathReferences::referencesPossiblyToSelf(const Ref & self) const +{ + StorePathSet references { references }; + if (hasSelfReference) + references.insert(self); + return references; +} + +template +void PathReferences::insertReferencePossiblyToSelf(const Ref & self, Ref && ref) +{ + if (ref == self) + hasSelfReference = true; + else + references.insert(std::move(ref)); +} -std::string renderContentAddress(std::optional ca); +template +void PathReferences::setReferencesPossiblyToSelf(const Ref & self, std::set && refs) +{ + if (refs.count(self)) + hasSelfReference = true; + refs.erase(self); -ContentAddress parseContentAddress(std::string_view rawCa); + references = refs; +} + +/* + * Full content address + * + * See the schema for store paths in store-api.cc + */ -std::optional parseContentAddressOpt(std::string_view rawCaOpt); +// This matches the additional info that we need for makeTextPath +struct TextInfo : TextHash { + // References for the paths, self references disallowed + StorePathSet references; +}; + +struct FixedOutputInfo : FixedOutputHash { + // References for the paths + PathReferences references; +}; + +struct FullContentAddress { + std::string name; + std::variant< + TextInfo, + FixedOutputInfo + > info; +}; } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index d4d68e8801c4..66ce8c2e26ae 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -593,7 +593,7 @@ static void performOp(TunnelLogger * logger, ref store, auto path = store->parseStorePath(readString(from)); logger->startWork(); SubstitutablePathInfos infos; - store->querySubstitutablePathInfos({{path, std::nullopt}}, infos); + store->querySubstitutablePathInfos({path}, {}, infos); logger->stopWork(); auto i = infos.find(path); if (i == infos.end()) @@ -610,15 +610,12 @@ static void performOp(TunnelLogger * logger, ref store, case wopQuerySubstitutablePathInfos: { SubstitutablePathInfos infos; - StorePathCAMap pathsMap = {}; - if (GET_PROTOCOL_MINOR(clientVersion) < 22) { - auto paths = readStorePaths(*store, from); - for (auto & path : paths) - pathsMap.emplace(path, std::nullopt); - } else - pathsMap = readStorePathCAMap(*store, from); + auto paths = readStorePaths(*store, from); + std::set caPaths; + if (GET_PROTOCOL_MINOR(clientVersion) > 22) { + caPaths = readFullCaSet(*store, from); logger->startWork(); - store->querySubstitutablePathInfos(pathsMap, infos); + store->querySubstitutablePathInfos(paths, caPaths, infos); logger->stopWork(); to << infos.size(); for (auto & i : infos) { @@ -658,7 +655,7 @@ static void performOp(TunnelLogger * logger, ref store, if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { to << info->ultimate << info->sigs - << renderContentAddress(info->ca); + << renderMiniContentAddress(info->ca); } } else { assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); @@ -716,7 +713,7 @@ static void performOp(TunnelLogger * logger, ref store, info.setReferencesPossiblyToSelf(readStorePaths(*store, from)); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); - info.ca = parseContentAddressOpt(readString(from)); + info.ca = parseMiniContentAddressOpt(readString(from)); from >> repair >> dontCheckSigs; if (!trusted && dontCheckSigs) dontCheckSigs = false; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index f01e642a0356..e33aa46342be 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -113,7 +113,7 @@ struct LegacySSHStore : public Store if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) { auto s = readString(conn->from); info->narHash = s.empty() ? Hash() : Hash(s); - info->ca = parseContentAddressOpt(readString(conn->from)); + info->ca = parseMiniContentAddressOpt(readString(conn->from)); info->sigs = readStrings(conn->from); } @@ -145,7 +145,7 @@ struct LegacySSHStore : public Store << info.narSize << info.ultimate << info.sigs - << renderContentAddress(info.ca); + << renderMiniContentAddress(info.ca); try { copyNAR(source, conn->to); } catch (...) { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 05d1f88acfda..b56bab201879 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -562,7 +562,10 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath)); check( - makeFixedOutputPath(drvName, *out->second.hash), + makeFixedOutputPath(drvName, FixedOutputInfo { + *out->second.hash, + {}, + }), out->second.path, "out"); } @@ -589,7 +592,7 @@ uint64_t LocalStore::addValidPath(State & state, (info.narSize, info.narSize != 0) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) - (renderContentAddress(info.ca), (bool) info.ca) + (renderMiniContentAddress(info.ca), (bool) info.ca) .exec(); uint64_t id = sqlite3_last_insert_rowid(state.db); @@ -663,7 +666,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, if (s) info->sigs = tokenizeString(s, " "); s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 7); - if (s) info->ca = parseContentAddressOpt(s); + if (s) info->ca = parseMiniContentAddressOpt(s); /* Get the references. */ auto useQueryReferences(state->stmtQueryReferences.use()(info->id)); @@ -688,7 +691,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) (info.narHash.to_string(Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) - (renderContentAddress(info.ca), (bool) info.ca) + (renderMiniContentAddress(info.ca), (bool) info.ca) (printStorePath(info.path)) .exec(); } @@ -842,46 +845,53 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) } -void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) +void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, const std::set & caPaths, SubstitutablePathInfos & infos) { if (!settings.useSubstitutes) return; + + auto query = [&](auto & sub, const StorePath & localPath, const StorePath & subPath) { + debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath)); + try { + auto info = sub->queryPathInfo(subPath); + + if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty())) + return; + + auto narInfo = std::dynamic_pointer_cast( + std::shared_ptr(info)); + infos.insert_or_assign(localPath, SubstitutablePathInfo { + info->references, + info->hasSelfReference, + info->deriver, + narInfo ? narInfo->fileSize : 0, + info->narSize, + }); + } catch (InvalidPath &) { + } catch (SubstituterDisabled &) { + } catch (Error & e) { + if (settings.tryFallback) + logError(e.info()); + else + throw; + } + }; + for (auto & sub : getDefaultSubstituters()) { for (auto & path : paths) { - auto subPath(path.first); - - // recompute store path so that we can use a different store root - if (path.second) { - subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second); - if (sub->storeDir == storeDir) - assert(subPath == path.first); - if (subPath != path.first) - debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri()); - } else if (sub->storeDir != storeDir) continue; - - debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath)); - try { - auto info = sub->queryPathInfo(subPath); - - if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty())) - continue; - - auto narInfo = std::dynamic_pointer_cast( - std::shared_ptr(info)); - infos.insert_or_assign(path.first, SubstitutablePathInfo { - info->references, - info->hasSelfReference, - info->deriver, - narInfo ? narInfo->fileSize : 0, - info->narSize, - }); - } catch (InvalidPath &) { - } catch (SubstituterDisabled &) { - } catch (Error & e) { - if (settings.tryFallback) - logError(e.info()); - else - throw; - } + if (sub->storeDir != storeDir) continue; + query(sub, path, path); + } + for (auto & ca : caPaths) { + // TODO Deal with references: either disallow, or require the + // store path lengths be the same and rewrite strings. + auto localPath = makeFixedOutputPathFromCA(ca); + auto subPath = sub->makeFixedOutputPathFromCA(ca); + if (sub->storeDir == storeDir) + assert(localPath == subPath); + if (localPath != subPath) + // TODO print CA too + debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(localPath), sub->printStorePath(subPath), sub->getUri()); + query(sub, localPath, subPath); } } } @@ -1047,9 +1057,10 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam { Hash h = hashString(hashAlgo, dump); - auto dstPath = makeFixedOutputPath(name, FixedOutputHash { - .method = method, - .hash = h, + auto dstPath = makeFixedOutputPath(name, FixedOutputInfo { + method, + h, + {}, }); addTempRoot(dstPath); @@ -1127,7 +1138,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { auto hash = hashString(htSHA256, s); - auto dstPath = makeTextPath(name, hash, references); + auto dstPath = makeTextPath(name, TextInfo { hash, references }); addTempRoot(dstPath); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index ab9fbfbe40c2..660d7a08fe83 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -139,7 +139,8 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; - void querySubstitutablePathInfos(const StorePathCAMap & paths, + void querySubstitutablePathInfos(const StorePathSet & paths, + const std::set & caPaths, SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & source, diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index d1448aa342be..1097dfdd3cfe 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -107,11 +107,15 @@ void Store::computeFSClosure(const StorePath & startPath, } -std::optional getDerivationCA(const BasicDerivation & drv) +std::optional getDerivationCA(const BasicDerivation & drv) { auto out = drv.outputs.find("out"); - if (out != drv.outputs.end()) - return out->second.hash; + if (out != drv.outputs.end() && out->second.hash) { + return FullContentAddress { + .name = std::string { out->second.path.name() }, + .info = FixedOutputInfo { *out->second.hash, {} }, + }; + } return std::nullopt; } @@ -164,7 +168,11 @@ void Store::queryMissing(const std::vector & targets, auto outPath = parseStorePath(outPathS); SubstitutablePathInfos infos; - querySubstitutablePathInfos({{outPath, getDerivationCA(*drv)}}, infos); + auto caOpt = getDerivationCA(*drv); + if (caOpt) + querySubstitutablePathInfos({}, { *std::move(caOpt) }, infos); + else + querySubstitutablePathInfos({outPath}, {}, infos); if (infos.empty()) { drvState_->lock()->done = true; @@ -221,7 +229,7 @@ void Store::queryMissing(const std::vector & targets, if (isValidPath(path.path)) return; SubstitutablePathInfos infos; - querySubstitutablePathInfos({{path.path, std::nullopt}}, infos); + querySubstitutablePathInfos({path.path}, {}, infos); if (infos.empty()) { auto state(state_.lock()); diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index c543f6ea297d..13924cc88c9d 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -203,7 +203,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString(queryNAR.getStr(10), " ")) narInfo->sigs.insert(sig); - narInfo->ca = parseContentAddressOpt(queryNAR.getStr(11)); + narInfo->ca = parseMiniContentAddressOpt(queryNAR.getStr(11)); return {oValid, narInfo}; }); @@ -237,7 +237,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) (concatStringsSep(" ", info->sigs)) - (renderContentAddress(info->ca)) + (renderMiniContentAddress(info->ca)) (time(0)).exec(); } else { diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index ef04bc859649..7005b26ec741 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -70,7 +70,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "CA") { if (ca) corrupt(); // FIXME: allow blank ca or require skipping field? - ca = parseContentAddressOpt(value); + ca = parseMiniContentAddressOpt(value); } pos = eol + 1; @@ -107,7 +107,7 @@ std::string NarInfo::to_string(const Store & store) const res += "Sig: " + sig + "\n"; if (ca) - res += "CA: " + renderContentAddress(*ca) + "\n"; + res += "CA: " + renderMiniContentAddress(*ca) + "\n"; return res; } diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index f1d57c85325d..f0d0e6b61724 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -20,47 +20,6 @@ namespace nix { class Store; -template -struct PathReferences -{ - std::set references; - bool hasSelfReference = false; - - /* Functions to view references + hasSelfReference as one set, mainly for - compatibility's sake. */ - StorePathSet referencesPossiblyToSelf(const Ref & self) const; - void insertReferencePossiblyToSelf(const Ref & self, Ref && ref); - void setReferencesPossiblyToSelf(const Ref & self, std::set && refs); -}; - -template -StorePathSet PathReferences::referencesPossiblyToSelf(const Ref & self) const -{ - StorePathSet references { references }; - if (hasSelfReference) - references.insert(self); - return references; -} - -template -void PathReferences::insertReferencePossiblyToSelf(const Ref & self, Ref && ref) -{ - if (ref == self) - hasSelfReference = true; - else - references.insert(std::move(ref)); -} - -template -void PathReferences::setReferencesPossiblyToSelf(const Ref & self, std::set && refs) -{ - if (refs.count(self)) - hasSelfReference = true; - refs.erase(self); - - references = refs; -} - struct ValidPathInfo : PathReferences { StorePath path; @@ -92,7 +51,7 @@ struct ValidPathInfo : PathReferences and the store path would be computed from the name component, ‘narHash’ and ‘references’. However, we support many types of content addresses. */ - std::optional ca; + std::optional ca; bool operator == (const ValidPathInfo & i) const { @@ -113,6 +72,8 @@ struct ValidPathInfo : PathReferences void sign(const Store & store, const SecretKey & secretKey); + std::optional fullContentAddressOpt() const; + /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; @@ -140,8 +101,7 @@ struct ValidPathInfo : PathReferences ValidPathInfo(const StorePath & path) : path(path) { }; ValidPathInfo(const Store & store, - std::string_view name, - ContentAddress && ca, PathReferences && refs); + FullContentAddress && ca); virtual ~ValidPathInfo() { } }; diff --git a/src/libstore/path.hh b/src/libstore/path.hh index a07186323adb..c82ed9f266f0 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -1,6 +1,5 @@ #pragma once -#include "content-address.hh" #include "types.hh" namespace nix { @@ -63,8 +62,6 @@ public: typedef std::set StorePathSet; typedef std::vector StorePaths; -typedef std::map> StorePathCAMap; - /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 4a9be4445714..7e8f4922e0da 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -38,22 +38,24 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths out << store.printStorePath(i); } -StorePathCAMap readStorePathCAMap(const Store & store, Source & from) +std::set readFullCaSet(const Store & store, Source & from) { - StorePathCAMap paths; - auto count = readNum(from); - while (count--) - paths.insert_or_assign(store.parseStorePath(readString(from)), parseContentAddressOpt(readString(from))); + std::set paths; + // TODO + // auto count = readNum(from); + // while (count--) + // paths.insert_or_assign(store.parseStorePath(readString(from)), parseMiniContentAddressOpt(readString(from))); return paths; } -void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths) +void writeFullCaSet(const Store & store, Sink & out, const std::set & paths) { - out << paths.size(); - for (auto & i : paths) { - out << store.printStorePath(i.first); - out << renderContentAddress(i.second); - } + // TODO + //out << paths.size(); + //for (auto & i : paths) { + // out << store.printStorePath(i.first); + // out << renderMiniContentAddress(i.second); + //} } @@ -326,43 +328,48 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) } -void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, SubstitutablePathInfos & infos) +void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, const std::set & caPaths, SubstitutablePathInfos & infos) { - if (pathsMap.empty()) return; + if (paths.empty() && caPaths.empty()) return; auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + auto combine = [&]() { + std::set combined { paths }; + for (auto & ca : caPaths) + combined.insert(makeFixedOutputPathFromCA(ca)); + return combined; + }; - for (auto & i : pathsMap) { + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + for (auto & path : combine()) { SubstitutablePathInfo info; - conn->to << wopQuerySubstitutablePathInfo << printStorePath(i.first); + conn->to << wopQuerySubstitutablePathInfo << printStorePath(path); conn.processStderr(); unsigned int reply = readInt(conn->from); if (reply == 0) continue; auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.setReferencesPossiblyToSelf(i.first, readStorePaths(*this, conn->from)); + info.setReferencesPossiblyToSelf(path, readStorePaths(*this, conn->from)); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); - infos.insert_or_assign(i.first, std::move(info)); + infos.insert_or_assign(path, std::move(info)); } } else { conn->to << wopQuerySubstitutablePathInfos; if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 22) { - StorePathSet paths; - for (auto & path : pathsMap) - paths.insert(path.first); + writeStorePaths(*this, conn->to, combine()); + } else { writeStorePaths(*this, conn->to, paths); - } else - writeStorePathCAMap(*this, conn->to, pathsMap); + writeFullCaSet(*this, conn->to, caPaths); + } conn.processStderr(); size_t count = readNum(conn->from); for (size_t n = 0; n < count; n++) { - auto path = parseStorePath(readString(conn->from)); + auto path = parseStorePath(readString(conn->from)); SubstitutablePathInfo & info { infos[path] }; auto deriver = readString(conn->from); if (deriver != "") @@ -405,7 +412,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { conn->from >> info->ultimate; info->sigs = readStrings(conn->from); - info->ca = parseContentAddressOpt(readString(conn->from)); + info->ca = parseMiniContentAddressOpt(readString(conn->from)); } } callback(std::move(info)); @@ -489,7 +496,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << info.narHash.to_string(Base16, false); writeStorePaths(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize - << info.ultimate << info.sigs << renderContentAddress(info.ca) + << info.ultimate << info.sigs << renderMiniContentAddress(info.ca) << repair << !checkSigs; bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21; if (!tunnel) copyNAR(source, conn->to); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 07ed2b51a1df..7a3acd6bfa65 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -55,7 +55,8 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; - void querySubstitutablePathInfos(const StorePathCAMap & paths, + void querySubstitutablePathInfos(const StorePathSet & paths, + const std::set & caPaths, SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & nar, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 9a645ec7da7a..91767e55f329 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -157,6 +157,9 @@ StorePath Store::makeOutputPath(const string & id, } +/* Stuff the references (if any) into the type. This is a bit + hacky, but we can't put them in `s' since that would be + ambiguous. */ static std::string makeType( const Store & store, string && type, @@ -171,52 +174,48 @@ static std::string makeType( } -StorePath Store::makeFixedOutputPath( - std::string_view name, - const FixedOutputHash & foh, - const PathReferences & references) const +StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { - if (*foh.hash.type == htSHA256 && foh.method == FileIngestionMethod::Recursive) { - return makeStorePath(makeType(*this, "source", references), foh.hash, name); + if (*info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) { + return makeStorePath(makeType(*this, "source", info.references), info.hash, name); } else { - assert(references.references.empty()); - assert(!references.hasSelfReference); + assert(info.references.references.size() == 0); + assert(!info.references.hasSelfReference); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" - + makeFileIngestionPrefix(foh.method) - + foh.hash.to_string(Base16, true) + ":"), + + makeFileIngestionPrefix(info.method) + + info.hash.to_string(Base16, true) + ":"), name); } } + +StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const +{ + assert(info.hash.type == htSHA256); + return makeStorePath( + makeType(*this, "text", PathReferences { info.references }), + info.hash, + name); +} + + // FIXME Put this somewhere? template struct overloaded : Ts... { using Ts::operator()...; }; template overloaded(Ts...) -> overloaded; -StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, - const PathReferences & references) const +StorePath Store::makeFixedOutputPathFromCA(const FullContentAddress & info) const { // New template return std::visit(overloaded { - [&](TextHash th) { - assert(!references.hasSelfReference); - return makeTextPath(name, th.hash, references.references); + [&](TextInfo ti) { + return makeTextPath(info.name, ti); }, - [&](FixedOutputHash fsh) { - return makeFixedOutputPath(name, fsh, references); + [&](FixedOutputInfo foi) { + return makeFixedOutputPath(info.name, foi); } - }, ca); -} - -StorePath Store::makeTextPath(std::string_view name, const Hash & hash, - const StorePathSet & references) const -{ - assert(hash.type == htSHA256); - /* Stuff the references (if any) into the type. This is a bit - hacky, but we can't put them in `s' since that would be - ambiguous. */ - return makeStorePath(makeType(*this, "text", PathReferences { references }), hash, name); + }, info.info); } @@ -226,17 +225,14 @@ std::pair Store::computeStorePathForPath(std::string_view name, Hash h = method == FileIngestionMethod::Recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); - return std::make_pair(makeFixedOutputPath(name, FixedOutputHash { - .method = method, - .hash = h, - }), h); + return std::make_pair(makeFixedOutputPath(name, FixedOutputInfo { method, h, {} }), h); } StorePath Store::computeStorePathForText(const string & name, const string & s, const StorePathSet & references) const { - return makeTextPath(name, hashString(htSHA256, s), references); + return makeTextPath(name, TextInfo { hashString(htSHA256, s), references }); } @@ -491,7 +487,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store } if (info->ca) - jsonPath.attr("ca", renderContentAddress(info->ca)); + jsonPath.attr("ca", renderMiniContentAddress(info->ca)); std::pair closureSizes; @@ -603,7 +599,7 @@ void copyStorePath(ref srcStore, ref dstStore, // recompute store path on the chance dstStore does it differently if (info->ca && info->references.empty()) { auto info2 = make_ref(*info); - info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), *info->ca); + info2->path = dstStore->makeFixedOutputPathFromCA(*info->fullContentAddressOpt()); if (dstStore->storeDir == srcStore->storeDir) assert(info->path == info2->path); info = info2; @@ -681,7 +677,7 @@ std::map copyPaths(ref srcStore, ref dstStor auto info = srcStore->queryPathInfo(storePath); auto storePathForDst = storePath; if (info->ca && info->references.empty() && !info->hasSelfReference) { - storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca); + storePathForDst = dstStore->makeFixedOutputPathFromCA(*info->fullContentAddressOpt()); if (dstStore->storeDir == srcStore->storeDir) assert(storePathForDst == storePath); if (storePathForDst != storePath) @@ -709,7 +705,7 @@ std::map copyPaths(ref srcStore, ref dstStor auto storePathForDst = storePath; if (info->ca && info->references.empty() && !info->hasSelfReference) { - storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca); + storePathForDst = dstStore->makeFixedOutputPathFromCA(*info->fullContentAddressOpt()); if (dstStore->storeDir == srcStore->storeDir) assert(storePathForDst == storePath); if (storePathForDst != storePath) @@ -827,19 +823,37 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } +std::optional ValidPathInfo::fullContentAddressOpt() const +{ + if (! ca) + return std::nullopt; + + return FullContentAddress { + .name = std::string { path.name() }, + .info = std::visit(overloaded { + [&](TextHash th) { + TextInfo info { th }; + assert(!hasSelfReference); + info.references = references; + return std::variant { info }; + }, + [&](FixedOutputHash foh) { + FixedOutputInfo info { foh }; + info.references = static_cast>(*this); + return std::variant { info }; + }, + }, *ca), + }; +} + bool ValidPathInfo::isContentAddressed(const Store & store) const { - if (! ca) return false; + auto fullCaOpt = fullContentAddressOpt(); - auto caPath = std::visit(overloaded { - [&](TextHash th) { - assert(!hasSelfReference); - return store.makeTextPath(path.name(), th.hash, references); - }, - [&](FixedOutputHash fsh) { - return store.makeFixedOutputPath(path.name(), fsh, *this); - } - }, *ca); + if (! fullCaOpt) + return false; + + auto caPath = store.makeFixedOutputPathFromCA(*fullCaOpt); bool res = caPath == path; @@ -879,13 +893,19 @@ Strings ValidPathInfo::shortRefs() const ValidPathInfo::ValidPathInfo( const Store & store, - std::string_view name, - ContentAddress && ca, - PathReferences && refs) - : PathReferences(std::move(refs)) - , path(store.makeFixedOutputPathFromCA(name, ca, *this)) - , ca(std::move(ca)) + FullContentAddress && info) + : path(store.makeFixedOutputPathFromCA(info)) { + std::visit(overloaded { + [this](TextInfo ti) { + this->references = ti.references; + this->ca = TextHash { std::move(ti) }; + }, + [this](FixedOutputInfo foi) { + *(static_cast *>(this)) = foi.references; + this->ca = FixedOutputHash { (FixedOutputHash) std::move(foi) }; + }, + }, std::move(info.info)); } } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index eaff7477c8cf..2e1e6d64354f 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -250,20 +250,11 @@ public: StorePath makeOutputPath(const string & id, const Hash & hash, std::string_view name) const; - StorePath makeFixedOutputPath( - std::string_view name, - const FixedOutputHash & hash, - const PathReferences & references = {}) const; + StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const; - StorePath makeTextPath( - std::string_view name, - const Hash & hash, - const StorePathSet & references = {}) const; + StorePath makeTextPath(std::string_view name, const TextInfo & info) const; - StorePath makeFixedOutputPathFromCA( - std::string_view name, - ContentAddress ca, - const PathReferences & references = {}) const; + StorePath makeFixedOutputPathFromCA(const FullContentAddress & info) const; /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store @@ -352,7 +343,8 @@ public: sizes) of a map of paths to their optional ca values. If a path does not have substitute info, it's omitted from the resulting ‘infos’ map. */ - virtual void querySubstitutablePathInfos(const StorePathCAMap & paths, + virtual void querySubstitutablePathInfos(const StorePathSet & paths, + const std::set & caPaths, SubstitutablePathInfos & infos) { return; }; /* Import a path into the store. */ @@ -754,6 +746,6 @@ std::optional decodeValidPathInfo( /* Split URI into protocol+hierarchy part and its parameter set. */ std::pair splitUriAndParams(const std::string & uri); -std::optional getDerivationCA(const BasicDerivation & drv); +std::optional getDerivationCA(const BasicDerivation & drv); } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 5a3e33c08948..14706a79c2df 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -69,9 +69,9 @@ template T readStorePaths(const Store & store, Source & from); void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths); -StorePathCAMap readStorePathCAMap(const Store & store, Source & from); +std::set readFullCaSet(const Store & store, Source & from); -void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths); +void writeFullCaSet(const Store & store, Sink & out, const std::set & paths); } diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index a39de041fc86..0480b357d078 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -104,7 +104,7 @@ class hintformat public: hintformat(const string &format) :fmt(format) { - fmt.exceptions(boost::io::all_error_bits ^ + fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit ^ boost::io::too_few_args_bit); } diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 64f2f956a8e1..c6042d384784 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -158,9 +158,10 @@ static int _main(int argc, char * * argv) if (args.size() == 2) { expectedHash = Hash(args[1], ht); const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - storePath = store->makeFixedOutputPath(name, FixedOutputHash { - .method = method, - .hash = expectedHash, + storePath = store->makeFixedOutputPath(name, FixedOutputInfo { + method, + expectedHash, + {}, }); if (store->isValidPath(*storePath)) hash = expectedHash; @@ -218,9 +219,10 @@ static int _main(int argc, char * * argv) into the Nix store. */ storePath = store->addToStore(name, tmpFile, method, ht); - assert(*storePath == store->makeFixedOutputPath(name, FixedOutputHash { - .method = method, - .hash = expectedHash, + assert(*storePath == store->makeFixedOutputPath(name, FixedOutputInfo { + method, + expectedHash, + {} })); } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 43f83b4d2a51..15f28382e224 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -208,9 +208,10 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) string hash = *i++; string name = *i++; - cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputHash { - .method = recursive, - .hash = Hash { hash, hashAlgo }, + cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputInfo { + recursive, + Hash { hash, hashAlgo }, + {}, }))); } @@ -867,7 +868,7 @@ static void opServe(Strings opFlags, Strings opArgs) out << info->narSize // downloadSize << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << renderContentAddress(info->ca) << info->sigs; + out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << renderMiniContentAddress(info->ca) << info->sigs; } catch (InvalidPath &) { } } @@ -955,7 +956,7 @@ static void opServe(Strings opFlags, Strings opArgs) info.setReferencesPossiblyToSelf(readStorePaths(*store, in)); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); - info.ca = parseContentAddressOpt(readString(in)); + info.ca = parseMiniContentAddressOpt(readString(in)); if (info.narSize == 0) throw Error("narInfo is too old and missing the narSize field"); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index fce7d3510ff3..38303270358b 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -67,16 +67,19 @@ struct CmdAddToStore : MixDryRun, StoreCommand } } - ValidPathInfo info(store->makeFixedOutputPath(*namePart, FixedOutputHash { - .method = ingestionMethod, - .hash = hash, - })); + ValidPathInfo info { + *store, + FullContentAddress { + .name = *namePart, + .info = FixedOutputInfo { + std::move(ingestionMethod), + std::move(hash), + {}, + }, + }, + }; info.narHash = narHash; info.narSize = sink.s->size(); - info.ca = std::optional { FixedOutputHash { - .method = ingestionMethod, - .hash = hash, - } }; if (!dryRun) { auto source = StringSource { *sink.s }; diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index f5fbca8e7249..313bca94e90c 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -6,9 +6,9 @@ using namespace nix; -struct CmdMakeContentAddressable : StorePathsCommand, MixJSON +struct CmdMakeMiniContentAddressable : StorePathsCommand, MixJSON { - CmdMakeContentAddressable() + CmdMakeMiniContentAddressable() { realiseMode = Build; } @@ -75,12 +75,14 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON ValidPathInfo info { *store, - path.name(), - FixedOutputHash { - .method = FileIngestionMethod::Recursive, - .hash = narHash, + FullContentAddress { + .name = std::string { path.name() }, + .info = FixedOutputInfo { + FileIngestionMethod::Recursive, + narHash, + std::move(refs), + }, }, - std::move(refs), }; info.narHash = narHash; info.narSize = sink.s->size(); @@ -104,4 +106,4 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON } }; -static auto r1 = registerCommand("make-content-addressable"); +static auto r1 = registerCommand("make-content-addressable"); diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index b89a44f83cc6..6a52513e62bc 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -115,7 +115,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON std::cout << '\t'; Strings ss; if (info->ultimate) ss.push_back("ultimate"); - if (info->ca) ss.push_back("ca:" + renderContentAddress(*info->ca)); + if (info->ca) ss.push_back("ca:" + renderMiniContentAddress(*info->ca)); for (auto & sig : info->sigs) ss.push_back(sig); std::cout << concatStringsSep(" ", ss); } From c8cd66fb66d63bf885c6ba7e450c52cec0d37dea Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 26 Jun 2020 17:22:28 -0400 Subject: [PATCH 04/31] Fix syntax --- src/libstore/daemon.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 66ce8c2e26ae..08d6c707ee68 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -612,7 +612,7 @@ static void performOp(TunnelLogger * logger, ref store, SubstitutablePathInfos infos; auto paths = readStorePaths(*store, from); std::set caPaths; - if (GET_PROTOCOL_MINOR(clientVersion) > 22) { + if (GET_PROTOCOL_MINOR(clientVersion) > 22) caPaths = readFullCaSet(*store, from); logger->startWork(); store->querySubstitutablePathInfos(paths, caPaths, infos); From bff0743b29bcc999b5ed30cfab3eb1b4ab082409 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 26 Jun 2020 17:30:32 -0400 Subject: [PATCH 05/31] Add order for FullContentAddress to put in a set, we need an ordering --- src/libstore/content-address.hh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index b10943da2282..8b56a81ac864 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -122,6 +122,12 @@ struct FullContentAddress { TextInfo, FixedOutputInfo > info; + + bool operator < (const FullContentAddress & other) const + { + return name < other.name; + } + }; } From 1db393fac79dc83f031763af237a1d20f2728301 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 29 Jun 2020 13:14:26 -0400 Subject: [PATCH 06/31] Fix referencesPossiblyToSelf --- src/libstore/content-address.hh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 8b56a81ac864..08b323729da4 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -74,10 +74,10 @@ struct PathReferences template StorePathSet PathReferences::referencesPossiblyToSelf(const Ref & self) const { - StorePathSet references { references }; + StorePathSet refs { references }; if (hasSelfReference) - references.insert(self); - return references; + refs.insert(self); + return refs; } template From ebba7f39d0ac48921131d393f5550f5f4bc8efaa Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 29 Jun 2020 19:58:31 +0000 Subject: [PATCH 07/31] Fix nar info parsing --- src/libstore/nar-info.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 7005b26ec741..03ddcb700755 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -57,7 +57,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & auto refs = tokenizeString(value, " "); if (!references.empty()) corrupt(); for (auto & r : refs) - references.insert(StorePath(r)); + insertReferencePossiblyToSelf(StorePath(r)); } else if (name == "Deriver") { if (value != "unknown-deriver") From 83f743b11d2048c464c160edc9b5bd8832ff25ba Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 30 Jun 2020 22:17:01 +0000 Subject: [PATCH 08/31] Fix perl bindings --- perl/lib/Nix/Store.xs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 945ed49c7543..3af628e1bd0b 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -287,7 +287,11 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) try { Hash h(hash, parseHashType(algo)); auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - auto path = store()->makeFixedOutputPath(method, h, name); + auto path = store()->makeFixedOutputPath(name, FixedOutputInfo { + method, + h, + {}, + }); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); From 5a8e8fa8b008d8db4e86d046fcbd450d6ea08c79 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Jul 2020 01:16:29 +0000 Subject: [PATCH 09/31] Use some named designators again with braces for base class --- perl/lib/Nix/Store.xs | 6 ++++-- src/libexpr/primops.cc | 12 ++++++++---- src/libfetchers/tarball.cc | 6 ++++-- src/libfetchers/tree-info.cc | 6 ++++-- src/libstore/binary-cache-store.cc | 6 ++++-- src/libstore/build.cc | 12 ++++++++---- src/libstore/local-store.cc | 6 ++++-- src/libstore/store-api.cc | 14 ++++++++++++-- src/nix-prefetch-url/nix-prefetch-url.cc | 12 ++++++++---- src/nix-store/nix-store.cc | 6 ++++-- src/nix/add-to-store.cc | 6 ++++-- src/nix/make-content-addressable.cc | 6 ++++-- 12 files changed, 68 insertions(+), 30 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 3af628e1bd0b..4671b9d54396 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -288,8 +288,10 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) Hash h(hash, parseHashType(algo)); auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; auto path = store()->makeFixedOutputPath(name, FixedOutputInfo { - method, - h, + { + .method = method, + .hash = h, + }, {}, }); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 3e220386b91f..9ed4dbb652a1 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -773,8 +773,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * Hash h = newHashAllowEmpty(*outputHash, ht); auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputInfo { - ingestionMethod, - h, + { + .method = ingestionMethod, + .hash = h, + }, {}, }); if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); @@ -1158,8 +1160,10 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con std::optional expectedStorePath; if (expectedHash) expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo { - method, - expectedHash, + { + .method = method, + .hash = expectedHash, + }, {}, }); Path dstPath; diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 197ad642853f..b508fc2bbb6d 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -72,8 +72,10 @@ DownloadFileResult downloadFile( FullContentAddress { .name = name, .info = FixedOutputInfo { - FileIngestionMethod::Flat, - hash, + { + .method = FileIngestionMethod::Flat, + .hash = hash, + }, {}, }, }, diff --git a/src/libfetchers/tree-info.cc b/src/libfetchers/tree-info.cc index f77b95dea0e0..71c95a2f6141 100644 --- a/src/libfetchers/tree-info.cc +++ b/src/libfetchers/tree-info.cc @@ -9,8 +9,10 @@ StorePath TreeInfo::computeStorePath(Store & store) const { assert(narHash); return store.makeFixedOutputPath("source", FixedOutputInfo { - FileIngestionMethod::Recursive, - narHash, + { + .method = FileIngestionMethod::Recursive, + .hash = narHash, + }, {}, }); } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 8c9bfccb7bfa..3512859f6842 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -348,8 +348,10 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath } ValidPathInfo info(makeFixedOutputPath(name, FixedOutputInfo { - method, - h, + { + .method = method, + .hash = h, + }, {}, })); diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 70d429a8242a..78482a3bf7b1 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3737,8 +3737,10 @@ void DerivationGoal::registerOutputs() : hashFile(*i.second.hash->hash.type, actualPath); auto dest = worker.store.makeFixedOutputPath(i.second.path.name(), FixedOutputInfo { - i.second.hash->method, - h2, + { + .method = i.second.hash->method, + .hash = h2, + }, {}, // TODO references }); @@ -3774,8 +3776,10 @@ void DerivationGoal::registerOutputs() ca = FullContentAddress { .name = std::string { i.second.path.name() }, .info = FixedOutputInfo { - i.second.hash->method, - h2, + { + .method = i.second.hash->method, + .hash = h2, + }, {}, }, }; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b56bab201879..aa9fba27a71f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1058,8 +1058,10 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam Hash h = hashString(hashAlgo, dump); auto dstPath = makeFixedOutputPath(name, FixedOutputInfo { - method, - h, + { + .method = method, + .hash = h, + }, {}, }); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 91767e55f329..74b25e4aa817 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -225,14 +225,24 @@ std::pair Store::computeStorePathForPath(std::string_view name, Hash h = method == FileIngestionMethod::Recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); - return std::make_pair(makeFixedOutputPath(name, FixedOutputInfo { method, h, {} }), h); + FixedOutputInfo caInfo { + { + .method = method, + .hash = h, + }, + {}, + }; + return std::make_pair(makeFixedOutputPath(name, caInfo), h); } StorePath Store::computeStorePathForText(const string & name, const string & s, const StorePathSet & references) const { - return makeTextPath(name, TextInfo { hashString(htSHA256, s), references }); + return makeTextPath(name, TextInfo { + { .hash = hashString(htSHA256, s) }, + references, + }); } diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index c6042d384784..a83437a3efe3 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -159,8 +159,10 @@ static int _main(int argc, char * * argv) expectedHash = Hash(args[1], ht); const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; storePath = store->makeFixedOutputPath(name, FixedOutputInfo { - method, - expectedHash, + { + .method = method, + .hash = expectedHash, + }, {}, }); if (store->isValidPath(*storePath)) @@ -220,8 +222,10 @@ static int _main(int argc, char * * argv) storePath = store->addToStore(name, tmpFile, method, ht); assert(*storePath == store->makeFixedOutputPath(name, FixedOutputInfo { - method, - expectedHash, + { + .method = method, + .hash = expectedHash, + }, {} })); } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 15f28382e224..ee031606014d 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -209,8 +209,10 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) string name = *i++; cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputInfo { - recursive, - Hash { hash, hashAlgo }, + { + .method = recursive, + .hash = Hash { hash, hashAlgo }, + }, {}, }))); } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 38303270358b..c59cdcc1d6af 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -72,8 +72,10 @@ struct CmdAddToStore : MixDryRun, StoreCommand FullContentAddress { .name = *namePart, .info = FixedOutputInfo { - std::move(ingestionMethod), - std::move(hash), + { + .method = std::move(ingestionMethod), + .hash = std::move(hash), + }, {}, }, }, diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 313bca94e90c..1ab77c1d2bbc 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -78,8 +78,10 @@ struct CmdMakeMiniContentAddressable : StorePathsCommand, MixJSON FullContentAddress { .name = std::string { path.name() }, .info = FixedOutputInfo { - FileIngestionMethod::Recursive, - narHash, + { + .method = FileIngestionMethod::Recursive, + .hash = narHash, + }, std::move(refs), }, }, From 7dacd78b0b29622458e1a8c694ccfd2751204d44 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Jul 2020 05:10:17 +0000 Subject: [PATCH 10/31] Fix more initialization warnings --- src/libstore/local-store.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index aa9fba27a71f..5e68e5ab344b 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -860,8 +860,7 @@ void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, const s auto narInfo = std::dynamic_pointer_cast( std::shared_ptr(info)); infos.insert_or_assign(localPath, SubstitutablePathInfo { - info->references, - info->hasSelfReference, + { *info }, info->deriver, narInfo ? narInfo->fileSize : 0, info->narSize, @@ -1140,7 +1139,10 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { auto hash = hashString(htSHA256, s); - auto dstPath = makeTextPath(name, TextInfo { hash, references }); + auto dstPath = makeTextPath(name, TextInfo { + { .hash = hash }, + references, + }); addTempRoot(dstPath); From b2e0ec825d6d6ef739ecd8d4920fbdd4d7c31059 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Jul 2020 05:10:34 +0000 Subject: [PATCH 11/31] Try to avoid include problem in perl --- src/libstore/path.hh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libstore/path.hh b/src/libstore/path.hh index c82ed9f266f0..c2599cf40770 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -1,5 +1,7 @@ #pragma once +#include + #include "types.hh" namespace nix { From c77f6c999ecb2c640da46882ee22a6f4f5b0fed3 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 1 Jul 2020 17:32:07 -0400 Subject: [PATCH 12/31] Rename FullContentAddress to ContentAddress LegacyContentAddress used for all other usages --- src/libfetchers/tarball.cc | 2 +- src/libstore/build.cc | 18 +++++++++--------- src/libstore/content-address.cc | 16 ++++++++-------- src/libstore/content-address.hh | 14 +++++++------- src/libstore/daemon.cc | 8 ++++---- src/libstore/legacy-ssh-store.cc | 4 ++-- src/libstore/local-store.cc | 8 ++++---- src/libstore/local-store.hh | 2 +- src/libstore/misc.cc | 4 ++-- src/libstore/nar-info-disk-cache.cc | 4 ++-- src/libstore/nar-info.cc | 4 ++-- src/libstore/path-info.hh | 6 +++--- src/libstore/remote-store.cc | 18 +++++++++--------- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.cc | 10 +++++----- src/libstore/store-api.hh | 6 +++--- src/libstore/worker-protocol.hh | 4 ++-- src/nix-store/nix-store.cc | 4 ++-- src/nix/add-to-store.cc | 2 +- src/nix/make-content-addressable.cc | 8 ++++---- src/nix/path-info.cc | 2 +- 21 files changed, 73 insertions(+), 73 deletions(-) diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index b508fc2bbb6d..c9fac531a44a 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -69,7 +69,7 @@ DownloadFileResult downloadFile( auto hash = hashString(htSHA256, *res.data); ValidPathInfo info { *store, - FullContentAddress { + ContentAddress { .name = name, .info = FixedOutputInfo { { diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 78482a3bf7b1..9cd7f92eb008 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -297,7 +297,7 @@ class Worker GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); std::shared_ptr makeBasicDerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal); - GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); /* Remove a dead goal. */ void removeGoal(GoalPtr goal); @@ -3717,7 +3717,7 @@ void DerivationGoal::registerOutputs() /* Check that fixed-output derivations produced the right outputs (i.e., the content hash should match the specified hash). */ - std::optional ca; + std::optional ca; if (fixedOutput) { @@ -3773,7 +3773,7 @@ void DerivationGoal::registerOutputs() else assert(worker.store.parseStorePath(path) == dest); - ca = FullContentAddress { + ca = ContentAddress { .name = std::string { i.second.path.name() }, .info = FixedOutputInfo { { @@ -3847,7 +3847,7 @@ void DerivationGoal::registerOutputs() } auto info = ca - ? ValidPathInfo { worker.store, FullContentAddress { *ca } } + ? ValidPathInfo { worker.store, ContentAddress { *ca } } : ValidPathInfo { worker.store.parseStorePath(path) }; info.narHash = hash.first; info.narSize = hash.second; @@ -4281,7 +4281,7 @@ class SubstitutionGoal : public Goal private: /* The store path that should be realised through a substitute. */ - // TODO std::variant storePath; + // TODO std::variant storePath; StorePath storePath; /* The remaining substituters. */ @@ -4319,10 +4319,10 @@ class SubstitutionGoal : public Goal /* Content address for recomputing store path */ // TODO delete once `storePath` is variant. - std::optional ca; + std::optional ca; public: - SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); ~SubstitutionGoal(); void timedOut(Error && ex) override { abort(); }; @@ -4352,7 +4352,7 @@ class SubstitutionGoal : public Goal }; -SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) +SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) : Goal(worker) , storePath(storePath) , repair(repair) @@ -4699,7 +4699,7 @@ std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath } -GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) +GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) { GoalPtr goal = substitutionGoals[path].lock(); // FIXME if (!goal) { diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 477b93c5bad6..a4d1c2912a2e 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -21,7 +21,7 @@ std::string makeFileIngestionPrefix(const FileIngestionMethod m) { template struct overloaded : Ts... { using Ts::operator()...; }; template overloaded(Ts...) -> overloaded; -std::string renderMiniContentAddress(MiniContentAddress ca) { +std::string renderLegacyContentAddress(LegacyContentAddress ca) { return std::visit(overloaded { [](TextHash th) { return "text:" @@ -35,7 +35,7 @@ std::string renderMiniContentAddress(MiniContentAddress ca) { }, ca); } -MiniContentAddress parseMiniContentAddress(std::string_view rawCa) { +LegacyContentAddress parseLegacyContentAddress(std::string_view rawCa) { auto prefixSeparator = rawCa.find(':'); if (prefixSeparator != string::npos) { auto prefix = string(rawCa, 0, prefixSeparator); @@ -43,7 +43,7 @@ MiniContentAddress parseMiniContentAddress(std::string_view rawCa) { auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos); Hash hash = Hash(string(hashTypeAndHash)); if (*hash.type != htSHA256) { - throw Error("parseMiniContentAddress: the text hash should have type SHA256"); + throw Error("parseLegacyContentAddress: the text hash should have type SHA256"); } return TextHash { hash }; } else if (prefix == "fixed") { @@ -62,19 +62,19 @@ MiniContentAddress parseMiniContentAddress(std::string_view rawCa) { }; } } else { - throw Error("parseMiniContentAddress: format not recognized; has to be text or fixed"); + throw Error("parseLegacyContentAddress: format not recognized; has to be text or fixed"); } } else { throw Error("Not a content address because it lacks an appropriate prefix"); } }; -std::optional parseMiniContentAddressOpt(std::string_view rawCaOpt) { - return rawCaOpt == "" ? std::optional {} : parseMiniContentAddress(rawCaOpt); +std::optional parseLegacyContentAddressOpt(std::string_view rawCaOpt) { + return rawCaOpt == "" ? std::optional {} : parseLegacyContentAddress(rawCaOpt); }; -std::string renderMiniContentAddress(std::optional ca) { - return ca ? renderMiniContentAddress(*ca) : ""; +std::string renderLegacyContentAddress(std::optional ca) { + return ca ? renderLegacyContentAddress(*ca) : ""; } } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 08b323729da4..84ff71a344fb 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -40,19 +40,19 @@ struct FixedOutputHash { typedef std::variant< TextHash, // for paths computed by makeTextPath() / addTextToStore FixedOutputHash // for path computed by makeFixedOutputPath -> MiniContentAddress; +> LegacyContentAddress; /* Compute the prefix to the hash algorithm which indicates how the files were ingested. */ std::string makeFileIngestionPrefix(const FileIngestionMethod m); -std::string renderMiniContentAddress(MiniContentAddress ca); +std::string renderLegacyContentAddress(LegacyContentAddress ca); -std::string renderMiniContentAddress(std::optional ca); +std::string renderLegacyContentAddress(std::optional ca); -MiniContentAddress parseMiniContentAddress(std::string_view rawCa); +LegacyContentAddress parseLegacyContentAddress(std::string_view rawCa); -std::optional parseMiniContentAddressOpt(std::string_view rawCaOpt); +std::optional parseLegacyContentAddressOpt(std::string_view rawCaOpt); /* * References set @@ -116,14 +116,14 @@ struct FixedOutputInfo : FixedOutputHash { PathReferences references; }; -struct FullContentAddress { +struct ContentAddress { std::string name; std::variant< TextInfo, FixedOutputInfo > info; - bool operator < (const FullContentAddress & other) const + bool operator < (const ContentAddress & other) const { return name < other.name; } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 08d6c707ee68..e3c599b17386 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -611,9 +611,9 @@ static void performOp(TunnelLogger * logger, ref store, case wopQuerySubstitutablePathInfos: { SubstitutablePathInfos infos; auto paths = readStorePaths(*store, from); - std::set caPaths; + std::set caPaths; if (GET_PROTOCOL_MINOR(clientVersion) > 22) - caPaths = readFullCaSet(*store, from); + caPaths = readContentAddressSet(*store, from); logger->startWork(); store->querySubstitutablePathInfos(paths, caPaths, infos); logger->stopWork(); @@ -655,7 +655,7 @@ static void performOp(TunnelLogger * logger, ref store, if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { to << info->ultimate << info->sigs - << renderMiniContentAddress(info->ca); + << renderLegacyContentAddress(info->ca); } } else { assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); @@ -713,7 +713,7 @@ static void performOp(TunnelLogger * logger, ref store, info.setReferencesPossiblyToSelf(readStorePaths(*store, from)); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); - info.ca = parseMiniContentAddressOpt(readString(from)); + info.ca = parseLegacyContentAddressOpt(readString(from)); from >> repair >> dontCheckSigs; if (!trusted && dontCheckSigs) dontCheckSigs = false; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index e33aa46342be..a607f1022bca 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -113,7 +113,7 @@ struct LegacySSHStore : public Store if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) { auto s = readString(conn->from); info->narHash = s.empty() ? Hash() : Hash(s); - info->ca = parseMiniContentAddressOpt(readString(conn->from)); + info->ca = parseLegacyContentAddressOpt(readString(conn->from)); info->sigs = readStrings(conn->from); } @@ -145,7 +145,7 @@ struct LegacySSHStore : public Store << info.narSize << info.ultimate << info.sigs - << renderMiniContentAddress(info.ca); + << renderLegacyContentAddress(info.ca); try { copyNAR(source, conn->to); } catch (...) { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 5e68e5ab344b..3086a976c309 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -592,7 +592,7 @@ uint64_t LocalStore::addValidPath(State & state, (info.narSize, info.narSize != 0) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) - (renderMiniContentAddress(info.ca), (bool) info.ca) + (renderLegacyContentAddress(info.ca), (bool) info.ca) .exec(); uint64_t id = sqlite3_last_insert_rowid(state.db); @@ -666,7 +666,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, if (s) info->sigs = tokenizeString(s, " "); s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 7); - if (s) info->ca = parseMiniContentAddressOpt(s); + if (s) info->ca = parseLegacyContentAddressOpt(s); /* Get the references. */ auto useQueryReferences(state->stmtQueryReferences.use()(info->id)); @@ -691,7 +691,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) (info.narHash.to_string(Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) - (renderMiniContentAddress(info.ca), (bool) info.ca) + (renderLegacyContentAddress(info.ca), (bool) info.ca) (printStorePath(info.path)) .exec(); } @@ -845,7 +845,7 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) } -void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, const std::set & caPaths, SubstitutablePathInfos & infos) +void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, const std::set & caPaths, SubstitutablePathInfos & infos) { if (!settings.useSubstitutes) return; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 660d7a08fe83..487a27e50860 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -140,7 +140,7 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; void querySubstitutablePathInfos(const StorePathSet & paths, - const std::set & caPaths, + const std::set & caPaths, SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & source, diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 1097dfdd3cfe..ece38fc7b2d6 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -107,11 +107,11 @@ void Store::computeFSClosure(const StorePath & startPath, } -std::optional getDerivationCA(const BasicDerivation & drv) +std::optional getDerivationCA(const BasicDerivation & drv) { auto out = drv.outputs.find("out"); if (out != drv.outputs.end() && out->second.hash) { - return FullContentAddress { + return ContentAddress { .name = std::string { out->second.path.name() }, .info = FixedOutputInfo { *out->second.hash, {} }, }; diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 13924cc88c9d..51d8d4f4b356 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -203,7 +203,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString(queryNAR.getStr(10), " ")) narInfo->sigs.insert(sig); - narInfo->ca = parseMiniContentAddressOpt(queryNAR.getStr(11)); + narInfo->ca = parseLegacyContentAddressOpt(queryNAR.getStr(11)); return {oValid, narInfo}; }); @@ -237,7 +237,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) (concatStringsSep(" ", info->sigs)) - (renderMiniContentAddress(info->ca)) + (renderLegacyContentAddress(info->ca)) (time(0)).exec(); } else { diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 03ddcb700755..ecec5bad80b2 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -70,7 +70,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "CA") { if (ca) corrupt(); // FIXME: allow blank ca or require skipping field? - ca = parseMiniContentAddressOpt(value); + ca = parseLegacyContentAddressOpt(value); } pos = eol + 1; @@ -107,7 +107,7 @@ std::string NarInfo::to_string(const Store & store) const res += "Sig: " + sig + "\n"; if (ca) - res += "CA: " + renderMiniContentAddress(*ca) + "\n"; + res += "CA: " + renderLegacyContentAddress(*ca) + "\n"; return res; } diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index f0d0e6b61724..c702c60f55c4 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -51,7 +51,7 @@ struct ValidPathInfo : PathReferences and the store path would be computed from the name component, ‘narHash’ and ‘references’. However, we support many types of content addresses. */ - std::optional ca; + std::optional ca; bool operator == (const ValidPathInfo & i) const { @@ -72,7 +72,7 @@ struct ValidPathInfo : PathReferences void sign(const Store & store, const SecretKey & secretKey); - std::optional fullContentAddressOpt() const; + std::optional fullContentAddressOpt() const; /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; @@ -101,7 +101,7 @@ struct ValidPathInfo : PathReferences ValidPathInfo(const StorePath & path) : path(path) { }; ValidPathInfo(const Store & store, - FullContentAddress && ca); + ContentAddress && ca); virtual ~ValidPathInfo() { } }; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 6eb1b002a44e..11bafc1ad53f 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -38,23 +38,23 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths out << store.printStorePath(i); } -std::set readFullCaSet(const Store & store, Source & from) +std::set readContentAddressSet(const Store & store, Source & from) { - std::set paths; + std::set paths; // TODO // auto count = readNum(from); // while (count--) - // paths.insert_or_assign(store.parseStorePath(readString(from)), parseMiniContentAddressOpt(readString(from))); + // paths.insert_or_assign(store.parseStorePath(readString(from)), parseLegacyContentAddressOpt(readString(from))); return paths; } -void writeFullCaSet(const Store & store, Sink & out, const std::set & paths) +void writeContentAddressSet(const Store & store, Sink & out, const std::set & paths) { // TODO //out << paths.size(); //for (auto & i : paths) { // out << store.printStorePath(i.first); - // out << renderMiniContentAddress(i.second); + // out << renderLegacyContentAddress(i.second); //} } @@ -328,7 +328,7 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) } -void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, const std::set & caPaths, SubstitutablePathInfos & infos) +void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, const std::set & caPaths, SubstitutablePathInfos & infos) { if (paths.empty() && caPaths.empty()) return; @@ -364,7 +364,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, const writeStorePaths(*this, conn->to, combine()); } else { writeStorePaths(*this, conn->to, paths); - writeFullCaSet(*this, conn->to, caPaths); + writeContentAddressSet(*this, conn->to, caPaths); } conn.processStderr(); size_t count = readNum(conn->from); @@ -412,7 +412,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { conn->from >> info->ultimate; info->sigs = readStrings(conn->from); - info->ca = parseMiniContentAddressOpt(readString(conn->from)); + info->ca = parseLegacyContentAddressOpt(readString(conn->from)); } } callback(std::move(info)); @@ -496,7 +496,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << info.narHash.to_string(Base16, false); writeStorePaths(*this, conn->to, info.referencesPossiblyToSelf()); conn->to << info.registrationTime << info.narSize - << info.ultimate << info.sigs << renderMiniContentAddress(info.ca) + << info.ultimate << info.sigs << renderLegacyContentAddress(info.ca) << repair << !checkSigs; bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21; if (!tunnel) copyNAR(source, conn->to); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 7a3acd6bfa65..e30c6f2024ec 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -56,7 +56,7 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; void querySubstitutablePathInfos(const StorePathSet & paths, - const std::set & caPaths, + const std::set & caPaths, SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & nar, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 74b25e4aa817..f3bebe3e26e4 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -205,7 +205,7 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons template struct overloaded : Ts... { using Ts::operator()...; }; template overloaded(Ts...) -> overloaded; -StorePath Store::makeFixedOutputPathFromCA(const FullContentAddress & info) const +StorePath Store::makeFixedOutputPathFromCA(const ContentAddress & info) const { // New template return std::visit(overloaded { @@ -497,7 +497,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store } if (info->ca) - jsonPath.attr("ca", renderMiniContentAddress(info->ca)); + jsonPath.attr("ca", renderLegacyContentAddress(info->ca)); std::pair closureSizes; @@ -833,12 +833,12 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } -std::optional ValidPathInfo::fullContentAddressOpt() const +std::optional ValidPathInfo::fullContentAddressOpt() const { if (! ca) return std::nullopt; - return FullContentAddress { + return ContentAddress { .name = std::string { path.name() }, .info = std::visit(overloaded { [&](TextHash th) { @@ -903,7 +903,7 @@ Strings ValidPathInfo::shortRefs() const ValidPathInfo::ValidPathInfo( const Store & store, - FullContentAddress && info) + ContentAddress && info) : path(store.makeFixedOutputPathFromCA(info)) { std::visit(overloaded { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 2e1e6d64354f..f3c6910cc6e2 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -254,7 +254,7 @@ public: StorePath makeTextPath(std::string_view name, const TextInfo & info) const; - StorePath makeFixedOutputPathFromCA(const FullContentAddress & info) const; + StorePath makeFixedOutputPathFromCA(const ContentAddress & info) const; /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store @@ -344,7 +344,7 @@ public: does not have substitute info, it's omitted from the resulting ‘infos’ map. */ virtual void querySubstitutablePathInfos(const StorePathSet & paths, - const std::set & caPaths, + const std::set & caPaths, SubstitutablePathInfos & infos) { return; }; /* Import a path into the store. */ @@ -746,6 +746,6 @@ std::optional decodeValidPathInfo( /* Split URI into protocol+hierarchy part and its parameter set. */ std::pair splitUriAndParams(const std::string & uri); -std::optional getDerivationCA(const BasicDerivation & drv); +std::optional getDerivationCA(const BasicDerivation & drv); } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 14706a79c2df..a692e855cd92 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -69,9 +69,9 @@ template T readStorePaths(const Store & store, Source & from); void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths); -std::set readFullCaSet(const Store & store, Source & from); +std::set readContentAddressSet(const Store & store, Source & from); -void writeFullCaSet(const Store & store, Sink & out, const std::set & paths); +void writeContentAddressSet(const Store & store, Sink & out, const std::set & paths); } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index ee031606014d..b96363e56679 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -870,7 +870,7 @@ static void opServe(Strings opFlags, Strings opArgs) out << info->narSize // downloadSize << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << renderMiniContentAddress(info->ca) << info->sigs; + out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << renderLegacyContentAddress(info->ca) << info->sigs; } catch (InvalidPath &) { } } @@ -958,7 +958,7 @@ static void opServe(Strings opFlags, Strings opArgs) info.setReferencesPossiblyToSelf(readStorePaths(*store, in)); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); - info.ca = parseMiniContentAddressOpt(readString(in)); + info.ca = parseLegacyContentAddressOpt(readString(in)); if (info.narSize == 0) throw Error("narInfo is too old and missing the narSize field"); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index c59cdcc1d6af..1ebe38344f2c 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -69,7 +69,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand ValidPathInfo info { *store, - FullContentAddress { + ContentAddress { .name = *namePart, .info = FixedOutputInfo { { diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 1ab77c1d2bbc..7e3c0ee4e05e 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -6,9 +6,9 @@ using namespace nix; -struct CmdMakeMiniContentAddressable : StorePathsCommand, MixJSON +struct CmdMakeContentAddressable : StorePathsCommand, MixJSON { - CmdMakeMiniContentAddressable() + CmdMakeContentAddressable() { realiseMode = Build; } @@ -75,7 +75,7 @@ struct CmdMakeMiniContentAddressable : StorePathsCommand, MixJSON ValidPathInfo info { *store, - FullContentAddress { + ContentAddress { .name = std::string { path.name() }, .info = FixedOutputInfo { { @@ -108,4 +108,4 @@ struct CmdMakeMiniContentAddressable : StorePathsCommand, MixJSON } }; -static auto r1 = registerCommand("make-content-addressable"); +static auto r1 = registerCommand("make-content-addressable"); diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 6a52513e62bc..126b69635e37 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -115,7 +115,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON std::cout << '\t'; Strings ss; if (info->ultimate) ss.push_back("ultimate"); - if (info->ca) ss.push_back("ca:" + renderMiniContentAddress(*info->ca)); + if (info->ca) ss.push_back("ca:" + renderLegacyContentAddress(*info->ca)); for (auto & sig : info->sigs) ss.push_back(sig); std::cout << concatStringsSep(" ", ss); } From 7271778defe9bad01e6b8e705ab75c047eb24d68 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 5 Jul 2020 21:39:04 +0000 Subject: [PATCH 13/31] See if setting -std=c++17 for PERL helps --- perl/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl/Makefile b/perl/Makefile index 7ddb0cf692cd..259ed7dc3f12 100644 --- a/perl/Makefile +++ b/perl/Makefile @@ -1,6 +1,6 @@ makefiles = local.mk -GLOBAL_CXXFLAGS += -g -Wall +GLOBAL_CXXFLAGS += -g -Wall -std=c++17 -include Makefile.config From ff072964402897cde7ed3cb71930c693dff6ebe6 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Mon, 13 Jul 2020 14:58:19 -0400 Subject: [PATCH 14/31] Rename ContentAddress to StorePathDescriptor --- src/libfetchers/tarball.cc | 2 +- src/libstore/build.cc | 18 +++++++++--------- src/libstore/content-address.hh | 4 ++-- src/libstore/daemon.cc | 2 +- src/libstore/local-store.cc | 2 +- src/libstore/local-store.hh | 2 +- src/libstore/misc.cc | 4 ++-- src/libstore/path-info.hh | 4 ++-- src/libstore/remote-store.cc | 8 ++++---- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.cc | 8 ++++---- src/libstore/store-api.hh | 6 +++--- src/libstore/worker-protocol.hh | 4 ++-- src/nix/add-to-store.cc | 2 +- src/nix/make-content-addressable.cc | 2 +- 15 files changed, 35 insertions(+), 35 deletions(-) diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index c9fac531a44a..7d4d3d6c711a 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -69,7 +69,7 @@ DownloadFileResult downloadFile( auto hash = hashString(htSHA256, *res.data); ValidPathInfo info { *store, - ContentAddress { + StorePathDescriptor { .name = name, .info = FixedOutputInfo { { diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 299887ec1fc0..1df95fb3e002 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -297,7 +297,7 @@ class Worker GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); std::shared_ptr makeBasicDerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal); - GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); /* Remove a dead goal. */ void removeGoal(GoalPtr goal); @@ -3717,7 +3717,7 @@ void DerivationGoal::registerOutputs() /* Check that fixed-output derivations produced the right outputs (i.e., the content hash should match the specified hash). */ - std::optional ca; + std::optional ca; if (fixedOutput) { @@ -3773,7 +3773,7 @@ void DerivationGoal::registerOutputs() else assert(worker.store.parseStorePath(path) == dest); - ca = ContentAddress { + ca = StorePathDescriptor { .name = std::string { i.second.path.name() }, .info = FixedOutputInfo { { @@ -3847,7 +3847,7 @@ void DerivationGoal::registerOutputs() } auto info = ca - ? ValidPathInfo { worker.store, ContentAddress { *ca } } + ? ValidPathInfo { worker.store, StorePathDescriptor { *ca } } : ValidPathInfo { worker.store.parseStorePath(path) }; info.narHash = hash.first; info.narSize = hash.second; @@ -4281,7 +4281,7 @@ class SubstitutionGoal : public Goal private: /* The store path that should be realised through a substitute. */ - // TODO std::variant storePath; + // TODO std::variant storePath; StorePath storePath; /* The remaining substituters. */ @@ -4319,10 +4319,10 @@ class SubstitutionGoal : public Goal /* Content address for recomputing store path */ // TODO delete once `storePath` is variant. - std::optional ca; + std::optional ca; public: - SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); ~SubstitutionGoal(); void timedOut(Error && ex) override { abort(); }; @@ -4352,7 +4352,7 @@ class SubstitutionGoal : public Goal }; -SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) +SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) : Goal(worker) , storePath(storePath) , repair(repair) @@ -4699,7 +4699,7 @@ std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath } -GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) +GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) { GoalPtr goal = substitutionGoals[path].lock(); // FIXME if (!goal) { diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 84ff71a344fb..10a5fcf6657d 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -116,14 +116,14 @@ struct FixedOutputInfo : FixedOutputHash { PathReferences references; }; -struct ContentAddress { +struct StorePathDescriptor { std::string name; std::variant< TextInfo, FixedOutputInfo > info; - bool operator < (const ContentAddress & other) const + bool operator < (const StorePathDescriptor & other) const { return name < other.name; } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index cb0aa282cfbd..10e842d8b74d 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -620,7 +620,7 @@ static void performOp(TunnelLogger * logger, ref store, case wopQuerySubstitutablePathInfos: { SubstitutablePathInfos infos; auto paths = readStorePaths(*store, from); - std::set caPaths; + std::set caPaths; if (GET_PROTOCOL_MINOR(clientVersion) > 22) caPaths = readContentAddressSet(*store, from); logger->startWork(); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c0ac1da03a5b..173e97bed591 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -848,7 +848,7 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) } -void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, const std::set & caPaths, SubstitutablePathInfos & infos) +void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, const std::set & caPaths, SubstitutablePathInfos & infos) { if (!settings.useSubstitutes) return; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index f835c1a9fa0f..a4fbd65d5de3 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -140,7 +140,7 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; void querySubstitutablePathInfos(const StorePathSet & paths, - const std::set & caPaths, + const std::set & caPaths, SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & source, diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index ece38fc7b2d6..13823a933c35 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -107,11 +107,11 @@ void Store::computeFSClosure(const StorePath & startPath, } -std::optional getDerivationCA(const BasicDerivation & drv) +std::optional getDerivationCA(const BasicDerivation & drv) { auto out = drv.outputs.find("out"); if (out != drv.outputs.end() && out->second.hash) { - return ContentAddress { + return StorePathDescriptor { .name = std::string { out->second.path.name() }, .info = FixedOutputInfo { *out->second.hash, {} }, }; diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index c702c60f55c4..febc3e6fe65d 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -72,7 +72,7 @@ struct ValidPathInfo : PathReferences void sign(const Store & store, const SecretKey & secretKey); - std::optional fullContentAddressOpt() const; + std::optional fullContentAddressOpt() const; /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; @@ -101,7 +101,7 @@ struct ValidPathInfo : PathReferences ValidPathInfo(const StorePath & path) : path(path) { }; ValidPathInfo(const Store & store, - ContentAddress && ca); + StorePathDescriptor && ca); virtual ~ValidPathInfo() { } }; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 443c9c45a7b4..27de5c28456c 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -39,9 +39,9 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths out << store.printStorePath(i); } -std::set readContentAddressSet(const Store & store, Source & from) +std::set readContentAddressSet(const Store & store, Source & from) { - std::set paths; + std::set paths; // TODO // auto count = readNum(from); // while (count--) @@ -49,7 +49,7 @@ std::set readContentAddressSet(const Store & store, Source & fro return paths; } -void writeContentAddressSet(const Store & store, Sink & out, const std::set & paths) +void writeContentAddressSet(const Store & store, Sink & out, const std::set & paths) { // TODO //out << paths.size(); @@ -352,7 +352,7 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) } -void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, const std::set & caPaths, SubstitutablePathInfos & infos) +void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, const std::set & caPaths, SubstitutablePathInfos & infos) { if (paths.empty() && caPaths.empty()) return; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index a2da5a0fc4a1..be931dabab06 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -57,7 +57,7 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; void querySubstitutablePathInfos(const StorePathSet & paths, - const std::set & caPaths, + const std::set & caPaths, SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & nar, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 4928a16f6c5f..5f7375abee2e 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -205,7 +205,7 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons template struct overloaded : Ts... { using Ts::operator()...; }; template overloaded(Ts...) -> overloaded; -StorePath Store::makeFixedOutputPathFromCA(const ContentAddress & info) const +StorePath Store::makeFixedOutputPathFromCA(const StorePathDescriptor & info) const { // New template return std::visit(overloaded { @@ -843,12 +843,12 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } -std::optional ValidPathInfo::fullContentAddressOpt() const +std::optional ValidPathInfo::fullContentAddressOpt() const { if (! ca) return std::nullopt; - return ContentAddress { + return StorePathDescriptor { .name = std::string { path.name() }, .info = std::visit(overloaded { [&](TextHash th) { @@ -913,7 +913,7 @@ Strings ValidPathInfo::shortRefs() const ValidPathInfo::ValidPathInfo( const Store & store, - ContentAddress && info) + StorePathDescriptor && info) : path(store.makeFixedOutputPathFromCA(info)) { std::visit(overloaded { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 37b75e730528..6f08151a9bfb 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -254,7 +254,7 @@ public: StorePath makeTextPath(std::string_view name, const TextInfo & info) const; - StorePath makeFixedOutputPathFromCA(const ContentAddress & info) const; + StorePath makeFixedOutputPathFromCA(const StorePathDescriptor & info) const; /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store @@ -347,7 +347,7 @@ public: does not have substitute info, it's omitted from the resulting ‘infos’ map. */ virtual void querySubstitutablePathInfos(const StorePathSet & paths, - const std::set & caPaths, + const std::set & caPaths, SubstitutablePathInfos & infos) { return; }; /* Import a path into the store. */ @@ -749,6 +749,6 @@ std::optional decodeValidPathInfo( /* Split URI into protocol+hierarchy part and its parameter set. */ std::pair splitUriAndParams(const std::string & uri); -std::optional getDerivationCA(const BasicDerivation & drv); +std::optional getDerivationCA(const BasicDerivation & drv); } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 0b6816abb425..cdb9b9be4e62 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -70,9 +70,9 @@ template T readStorePaths(const Store & store, Source & from); void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths); -std::set readContentAddressSet(const Store & store, Source & from); +std::set readContentAddressSet(const Store & store, Source & from); -void writeContentAddressSet(const Store & store, Sink & out, const std::set & paths); +void writeContentAddressSet(const Store & store, Sink & out, const std::set & paths); void writeOutputPathMap(const Store & store, Sink & out, const OutputPathMap & paths); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 1ebe38344f2c..b9aa8dd162fd 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -69,7 +69,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand ValidPathInfo info { *store, - ContentAddress { + StorePathDescriptor { .name = *namePart, .info = FixedOutputInfo { { diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 7e3c0ee4e05e..028c0f1442d1 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -75,7 +75,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON ValidPathInfo info { *store, - ContentAddress { + StorePathDescriptor { .name = std::string { path.name() }, .info = FixedOutputInfo { { From ade0f1a440120475bfe4a985f66d050f6cd01279 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Tue, 14 Jul 2020 10:41:47 -0400 Subject: [PATCH 15/31] Rename functions related to ContentAddress --- src/libstore/daemon.cc | 2 +- src/libstore/path-info.hh | 2 +- src/libstore/remote-store.cc | 6 +++--- src/libstore/store-api.cc | 10 +++++----- src/libstore/worker-protocol.hh | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 10e842d8b74d..5ad7f6ade49c 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -622,7 +622,7 @@ static void performOp(TunnelLogger * logger, ref store, auto paths = readStorePaths(*store, from); std::set caPaths; if (GET_PROTOCOL_MINOR(clientVersion) > 22) - caPaths = readContentAddressSet(*store, from); + caPaths = readStorePathDescriptorSet(*store, from); logger->startWork(); store->querySubstitutablePathInfos(paths, caPaths, infos); logger->stopWork(); diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index febc3e6fe65d..5b2716b769f2 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -72,7 +72,7 @@ struct ValidPathInfo : PathReferences void sign(const Store & store, const SecretKey & secretKey); - std::optional fullContentAddressOpt() const; + std::optional fullStorePathDescriptorOpt() const; /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 27de5c28456c..aee82599bf07 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -39,7 +39,7 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths out << store.printStorePath(i); } -std::set readContentAddressSet(const Store & store, Source & from) +std::set readStorePathDescriptorSet(const Store & store, Source & from) { std::set paths; // TODO @@ -49,7 +49,7 @@ std::set readContentAddressSet(const Store & store, Source return paths; } -void writeContentAddressSet(const Store & store, Sink & out, const std::set & paths) +void writeStorePathDescriptorSet(const Store & store, Sink & out, const std::set & paths) { // TODO //out << paths.size(); @@ -388,7 +388,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, const writeStorePaths(*this, conn->to, combine()); } else { writeStorePaths(*this, conn->to, paths); - writeContentAddressSet(*this, conn->to, caPaths); + writeStorePathDescriptorSet(*this, conn->to, caPaths); } conn.processStderr(); size_t count = readNum(conn->from); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 5f7375abee2e..80d336209a63 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -619,7 +619,7 @@ void copyStorePath(ref srcStore, ref dstStore, // recompute store path on the chance dstStore does it differently if (info->ca && info->references.empty()) { auto info2 = make_ref(*info); - info2->path = dstStore->makeFixedOutputPathFromCA(*info->fullContentAddressOpt()); + info2->path = dstStore->makeFixedOutputPathFromCA(*info->fullStorePathDescriptorOpt()); if (dstStore->storeDir == srcStore->storeDir) assert(info->path == info2->path); info = info2; @@ -697,7 +697,7 @@ std::map copyPaths(ref srcStore, ref dstStor auto info = srcStore->queryPathInfo(storePath); auto storePathForDst = storePath; if (info->ca && info->references.empty() && !info->hasSelfReference) { - storePathForDst = dstStore->makeFixedOutputPathFromCA(*info->fullContentAddressOpt()); + storePathForDst = dstStore->makeFixedOutputPathFromCA(*info->fullStorePathDescriptorOpt()); if (dstStore->storeDir == srcStore->storeDir) assert(storePathForDst == storePath); if (storePathForDst != storePath) @@ -725,7 +725,7 @@ std::map copyPaths(ref srcStore, ref dstStor auto storePathForDst = storePath; if (info->ca && info->references.empty() && !info->hasSelfReference) { - storePathForDst = dstStore->makeFixedOutputPathFromCA(*info->fullContentAddressOpt()); + storePathForDst = dstStore->makeFixedOutputPathFromCA(*info->fullStorePathDescriptorOpt()); if (dstStore->storeDir == srcStore->storeDir) assert(storePathForDst == storePath); if (storePathForDst != storePath) @@ -843,7 +843,7 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } -std::optional ValidPathInfo::fullContentAddressOpt() const +std::optional ValidPathInfo::fullStorePathDescriptorOpt() const { if (! ca) return std::nullopt; @@ -868,7 +868,7 @@ std::optional ValidPathInfo::fullContentAddressOpt() const bool ValidPathInfo::isContentAddressed(const Store & store) const { - auto fullCaOpt = fullContentAddressOpt(); + auto fullCaOpt = fullStorePathDescriptorOpt(); if (! fullCaOpt) return false; diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index cdb9b9be4e62..f42ce125f2ff 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -70,9 +70,9 @@ template T readStorePaths(const Store & store, Source & from); void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths); -std::set readContentAddressSet(const Store & store, Source & from); +std::set readStorePathDescriptorSet(const Store & store, Source & from); -void writeContentAddressSet(const Store & store, Sink & out, const std::set & paths); +void writeStorePathDescriptorSet(const Store & store, Sink & out, const std::set & paths); void writeOutputPathMap(const Store & store, Sink & out, const OutputPathMap & paths); From 08bf6c414157dc14221b760f395c0920149c5095 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Tue, 14 Jul 2020 10:46:18 -0400 Subject: [PATCH 16/31] Rename LegacyContentAddress and related functions --- src/libstore/content-address.cc | 16 ++++++++-------- src/libstore/content-address.hh | 10 +++++----- src/libstore/daemon.cc | 4 ++-- src/libstore/legacy-ssh-store.cc | 4 ++-- src/libstore/local-store.cc | 6 +++--- src/libstore/nar-info-disk-cache.cc | 4 ++-- src/libstore/nar-info.cc | 4 ++-- src/libstore/path-info.hh | 2 +- src/libstore/remote-store.cc | 8 ++++---- src/libstore/store-api.cc | 2 +- src/nix-store/nix-store.cc | 4 ++-- src/nix/path-info.cc | 2 +- 12 files changed, 33 insertions(+), 33 deletions(-) diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index a4d1c2912a2e..ebd1763f522d 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -21,7 +21,7 @@ std::string makeFileIngestionPrefix(const FileIngestionMethod m) { template struct overloaded : Ts... { using Ts::operator()...; }; template overloaded(Ts...) -> overloaded; -std::string renderLegacyContentAddress(LegacyContentAddress ca) { +std::string renderContentAddress(ContentAddress ca) { return std::visit(overloaded { [](TextHash th) { return "text:" @@ -35,7 +35,7 @@ std::string renderLegacyContentAddress(LegacyContentAddress ca) { }, ca); } -LegacyContentAddress parseLegacyContentAddress(std::string_view rawCa) { +ContentAddress parseContentAddress(std::string_view rawCa) { auto prefixSeparator = rawCa.find(':'); if (prefixSeparator != string::npos) { auto prefix = string(rawCa, 0, prefixSeparator); @@ -43,7 +43,7 @@ LegacyContentAddress parseLegacyContentAddress(std::string_view rawCa) { auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos); Hash hash = Hash(string(hashTypeAndHash)); if (*hash.type != htSHA256) { - throw Error("parseLegacyContentAddress: the text hash should have type SHA256"); + throw Error("parseContentAddress: the text hash should have type SHA256"); } return TextHash { hash }; } else if (prefix == "fixed") { @@ -62,19 +62,19 @@ LegacyContentAddress parseLegacyContentAddress(std::string_view rawCa) { }; } } else { - throw Error("parseLegacyContentAddress: format not recognized; has to be text or fixed"); + throw Error("parseContentAddress: format not recognized; has to be text or fixed"); } } else { throw Error("Not a content address because it lacks an appropriate prefix"); } }; -std::optional parseLegacyContentAddressOpt(std::string_view rawCaOpt) { - return rawCaOpt == "" ? std::optional {} : parseLegacyContentAddress(rawCaOpt); +std::optional parseContentAddressOpt(std::string_view rawCaOpt) { + return rawCaOpt == "" ? std::optional {} : parseContentAddress(rawCaOpt); }; -std::string renderLegacyContentAddress(std::optional ca) { - return ca ? renderLegacyContentAddress(*ca) : ""; +std::string renderContentAddress(std::optional ca) { + return ca ? renderContentAddress(*ca) : ""; } } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 10a5fcf6657d..5f18b0b9d4f5 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -40,19 +40,19 @@ struct FixedOutputHash { typedef std::variant< TextHash, // for paths computed by makeTextPath() / addTextToStore FixedOutputHash // for path computed by makeFixedOutputPath -> LegacyContentAddress; +> ContentAddress; /* Compute the prefix to the hash algorithm which indicates how the files were ingested. */ std::string makeFileIngestionPrefix(const FileIngestionMethod m); -std::string renderLegacyContentAddress(LegacyContentAddress ca); +std::string renderContentAddress(ContentAddress ca); -std::string renderLegacyContentAddress(std::optional ca); +std::string renderContentAddress(std::optional ca); -LegacyContentAddress parseLegacyContentAddress(std::string_view rawCa); +ContentAddress parseContentAddress(std::string_view rawCa); -std::optional parseLegacyContentAddressOpt(std::string_view rawCaOpt); +std::optional parseContentAddressOpt(std::string_view rawCaOpt); /* * References set diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 5ad7f6ade49c..9a7524d39ba1 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -664,7 +664,7 @@ static void performOp(TunnelLogger * logger, ref store, if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { to << info->ultimate << info->sigs - << renderLegacyContentAddress(info->ca); + << renderContentAddress(info->ca); } } else { assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); @@ -722,7 +722,7 @@ static void performOp(TunnelLogger * logger, ref store, info.setReferencesPossiblyToSelf(readStorePaths(*store, from)); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); - info.ca = parseLegacyContentAddressOpt(readString(from)); + info.ca = parseContentAddressOpt(readString(from)); from >> repair >> dontCheckSigs; if (!trusted && dontCheckSigs) dontCheckSigs = false; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index a607f1022bca..f01e642a0356 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -113,7 +113,7 @@ struct LegacySSHStore : public Store if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) { auto s = readString(conn->from); info->narHash = s.empty() ? Hash() : Hash(s); - info->ca = parseLegacyContentAddressOpt(readString(conn->from)); + info->ca = parseContentAddressOpt(readString(conn->from)); info->sigs = readStrings(conn->from); } @@ -145,7 +145,7 @@ struct LegacySSHStore : public Store << info.narSize << info.ultimate << info.sigs - << renderLegacyContentAddress(info.ca); + << renderContentAddress(info.ca); try { copyNAR(source, conn->to); } catch (...) { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 173e97bed591..a73966df306e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -592,7 +592,7 @@ uint64_t LocalStore::addValidPath(State & state, (info.narSize, info.narSize != 0) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) - (renderLegacyContentAddress(info.ca), (bool) info.ca) + (renderContentAddress(info.ca), (bool) info.ca) .exec(); uint64_t id = sqlite3_last_insert_rowid(state.db); @@ -666,7 +666,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, if (s) info->sigs = tokenizeString(s, " "); s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 7); - if (s) info->ca = parseLegacyContentAddressOpt(s); + if (s) info->ca = parseContentAddressOpt(s); /* Get the references. */ auto useQueryReferences(state->stmtQueryReferences.use()(info->id)); @@ -691,7 +691,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) (info.narHash.to_string(Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) - (renderLegacyContentAddress(info.ca), (bool) info.ca) + (renderContentAddress(info.ca), (bool) info.ca) (printStorePath(info.path)) .exec(); } diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 51d8d4f4b356..c543f6ea297d 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -203,7 +203,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString(queryNAR.getStr(10), " ")) narInfo->sigs.insert(sig); - narInfo->ca = parseLegacyContentAddressOpt(queryNAR.getStr(11)); + narInfo->ca = parseContentAddressOpt(queryNAR.getStr(11)); return {oValid, narInfo}; }); @@ -237,7 +237,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) (concatStringsSep(" ", info->sigs)) - (renderLegacyContentAddress(info->ca)) + (renderContentAddress(info->ca)) (time(0)).exec(); } else { diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index ecec5bad80b2..0796de466487 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -70,7 +70,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "CA") { if (ca) corrupt(); // FIXME: allow blank ca or require skipping field? - ca = parseLegacyContentAddressOpt(value); + ca = parseContentAddressOpt(value); } pos = eol + 1; @@ -107,7 +107,7 @@ std::string NarInfo::to_string(const Store & store) const res += "Sig: " + sig + "\n"; if (ca) - res += "CA: " + renderLegacyContentAddress(*ca) + "\n"; + res += "CA: " + renderContentAddress(*ca) + "\n"; return res; } diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 5b2716b769f2..fcd080933fb8 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -51,7 +51,7 @@ struct ValidPathInfo : PathReferences and the store path would be computed from the name component, ‘narHash’ and ‘references’. However, we support many types of content addresses. */ - std::optional ca; + std::optional ca; bool operator == (const ValidPathInfo & i) const { diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index aee82599bf07..100feeafe8dd 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -45,7 +45,7 @@ std::set readStorePathDescriptorSet(const Store & store, So // TODO // auto count = readNum(from); // while (count--) - // paths.insert_or_assign(store.parseStorePath(readString(from)), parseLegacyContentAddressOpt(readString(from))); + // paths.insert_or_assign(store.parseStorePath(readString(from)), parseContentAddressOpt(readString(from))); return paths; } @@ -55,7 +55,7 @@ void writeStorePathDescriptorSet(const Store & store, Sink & out, const std::set //out << paths.size(); //for (auto & i : paths) { // out << store.printStorePath(i.first); - // out << renderLegacyContentAddress(i.second); + // out << renderContentAddress(i.second); //} } @@ -436,7 +436,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { conn->from >> info->ultimate; info->sigs = readStrings(conn->from); - info->ca = parseLegacyContentAddressOpt(readString(conn->from)); + info->ca = parseContentAddressOpt(readString(conn->from)); } } callback(std::move(info)); @@ -532,7 +532,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << info.narHash.to_string(Base16, false); writeStorePaths(*this, conn->to, info.referencesPossiblyToSelf()); conn->to << info.registrationTime << info.narSize - << info.ultimate << info.sigs << renderLegacyContentAddress(info.ca) + << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21; if (!tunnel) copyNAR(source, conn->to); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 80d336209a63..d92537845d89 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -507,7 +507,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store } if (info->ca) - jsonPath.attr("ca", renderLegacyContentAddress(info->ca)); + jsonPath.attr("ca", renderContentAddress(info->ca)); std::pair closureSizes; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index b96363e56679..04e408c73fa4 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -870,7 +870,7 @@ static void opServe(Strings opFlags, Strings opArgs) out << info->narSize // downloadSize << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << renderLegacyContentAddress(info->ca) << info->sigs; + out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << renderContentAddress(info->ca) << info->sigs; } catch (InvalidPath &) { } } @@ -958,7 +958,7 @@ static void opServe(Strings opFlags, Strings opArgs) info.setReferencesPossiblyToSelf(readStorePaths(*store, in)); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); - info.ca = parseLegacyContentAddressOpt(readString(in)); + info.ca = parseContentAddressOpt(readString(in)); if (info.narSize == 0) throw Error("narInfo is too old and missing the narSize field"); diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 126b69635e37..b89a44f83cc6 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -115,7 +115,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON std::cout << '\t'; Strings ss; if (info->ultimate) ss.push_back("ultimate"); - if (info->ca) ss.push_back("ca:" + renderLegacyContentAddress(*info->ca)); + if (info->ca) ss.push_back("ca:" + renderContentAddress(*info->ca)); for (auto & sig : info->sigs) ss.push_back(sig); std::cout << concatStringsSep(" ", ss); } From d0eaf2bc51c82e5a72cd6bf2f035156c85b083aa Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Mon, 13 Jul 2020 14:58:19 -0400 Subject: [PATCH 17/31] Rename ContentAddress to StorePathDescriptor (cherry picked from commit ff072964402897cde7ed3cb71930c693dff6ebe6) --- src/libfetchers/tarball.cc | 2 +- src/libstore/build.cc | 18 +++++++++--------- src/libstore/content-address.hh | 4 ++-- src/libstore/daemon.cc | 2 +- src/libstore/local-store.cc | 2 +- src/libstore/local-store.hh | 2 +- src/libstore/misc.cc | 4 ++-- src/libstore/path-info.hh | 4 ++-- src/libstore/remote-store.cc | 8 ++++---- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.cc | 8 ++++---- src/libstore/store-api.hh | 6 +++--- src/libstore/worker-protocol.hh | 4 ++-- src/nix/add-to-store.cc | 2 +- src/nix/make-content-addressable.cc | 2 +- 15 files changed, 35 insertions(+), 35 deletions(-) diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index c9fac531a44a..7d4d3d6c711a 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -69,7 +69,7 @@ DownloadFileResult downloadFile( auto hash = hashString(htSHA256, *res.data); ValidPathInfo info { *store, - ContentAddress { + StorePathDescriptor { .name = name, .info = FixedOutputInfo { { diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 299887ec1fc0..1df95fb3e002 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -297,7 +297,7 @@ class Worker GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); std::shared_ptr makeBasicDerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal); - GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); /* Remove a dead goal. */ void removeGoal(GoalPtr goal); @@ -3717,7 +3717,7 @@ void DerivationGoal::registerOutputs() /* Check that fixed-output derivations produced the right outputs (i.e., the content hash should match the specified hash). */ - std::optional ca; + std::optional ca; if (fixedOutput) { @@ -3773,7 +3773,7 @@ void DerivationGoal::registerOutputs() else assert(worker.store.parseStorePath(path) == dest); - ca = ContentAddress { + ca = StorePathDescriptor { .name = std::string { i.second.path.name() }, .info = FixedOutputInfo { { @@ -3847,7 +3847,7 @@ void DerivationGoal::registerOutputs() } auto info = ca - ? ValidPathInfo { worker.store, ContentAddress { *ca } } + ? ValidPathInfo { worker.store, StorePathDescriptor { *ca } } : ValidPathInfo { worker.store.parseStorePath(path) }; info.narHash = hash.first; info.narSize = hash.second; @@ -4281,7 +4281,7 @@ class SubstitutionGoal : public Goal private: /* The store path that should be realised through a substitute. */ - // TODO std::variant storePath; + // TODO std::variant storePath; StorePath storePath; /* The remaining substituters. */ @@ -4319,10 +4319,10 @@ class SubstitutionGoal : public Goal /* Content address for recomputing store path */ // TODO delete once `storePath` is variant. - std::optional ca; + std::optional ca; public: - SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); ~SubstitutionGoal(); void timedOut(Error && ex) override { abort(); }; @@ -4352,7 +4352,7 @@ class SubstitutionGoal : public Goal }; -SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) +SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) : Goal(worker) , storePath(storePath) , repair(repair) @@ -4699,7 +4699,7 @@ std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath } -GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) +GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) { GoalPtr goal = substitutionGoals[path].lock(); // FIXME if (!goal) { diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 84ff71a344fb..10a5fcf6657d 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -116,14 +116,14 @@ struct FixedOutputInfo : FixedOutputHash { PathReferences references; }; -struct ContentAddress { +struct StorePathDescriptor { std::string name; std::variant< TextInfo, FixedOutputInfo > info; - bool operator < (const ContentAddress & other) const + bool operator < (const StorePathDescriptor & other) const { return name < other.name; } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index e26fc9f1c161..f56576818d9f 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -620,7 +620,7 @@ static void performOp(TunnelLogger * logger, ref store, case wopQuerySubstitutablePathInfos: { SubstitutablePathInfos infos; auto paths = readStorePaths(*store, from); - std::set caPaths; + std::set caPaths; if (GET_PROTOCOL_MINOR(clientVersion) > 22) caPaths = readContentAddressSet(*store, from); logger->startWork(); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c0ac1da03a5b..173e97bed591 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -848,7 +848,7 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) } -void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, const std::set & caPaths, SubstitutablePathInfos & infos) +void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, const std::set & caPaths, SubstitutablePathInfos & infos) { if (!settings.useSubstitutes) return; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index f835c1a9fa0f..a4fbd65d5de3 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -140,7 +140,7 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; void querySubstitutablePathInfos(const StorePathSet & paths, - const std::set & caPaths, + const std::set & caPaths, SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & source, diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index ece38fc7b2d6..13823a933c35 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -107,11 +107,11 @@ void Store::computeFSClosure(const StorePath & startPath, } -std::optional getDerivationCA(const BasicDerivation & drv) +std::optional getDerivationCA(const BasicDerivation & drv) { auto out = drv.outputs.find("out"); if (out != drv.outputs.end() && out->second.hash) { - return ContentAddress { + return StorePathDescriptor { .name = std::string { out->second.path.name() }, .info = FixedOutputInfo { *out->second.hash, {} }, }; diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index c702c60f55c4..febc3e6fe65d 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -72,7 +72,7 @@ struct ValidPathInfo : PathReferences void sign(const Store & store, const SecretKey & secretKey); - std::optional fullContentAddressOpt() const; + std::optional fullContentAddressOpt() const; /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; @@ -101,7 +101,7 @@ struct ValidPathInfo : PathReferences ValidPathInfo(const StorePath & path) : path(path) { }; ValidPathInfo(const Store & store, - ContentAddress && ca); + StorePathDescriptor && ca); virtual ~ValidPathInfo() { } }; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 8e2a9dac2794..bb2f49a3b26a 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -38,9 +38,9 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths out << store.printStorePath(i); } -std::set readContentAddressSet(const Store & store, Source & from) +std::set readContentAddressSet(const Store & store, Source & from) { - std::set paths; + std::set paths; // TODO // auto count = readNum(from); // while (count--) @@ -48,7 +48,7 @@ std::set readContentAddressSet(const Store & store, Source & fro return paths; } -void writeContentAddressSet(const Store & store, Sink & out, const std::set & paths) +void writeContentAddressSet(const Store & store, Sink & out, const std::set & paths) { // TODO //out << paths.size(); @@ -351,7 +351,7 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) } -void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, const std::set & caPaths, SubstitutablePathInfos & infos) +void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, const std::set & caPaths, SubstitutablePathInfos & infos) { if (paths.empty() && caPaths.empty()) return; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index a2da5a0fc4a1..be931dabab06 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -57,7 +57,7 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; void querySubstitutablePathInfos(const StorePathSet & paths, - const std::set & caPaths, + const std::set & caPaths, SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & nar, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 4928a16f6c5f..5f7375abee2e 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -205,7 +205,7 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons template struct overloaded : Ts... { using Ts::operator()...; }; template overloaded(Ts...) -> overloaded; -StorePath Store::makeFixedOutputPathFromCA(const ContentAddress & info) const +StorePath Store::makeFixedOutputPathFromCA(const StorePathDescriptor & info) const { // New template return std::visit(overloaded { @@ -843,12 +843,12 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } -std::optional ValidPathInfo::fullContentAddressOpt() const +std::optional ValidPathInfo::fullContentAddressOpt() const { if (! ca) return std::nullopt; - return ContentAddress { + return StorePathDescriptor { .name = std::string { path.name() }, .info = std::visit(overloaded { [&](TextHash th) { @@ -913,7 +913,7 @@ Strings ValidPathInfo::shortRefs() const ValidPathInfo::ValidPathInfo( const Store & store, - ContentAddress && info) + StorePathDescriptor && info) : path(store.makeFixedOutputPathFromCA(info)) { std::visit(overloaded { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 37b75e730528..6f08151a9bfb 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -254,7 +254,7 @@ public: StorePath makeTextPath(std::string_view name, const TextInfo & info) const; - StorePath makeFixedOutputPathFromCA(const ContentAddress & info) const; + StorePath makeFixedOutputPathFromCA(const StorePathDescriptor & info) const; /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store @@ -347,7 +347,7 @@ public: does not have substitute info, it's omitted from the resulting ‘infos’ map. */ virtual void querySubstitutablePathInfos(const StorePathSet & paths, - const std::set & caPaths, + const std::set & caPaths, SubstitutablePathInfos & infos) { return; }; /* Import a path into the store. */ @@ -749,6 +749,6 @@ std::optional decodeValidPathInfo( /* Split URI into protocol+hierarchy part and its parameter set. */ std::pair splitUriAndParams(const std::string & uri); -std::optional getDerivationCA(const BasicDerivation & drv); +std::optional getDerivationCA(const BasicDerivation & drv); } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 0b6816abb425..cdb9b9be4e62 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -70,9 +70,9 @@ template T readStorePaths(const Store & store, Source & from); void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths); -std::set readContentAddressSet(const Store & store, Source & from); +std::set readContentAddressSet(const Store & store, Source & from); -void writeContentAddressSet(const Store & store, Sink & out, const std::set & paths); +void writeContentAddressSet(const Store & store, Sink & out, const std::set & paths); void writeOutputPathMap(const Store & store, Sink & out, const OutputPathMap & paths); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 1ebe38344f2c..b9aa8dd162fd 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -69,7 +69,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand ValidPathInfo info { *store, - ContentAddress { + StorePathDescriptor { .name = *namePart, .info = FixedOutputInfo { { diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 7e3c0ee4e05e..028c0f1442d1 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -75,7 +75,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON ValidPathInfo info { *store, - ContentAddress { + StorePathDescriptor { .name = std::string { path.name() }, .info = FixedOutputInfo { { From 1cf131f78a97a1050e9a1de1f2dfc67f7ce4014d Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Tue, 14 Jul 2020 10:41:47 -0400 Subject: [PATCH 18/31] Rename functions related to ContentAddress (cherry picked from commit ade0f1a440120475bfe4a985f66d050f6cd01279) --- src/libstore/daemon.cc | 2 +- src/libstore/path-info.hh | 2 +- src/libstore/remote-store.cc | 6 +++--- src/libstore/store-api.cc | 10 +++++----- src/libstore/worker-protocol.hh | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index f56576818d9f..97666f06a560 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -622,7 +622,7 @@ static void performOp(TunnelLogger * logger, ref store, auto paths = readStorePaths(*store, from); std::set caPaths; if (GET_PROTOCOL_MINOR(clientVersion) > 22) - caPaths = readContentAddressSet(*store, from); + caPaths = readStorePathDescriptorSet(*store, from); logger->startWork(); store->querySubstitutablePathInfos(paths, caPaths, infos); logger->stopWork(); diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index febc3e6fe65d..5b2716b769f2 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -72,7 +72,7 @@ struct ValidPathInfo : PathReferences void sign(const Store & store, const SecretKey & secretKey); - std::optional fullContentAddressOpt() const; + std::optional fullStorePathDescriptorOpt() const; /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index bb2f49a3b26a..f8687f6bdf3f 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -38,7 +38,7 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths out << store.printStorePath(i); } -std::set readContentAddressSet(const Store & store, Source & from) +std::set readStorePathDescriptorSet(const Store & store, Source & from) { std::set paths; // TODO @@ -48,7 +48,7 @@ std::set readContentAddressSet(const Store & store, Source return paths; } -void writeContentAddressSet(const Store & store, Sink & out, const std::set & paths) +void writeStorePathDescriptorSet(const Store & store, Sink & out, const std::set & paths) { // TODO //out << paths.size(); @@ -387,7 +387,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, const writeStorePaths(*this, conn->to, combine()); } else { writeStorePaths(*this, conn->to, paths); - writeContentAddressSet(*this, conn->to, caPaths); + writeStorePathDescriptorSet(*this, conn->to, caPaths); } conn.processStderr(); size_t count = readNum(conn->from); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 5f7375abee2e..80d336209a63 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -619,7 +619,7 @@ void copyStorePath(ref srcStore, ref dstStore, // recompute store path on the chance dstStore does it differently if (info->ca && info->references.empty()) { auto info2 = make_ref(*info); - info2->path = dstStore->makeFixedOutputPathFromCA(*info->fullContentAddressOpt()); + info2->path = dstStore->makeFixedOutputPathFromCA(*info->fullStorePathDescriptorOpt()); if (dstStore->storeDir == srcStore->storeDir) assert(info->path == info2->path); info = info2; @@ -697,7 +697,7 @@ std::map copyPaths(ref srcStore, ref dstStor auto info = srcStore->queryPathInfo(storePath); auto storePathForDst = storePath; if (info->ca && info->references.empty() && !info->hasSelfReference) { - storePathForDst = dstStore->makeFixedOutputPathFromCA(*info->fullContentAddressOpt()); + storePathForDst = dstStore->makeFixedOutputPathFromCA(*info->fullStorePathDescriptorOpt()); if (dstStore->storeDir == srcStore->storeDir) assert(storePathForDst == storePath); if (storePathForDst != storePath) @@ -725,7 +725,7 @@ std::map copyPaths(ref srcStore, ref dstStor auto storePathForDst = storePath; if (info->ca && info->references.empty() && !info->hasSelfReference) { - storePathForDst = dstStore->makeFixedOutputPathFromCA(*info->fullContentAddressOpt()); + storePathForDst = dstStore->makeFixedOutputPathFromCA(*info->fullStorePathDescriptorOpt()); if (dstStore->storeDir == srcStore->storeDir) assert(storePathForDst == storePath); if (storePathForDst != storePath) @@ -843,7 +843,7 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } -std::optional ValidPathInfo::fullContentAddressOpt() const +std::optional ValidPathInfo::fullStorePathDescriptorOpt() const { if (! ca) return std::nullopt; @@ -868,7 +868,7 @@ std::optional ValidPathInfo::fullContentAddressOpt() const bool ValidPathInfo::isContentAddressed(const Store & store) const { - auto fullCaOpt = fullContentAddressOpt(); + auto fullCaOpt = fullStorePathDescriptorOpt(); if (! fullCaOpt) return false; diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index cdb9b9be4e62..f42ce125f2ff 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -70,9 +70,9 @@ template T readStorePaths(const Store & store, Source & from); void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths); -std::set readContentAddressSet(const Store & store, Source & from); +std::set readStorePathDescriptorSet(const Store & store, Source & from); -void writeContentAddressSet(const Store & store, Sink & out, const std::set & paths); +void writeStorePathDescriptorSet(const Store & store, Sink & out, const std::set & paths); void writeOutputPathMap(const Store & store, Sink & out, const OutputPathMap & paths); From 74de4c79bfded18508c3d1b3c5f64c21d5182c06 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Tue, 14 Jul 2020 10:46:18 -0400 Subject: [PATCH 19/31] Rename LegacyContentAddress and related functions (cherry picked from commit 08bf6c414157dc14221b760f395c0920149c5095) --- src/libstore/content-address.cc | 16 ++++++++-------- src/libstore/content-address.hh | 10 +++++----- src/libstore/daemon.cc | 4 ++-- src/libstore/legacy-ssh-store.cc | 4 ++-- src/libstore/local-store.cc | 6 +++--- src/libstore/nar-info-disk-cache.cc | 4 ++-- src/libstore/nar-info.cc | 4 ++-- src/libstore/path-info.hh | 2 +- src/libstore/remote-store.cc | 8 ++++---- src/libstore/store-api.cc | 2 +- src/nix-store/nix-store.cc | 4 ++-- src/nix/path-info.cc | 2 +- 12 files changed, 33 insertions(+), 33 deletions(-) diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index a4d1c2912a2e..ebd1763f522d 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -21,7 +21,7 @@ std::string makeFileIngestionPrefix(const FileIngestionMethod m) { template struct overloaded : Ts... { using Ts::operator()...; }; template overloaded(Ts...) -> overloaded; -std::string renderLegacyContentAddress(LegacyContentAddress ca) { +std::string renderContentAddress(ContentAddress ca) { return std::visit(overloaded { [](TextHash th) { return "text:" @@ -35,7 +35,7 @@ std::string renderLegacyContentAddress(LegacyContentAddress ca) { }, ca); } -LegacyContentAddress parseLegacyContentAddress(std::string_view rawCa) { +ContentAddress parseContentAddress(std::string_view rawCa) { auto prefixSeparator = rawCa.find(':'); if (prefixSeparator != string::npos) { auto prefix = string(rawCa, 0, prefixSeparator); @@ -43,7 +43,7 @@ LegacyContentAddress parseLegacyContentAddress(std::string_view rawCa) { auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos); Hash hash = Hash(string(hashTypeAndHash)); if (*hash.type != htSHA256) { - throw Error("parseLegacyContentAddress: the text hash should have type SHA256"); + throw Error("parseContentAddress: the text hash should have type SHA256"); } return TextHash { hash }; } else if (prefix == "fixed") { @@ -62,19 +62,19 @@ LegacyContentAddress parseLegacyContentAddress(std::string_view rawCa) { }; } } else { - throw Error("parseLegacyContentAddress: format not recognized; has to be text or fixed"); + throw Error("parseContentAddress: format not recognized; has to be text or fixed"); } } else { throw Error("Not a content address because it lacks an appropriate prefix"); } }; -std::optional parseLegacyContentAddressOpt(std::string_view rawCaOpt) { - return rawCaOpt == "" ? std::optional {} : parseLegacyContentAddress(rawCaOpt); +std::optional parseContentAddressOpt(std::string_view rawCaOpt) { + return rawCaOpt == "" ? std::optional {} : parseContentAddress(rawCaOpt); }; -std::string renderLegacyContentAddress(std::optional ca) { - return ca ? renderLegacyContentAddress(*ca) : ""; +std::string renderContentAddress(std::optional ca) { + return ca ? renderContentAddress(*ca) : ""; } } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 10a5fcf6657d..5f18b0b9d4f5 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -40,19 +40,19 @@ struct FixedOutputHash { typedef std::variant< TextHash, // for paths computed by makeTextPath() / addTextToStore FixedOutputHash // for path computed by makeFixedOutputPath -> LegacyContentAddress; +> ContentAddress; /* Compute the prefix to the hash algorithm which indicates how the files were ingested. */ std::string makeFileIngestionPrefix(const FileIngestionMethod m); -std::string renderLegacyContentAddress(LegacyContentAddress ca); +std::string renderContentAddress(ContentAddress ca); -std::string renderLegacyContentAddress(std::optional ca); +std::string renderContentAddress(std::optional ca); -LegacyContentAddress parseLegacyContentAddress(std::string_view rawCa); +ContentAddress parseContentAddress(std::string_view rawCa); -std::optional parseLegacyContentAddressOpt(std::string_view rawCaOpt); +std::optional parseContentAddressOpt(std::string_view rawCaOpt); /* * References set diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 97666f06a560..767397d10551 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -664,7 +664,7 @@ static void performOp(TunnelLogger * logger, ref store, if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { to << info->ultimate << info->sigs - << renderLegacyContentAddress(info->ca); + << renderContentAddress(info->ca); } } else { assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); @@ -722,7 +722,7 @@ static void performOp(TunnelLogger * logger, ref store, info.setReferencesPossiblyToSelf(readStorePaths(*store, from)); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); - info.ca = parseLegacyContentAddressOpt(readString(from)); + info.ca = parseContentAddressOpt(readString(from)); from >> repair >> dontCheckSigs; if (!trusted && dontCheckSigs) dontCheckSigs = false; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index a607f1022bca..f01e642a0356 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -113,7 +113,7 @@ struct LegacySSHStore : public Store if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) { auto s = readString(conn->from); info->narHash = s.empty() ? Hash() : Hash(s); - info->ca = parseLegacyContentAddressOpt(readString(conn->from)); + info->ca = parseContentAddressOpt(readString(conn->from)); info->sigs = readStrings(conn->from); } @@ -145,7 +145,7 @@ struct LegacySSHStore : public Store << info.narSize << info.ultimate << info.sigs - << renderLegacyContentAddress(info.ca); + << renderContentAddress(info.ca); try { copyNAR(source, conn->to); } catch (...) { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 173e97bed591..a73966df306e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -592,7 +592,7 @@ uint64_t LocalStore::addValidPath(State & state, (info.narSize, info.narSize != 0) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) - (renderLegacyContentAddress(info.ca), (bool) info.ca) + (renderContentAddress(info.ca), (bool) info.ca) .exec(); uint64_t id = sqlite3_last_insert_rowid(state.db); @@ -666,7 +666,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, if (s) info->sigs = tokenizeString(s, " "); s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 7); - if (s) info->ca = parseLegacyContentAddressOpt(s); + if (s) info->ca = parseContentAddressOpt(s); /* Get the references. */ auto useQueryReferences(state->stmtQueryReferences.use()(info->id)); @@ -691,7 +691,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) (info.narHash.to_string(Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) - (renderLegacyContentAddress(info.ca), (bool) info.ca) + (renderContentAddress(info.ca), (bool) info.ca) (printStorePath(info.path)) .exec(); } diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 51d8d4f4b356..c543f6ea297d 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -203,7 +203,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString(queryNAR.getStr(10), " ")) narInfo->sigs.insert(sig); - narInfo->ca = parseLegacyContentAddressOpt(queryNAR.getStr(11)); + narInfo->ca = parseContentAddressOpt(queryNAR.getStr(11)); return {oValid, narInfo}; }); @@ -237,7 +237,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) (concatStringsSep(" ", info->sigs)) - (renderLegacyContentAddress(info->ca)) + (renderContentAddress(info->ca)) (time(0)).exec(); } else { diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index ecec5bad80b2..0796de466487 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -70,7 +70,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "CA") { if (ca) corrupt(); // FIXME: allow blank ca or require skipping field? - ca = parseLegacyContentAddressOpt(value); + ca = parseContentAddressOpt(value); } pos = eol + 1; @@ -107,7 +107,7 @@ std::string NarInfo::to_string(const Store & store) const res += "Sig: " + sig + "\n"; if (ca) - res += "CA: " + renderLegacyContentAddress(*ca) + "\n"; + res += "CA: " + renderContentAddress(*ca) + "\n"; return res; } diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 5b2716b769f2..fcd080933fb8 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -51,7 +51,7 @@ struct ValidPathInfo : PathReferences and the store path would be computed from the name component, ‘narHash’ and ‘references’. However, we support many types of content addresses. */ - std::optional ca; + std::optional ca; bool operator == (const ValidPathInfo & i) const { diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index f8687f6bdf3f..57ccba32f1e7 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -44,7 +44,7 @@ std::set readStorePathDescriptorSet(const Store & store, So // TODO // auto count = readNum(from); // while (count--) - // paths.insert_or_assign(store.parseStorePath(readString(from)), parseLegacyContentAddressOpt(readString(from))); + // paths.insert_or_assign(store.parseStorePath(readString(from)), parseContentAddressOpt(readString(from))); return paths; } @@ -54,7 +54,7 @@ void writeStorePathDescriptorSet(const Store & store, Sink & out, const std::set //out << paths.size(); //for (auto & i : paths) { // out << store.printStorePath(i.first); - // out << renderLegacyContentAddress(i.second); + // out << renderContentAddress(i.second); //} } @@ -435,7 +435,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { conn->from >> info->ultimate; info->sigs = readStrings(conn->from); - info->ca = parseLegacyContentAddressOpt(readString(conn->from)); + info->ca = parseContentAddressOpt(readString(conn->from)); } } callback(std::move(info)); @@ -531,7 +531,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << info.narHash.to_string(Base16, false); writeStorePaths(*this, conn->to, info.referencesPossiblyToSelf()); conn->to << info.registrationTime << info.narSize - << info.ultimate << info.sigs << renderLegacyContentAddress(info.ca) + << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21; if (!tunnel) copyNAR(source, conn->to); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 80d336209a63..d92537845d89 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -507,7 +507,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store } if (info->ca) - jsonPath.attr("ca", renderLegacyContentAddress(info->ca)); + jsonPath.attr("ca", renderContentAddress(info->ca)); std::pair closureSizes; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index b96363e56679..04e408c73fa4 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -870,7 +870,7 @@ static void opServe(Strings opFlags, Strings opArgs) out << info->narSize // downloadSize << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << renderLegacyContentAddress(info->ca) << info->sigs; + out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << renderContentAddress(info->ca) << info->sigs; } catch (InvalidPath &) { } } @@ -958,7 +958,7 @@ static void opServe(Strings opFlags, Strings opArgs) info.setReferencesPossiblyToSelf(readStorePaths(*store, in)); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); - info.ca = parseLegacyContentAddressOpt(readString(in)); + info.ca = parseContentAddressOpt(readString(in)); if (info.narSize == 0) throw Error("narInfo is too old and missing the narSize field"); diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 126b69635e37..b89a44f83cc6 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -115,7 +115,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON std::cout << '\t'; Strings ss; if (info->ultimate) ss.push_back("ultimate"); - if (info->ca) ss.push_back("ca:" + renderLegacyContentAddress(*info->ca)); + if (info->ca) ss.push_back("ca:" + renderContentAddress(*info->ca)); for (auto & sig : info->sigs) ss.push_back(sig); std::cout << concatStringsSep(" ", ss); } From ecf65a6aab9f15776fa00ac99e8b87fa75ffac38 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 21 Jul 2020 19:35:13 +0000 Subject: [PATCH 20/31] Use StorePathDescriptor more widely --- src/build-remote/build-remote.cc | 7 +- src/libexpr/primops.cc | 8 +- src/libexpr/primops/context.cc | 6 +- src/libexpr/primops/fetchTree.cc | 14 ++ src/libexpr/value-to-xml.hh | 2 +- src/libfetchers/fetchers.cc | 42 +++++- src/libfetchers/fetchers.hh | 4 + src/libfetchers/tree-info.hh | 4 + src/libstore/binary-cache-store.cc | 15 ++- src/libstore/binary-cache-store.hh | 8 +- src/libstore/build.cc | 76 ++++++----- src/libstore/content-address.cc | 161 ++++++++++++++++++---- src/libstore/content-address.hh | 27 +++- src/libstore/daemon.cc | 27 ++-- src/libstore/export-import.cc | 12 +- src/libstore/globals.cc | 13 +- src/libstore/globals.hh | 4 + src/libstore/legacy-ssh-store.cc | 8 +- src/libstore/local-fs-store.cc | 9 +- src/libstore/local-store.cc | 184 +++++++++----------------- src/libstore/local-store.hh | 8 +- src/libstore/misc.cc | 3 +- src/libstore/nar-info.hh | 2 + src/libstore/remote-store.cc | 9 +- src/libstore/remote-store.hh | 6 +- src/libstore/s3-binary-cache-store.cc | 3 +- src/libstore/ssh-store.cc | 5 +- src/libstore/store-api.cc | 97 +++++++++----- src/libstore/store-api.hh | 30 +++-- src/libutil/archive.hh | 8 -- src/libutil/error.hh | 3 +- src/libutil/serialise.cc | 17 +++ src/libutil/serialise.hh | 17 ++- src/libutil/types.hh | 1 + src/libutil/util.hh | 5 + src/nix-env/nix-env.cc | 3 +- src/nix-env/user-env.cc | 3 +- src/nix-store/nix-store.cc | 15 ++- src/nix/command.cc | 3 +- src/nix/repl.cc | 3 +- src/nix/upgrade-nix.cc | 3 +- src/nix/verify.cc | 6 +- 42 files changed, 572 insertions(+), 309 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index e071174960a6..26c7d3b5863d 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -253,8 +253,11 @@ static int _main(int argc, char * * argv) throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg); StorePathSet missing; - for (auto & path : outputs) - if (!store->isValidPath(store->parseStorePath(path))) missing.insert(store->parseStorePath(path)); + for (auto & path : outputs) { + auto storePath = store->parseStorePath(path); + if (!store->isValidPath(storePath)) + missing.insert(store->parseStorePath(path)); + } if (!missing.empty()) { Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri)); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index d1ad2f7db306..92110d450195 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -103,7 +103,13 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args Path realPath = state.checkSourcePath(state.toRealPath(path, context)); // FIXME - if (state.store->isStorePath(path) && state.store->isValidPath(state.store->parseStorePath(path)) && isDerivation(path)) { + auto complicatedCondition = [&]() -> bool { + if (!state.store->isStorePath(path)) + return false; + auto path2 = state.store->parseStorePath(path); + return state.store->isValidPath(path2) && isDerivation(path); + }; + if (complicatedCondition()) { Derivation drv = readDerivation(*state.store, realPath); Value & w = *state.allocValue(); state.mkAttrs(w, 3 + drv.outputs.size()); diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index dbb93bae6a23..c0eb1b7c6cd4 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -150,8 +150,10 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg .hint = hintfmt("Context key '%s' is not a store path", i.name), .errPos = *i.pos }); - if (!settings.readOnlyMode) - state.store->ensurePath(state.store->parseStorePath(i.name)); + if (!settings.readOnlyMode) { + auto path = state.store->parseStorePath(i.name); + state.store->ensurePath(path); + } state.forceAttrs(*i.value, *i.pos); auto iter = i.value->attrs->find(sPath); if (iter != i.value->attrs->end()) { diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index f6d8cca44b20..3a131f2d20fd 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -134,6 +134,20 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, if (evalSettings.pureEval && !expectedHash) throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); + // try to substitute if we can + if (settings.useSubstitutes && expectedHash) { + auto substitutableStorePath = fetchers::trySubstitute(state.store, + unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, *expectedHash, name); + if (substitutableStorePath) { + auto substitutablePath = state.store->toRealPath(*substitutableStorePath); + if (state.allowedPaths) + state.allowedPaths->insert(substitutablePath); + + mkString(v, substitutablePath, PathSet({substitutablePath})); + return; + } + } + auto storePath = unpack ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath diff --git a/src/libexpr/value-to-xml.hh b/src/libexpr/value-to-xml.hh index 02ccd714a908..c5f327bd8540 100644 --- a/src/libexpr/value-to-xml.hh +++ b/src/libexpr/value-to-xml.hh @@ -10,5 +10,5 @@ namespace nix { void printValueAsXML(EvalState & state, bool strict, bool location, Value & v, std::ostream & out, PathSet & context); - + } diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index b1782feab060..b77e4907fe42 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -1,5 +1,6 @@ #include "fetchers.hh" #include "store-api.hh" +#include "archive.hh" #include @@ -58,8 +59,21 @@ std::pair> Input::fetchTree(ref store) if (tree.actualPath == "") tree.actualPath = store->toRealPath(tree.storePath); + if (!tree.info.narHash) - tree.info.narHash = store->queryPathInfo(tree.storePath)->narHash; + { + auto pathOrCa = tree.info.ca + ? StorePathOrDesc {*tree.info.ca} + : StorePathOrDesc {tree.storePath}; + tree.info.narHash = store->queryPathInfo(pathOrCa)->narHash; + } + + + if (!tree.info.narHash) { + HashSink hashSink(htSHA256); + store->narFromPath(tree.storePath, hashSink); + tree.info.narHash = hashSink.finish().first; + } if (input->narHash) assert(input->narHash == tree.info.narHash); @@ -71,4 +85,30 @@ std::pair> Input::fetchTree(ref store) return {std::move(tree), input}; } +std::optional trySubstitute(ref store, FileIngestionMethod ingestionMethod, + Hash hash, std::string_view name) +{ + auto ca = StorePathDescriptor { + .name = std::string { name }, + .info = FixedOutputInfo { + ingestionMethod, + hash, + {} + }, + }; + auto substitutablePath = store->makeFixedOutputPathFromCA(ca); + + try { + store->ensurePath(ca); + + debug("using substituted path '%s'", store->printStorePath(substitutablePath)); + + return substitutablePath; + } catch (Error & e) { + debug("substitution of path '%s' failed: %s", store->printStorePath(substitutablePath), e.what()); + } + + return std::nullopt; +} + } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 59a58ae67819..ec73e1f116d3 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -6,6 +6,7 @@ #include "tree-info.hh" #include "attrs.hh" #include "url.hh" +#include "content-address.hh" #include @@ -100,4 +101,7 @@ Tree downloadTarball( const std::string & name, bool immutable); +std::optional trySubstitute(ref store, FileIngestionMethod ingestionMethod, + Hash hash, std::string_view name); + } diff --git a/src/libfetchers/tree-info.hh b/src/libfetchers/tree-info.hh index 9d1872097427..57a2c5d6b9bb 100644 --- a/src/libfetchers/tree-info.hh +++ b/src/libfetchers/tree-info.hh @@ -2,6 +2,7 @@ #include "path.hh" #include "hash.hh" +#include "content-address.hh" #include @@ -12,6 +13,9 @@ namespace nix::fetchers { struct TreeInfo { std::optional narHash; + + std::optional ca; + std::optional revCount; std::optional lastModified; diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index cdff2e21a53e..d9011737d8ec 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -106,7 +106,7 @@ std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath) void BinaryCacheStore::writeNarInfo(ref narInfo) { - auto narInfoFile = narInfoFileFor(narInfo->path); + auto narInfoFile = narInfoFileFor(bakeCaIfNeeded(narInfo->path)); upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo"); @@ -296,15 +296,15 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource stats.narInfoWrite++; } -bool BinaryCacheStore::isValidPathUncached(const StorePath & storePath) +bool BinaryCacheStore::isValidPathUncached(StorePathOrDesc storePath) { // FIXME: this only checks whether a .narinfo with a matching hash // part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even // though they shouldn't. Not easily fixed. - return fileExists(narInfoFileFor(storePath)); + return fileExists(narInfoFileFor(bakeCaIfNeeded(storePath))); } -void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) +void BinaryCacheStore::narFromPath(StorePathOrDesc storePath, Sink & sink) { auto info = queryPathInfo(storePath).cast(); @@ -330,16 +330,17 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) stats.narReadBytes += narSize; } -void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, +void BinaryCacheStore::queryPathInfoUncached(StorePathOrDesc storePath, Callback> callback) noexcept { auto uri = getUri(); - auto storePathS = printStorePath(storePath); + auto actualStorePath = bakeCaIfNeeded(storePath); + auto storePathS = printStorePath(actualStorePath); auto act = std::make_shared(*logger, lvlTalkative, actQueryPathInfo, fmt("querying info about '%s' on '%s'", storePathS, uri), Logger::Fields{storePathS, uri}); PushActivity pact(act->id); - auto narInfoFile = narInfoFileFor(storePath); + auto narInfoFile = narInfoFileFor(actualStorePath); auto callbackPtr = std::make_shared(std::move(callback)); diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 9bcdf5901b8f..66c2ed56c95d 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -70,9 +70,9 @@ private: public: - bool isValidPathUncached(const StorePath & path) override; + bool isValidPathUncached(StorePathOrDesc path) override; - void queryPathInfoUncached(const StorePath & path, + void queryPathInfoUncached(StorePathOrDesc path, Callback> callback) noexcept override; std::optional queryPathFromHashPart(const std::string & hashPart) override @@ -88,13 +88,13 @@ public: StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; - void narFromPath(const StorePath & path, Sink & sink) override; + void narFromPath(StorePathOrDesc path, Sink & sink) override; BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override { unsupported("buildDerivation"); } - void ensurePath(const StorePath & path) override + void ensurePath(StorePathOrDesc path) override { unsupported("ensurePath"); } ref getFSAccessor() override; diff --git a/src/libstore/build.cc b/src/libstore/build.cc index fd96a85fb79c..0d1a11d67568 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -297,7 +297,7 @@ class Worker GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); std::shared_ptr makeBasicDerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal); - GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + GoalPtr makeSubstitutionGoal(StorePathOrDesc storePath, RepairFlag repair = NoRepair); /* Remove a dead goal. */ void removeGoal(GoalPtr goal); @@ -1205,8 +1205,11 @@ void DerivationGoal::haveDerivation() through substitutes. If that doesn't work, we'll build them. */ if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) - for (auto & i : invalidOutputs) - addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair, getDerivationCA(*drv))); + for (auto & i : invalidOutputs) { + auto optCA = getDerivationCA(*drv); + auto p = optCA ? StorePathOrDesc { *optCA } : i; + addWaitee(worker.makeSubstitutionGoal(p, buildMode == bmRepair ? Repair : NoRepair)); + } if (waitees.empty()) /* to prevent hang (no wake-up event) */ outputsSubstituted(); @@ -1951,7 +1954,7 @@ void linkOrCopy(const Path & from, const Path & to) file (e.g. 32000 of ext3), which is quite possible after a 'nix-store --optimise'. FIXME: actually, why don't we just bind-mount in this case? - + It can also fail with EPERM in BeegFS v7 and earlier versions which don't allow hard-links to other directories */ if (errno != EMLINK && errno != EPERM) @@ -2734,10 +2737,10 @@ struct RestrictedStore : public LocalFSStore return paths; } - void queryPathInfoUncached(const StorePath & path, + void queryPathInfoUncached(StorePathOrDesc path, Callback> callback) noexcept override { - if (goal.isAllowed(path)) { + if (goal.isAllowed(bakeCaIfNeeded(path))) { try { /* Censor impure information. */ auto info = std::make_shared(*next->queryPathInfo(path)); @@ -2774,7 +2777,7 @@ struct RestrictedStore : public LocalFSStore goal.addDependency(info.path); } - StorePath addToStoreFromDump(const string & dump, const string & name, + StorePath addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override { auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair); @@ -2790,15 +2793,17 @@ struct RestrictedStore : public LocalFSStore return path; } - void narFromPath(const StorePath & path, Sink & sink) override + void narFromPath(StorePathOrDesc pathOrDesc, Sink & sink) override { + auto path = bakeCaIfNeeded(pathOrDesc); if (!goal.isAllowed(path)) throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path)); - LocalFSStore::narFromPath(path, sink); + LocalFSStore::narFromPath(pathOrDesc, sink); } - void ensurePath(const StorePath & path) override + void ensurePath(StorePathOrDesc pathOrDesc) override { + auto path = bakeCaIfNeeded(pathOrDesc); if (!goal.isAllowed(path)) throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path)); /* Nothing to be done; 'path' must already be valid. */ @@ -3773,7 +3778,7 @@ void DerivationGoal::registerOutputs() actualPath = actualDest; } else - assert(worker.store.parseStorePath(path) == dest); + assert(i.second.path == dest); ca = StorePathDescriptor { .name = std::string { i.second.path.name() }, @@ -3803,8 +3808,8 @@ void DerivationGoal::registerOutputs() HashResult hash = pathSetAndHash.second; if (buildMode == bmCheck) { - if (!worker.store.isValidPath(worker.store.parseStorePath(path))) continue; - ValidPathInfo info(*worker.store.queryPathInfo(worker.store.parseStorePath(path))); + if (!worker.store.isValidPath(i.second.path)) continue; + ValidPathInfo info(*worker.store.queryPathInfo(i.second.path)); if (hash.first != info.narHash) { worker.checkMismatch = true; if (settings.runDiffHook || settings.keepFailed) { @@ -3847,12 +3852,12 @@ void DerivationGoal::registerOutputs() if (curRound == nrRounds) { worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences() - worker.markContentsGood(worker.store.parseStorePath(path)); + worker.markContentsGood(i.second.path); } auto info = ca ? ValidPathInfo { worker.store, StorePathDescriptor { *ca } } - : ValidPathInfo { worker.store.parseStorePath(path) }; + : ValidPathInfo { i.second.path }; info.narHash = hash.first; info.narSize = hash.second; info.setReferencesPossiblyToSelf(std::move(references)); @@ -4435,16 +4440,16 @@ void SubstitutionGoal::tryNext() sub = subs.front(); subs.pop_front(); - auto subPath = storePath; if (ca) { - subPath = sub->makeFixedOutputPathFromCA(*ca); + auto subPath = sub->makeFixedOutputPathFromCA(*ca); if (sub->storeDir == worker.store.storeDir) assert(subPath == storePath); } try { // FIXME: make async - info = sub->queryPathInfo(subPath); + auto p = ca ? StorePathOrDesc { std::cref(*ca) } : std::cref(storePath); + info = sub->queryPathInfo(p); } catch (InvalidPath &) { tryNext(); return; @@ -4562,15 +4567,10 @@ void SubstitutionGoal::tryToRun() Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); PushActivity pact(act.id); - auto subPath = storePath; - if (ca) { - subPath = sub->makeFixedOutputPathFromCA(*ca); - if (sub->storeDir == worker.store.storeDir) - assert(subPath == storePath); - } + auto p = ca ? StorePathOrDesc { std::cref(*ca) } : std::cref(storePath); copyStorePath(ref(sub), ref(worker.store.shared_from_this()), - subPath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); + p, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); promise.set_value(); } catch (...) { @@ -4703,12 +4703,18 @@ std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath } -GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) +GoalPtr Worker::makeSubstitutionGoal(StorePathOrDesc path, RepairFlag repair) { - GoalPtr goal = substitutionGoals[path].lock(); // FIXME + auto p = store.bakeCaIfNeeded(path); + GoalPtr goal = substitutionGoals[p].lock(); // FIXME if (!goal) { - goal = std::make_shared(path, *this, repair, ca); - substitutionGoals.insert_or_assign(path, goal); + auto optCA = std::get_if<1>(&path); + goal = std::make_shared( + p, + *this, + repair, + optCA ? std::optional { *optCA } : std::nullopt); + substitutionGoals.insert_or_assign(p, goal); wakeUp(goal); } return goal; @@ -5146,15 +5152,15 @@ BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDe } -void LocalStore::ensurePath(const StorePath & path) +void LocalStore::ensurePath(StorePathOrDesc path) { /* If the path is already valid, we're done. */ if (isValidPath(path)) return; - primeCache(*this, {{path}}); + // primeCache(*this, {{path}}); Worker worker(*this); - GoalPtr goal = worker.makeSubstitutionGoal(path); + GoalPtr goal = worker.makeSubstitutionGoal(path, NoRepair); Goals goals = {goal}; worker.run(goals); @@ -5163,8 +5169,10 @@ void LocalStore::ensurePath(const StorePath & path) if (goal->ex) { goal->ex->status = worker.exitStatus(); throw *goal->ex; - } else - throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); + } else { + auto p = this->bakeCaIfNeeded(path); + throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(p)); + } } } diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index c8557bda7d4a..af0780e07035 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -8,6 +8,7 @@ std::string FixedOutputHash::printMethodAlgo() const { return makeFileIngestionPrefix(method) + printHashType(hash.type); } + std::string makeFileIngestionPrefix(const FileIngestionMethod m) { switch (m) { case FileIngestionMethod::Flat: @@ -17,11 +18,9 @@ std::string makeFileIngestionPrefix(const FileIngestionMethod m) { default: throw Error("impossible, caught both cases"); } + abort(); } -// FIXME Put this somewhere? -template struct overloaded : Ts... { using Ts::operator()...; }; -template overloaded(Ts...) -> overloaded; std::string renderContentAddress(ContentAddress ca) { return std::visit(overloaded { @@ -30,13 +29,26 @@ std::string renderContentAddress(ContentAddress ca) { + th.hash.to_string(Base32, true); }, [](FixedOutputHash fsh) { - return "fixed:" - + makeFileIngestionPrefix(fsh.method) - + fsh.hash.to_string(Base32, true); + return "fixed:" + + makeFileIngestionPrefix(fsh.method) + + fsh.hash.to_string(Base32, true); } }, ca); } +static HashType parseHashType_(std::string_view & rest) { + auto hashTypeRaw = splitPrefixTo(rest, ':'); + if (!hashTypeRaw) + throw UsageError("hash must be in form \":\", but found: %s", rest); + return parseHashType(*hashTypeRaw); +}; + +static FileIngestionMethod parseFileIngestionMethod_(std::string_view & rest) { + if (splitPrefix(rest, "r:")) + return FileIngestionMethod::Recursive; + return FileIngestionMethod::Flat; +} + ContentAddress parseContentAddress(std::string_view rawCa) { auto rest = rawCa; @@ -44,22 +56,14 @@ ContentAddress parseContentAddress(std::string_view rawCa) { { auto optPrefix = splitPrefixTo(rest, ':'); if (!optPrefix) - throw UsageError("not a content address because it is not in the form \":\": %s", rawCa); + throw UsageError("not a path-info content address because it is not in the form \":\": %s", rawCa); prefix = *optPrefix; } - auto parseHashType_ = [&](){ - auto hashTypeRaw = splitPrefixTo(rest, ':'); - if (!hashTypeRaw) - throw UsageError("content address hash must be in form \":\", but found: %s", rawCa); - HashType hashType = parseHashType(*hashTypeRaw); - return std::move(hashType); - }; - // Switch on prefix if (prefix == "text") { // No parsing of the method, "text" only support flat. - HashType hashType = parseHashType_(); + HashType hashType = parseHashType_(rest); if (hashType != htSHA256) throw Error("text content address hash should use %s, but instead uses %s", printHashType(htSHA256), printHashType(hashType)); @@ -67,17 +71,14 @@ ContentAddress parseContentAddress(std::string_view rawCa) { .hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)), }; } else if (prefix == "fixed") { - // Parse method - auto method = FileIngestionMethod::Flat; - if (splitPrefix(rest, "r:")) - method = FileIngestionMethod::Recursive; - HashType hashType = parseHashType_(); + auto method = parseFileIngestionMethod_(rest); + HashType hashType = parseHashType_(rest); return FixedOutputHash { .method = method, .hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)), }; } else - throw UsageError("content address prefix \"%s\" is unrecognized. Recogonized prefixes are \"text\" or \"fixed\"", prefix); + throw UsageError("path-info content address prefix \"%s\" is unrecognized. Recogonized prefixes are \"text\", \"fixed\", or \"ipfs\"", prefix); }; std::optional parseContentAddressOpt(std::string_view rawCaOpt) { @@ -88,6 +89,120 @@ std::string renderContentAddress(std::optional ca) { return ca ? renderContentAddress(*ca) : ""; } + +// FIXME Deduplicate with store-api.cc path computation +std::string renderStorePathDescriptor(StorePathDescriptor ca) +{ + std::string result = ca.name; + + auto dumpRefs = [&](auto references, bool hasSelfReference) { + result += "refs:"; + result += std::to_string(references.size()); + for (auto & i : references) { + result += ":"; + result += i.to_string(); + } + if (hasSelfReference) result += ":self"; + }; + + std::visit(overloaded { + [&](TextInfo th) { + result += "text:"; + dumpRefs(th.references, false); + result += ":" + renderContentAddress(ContentAddress {TextHash { + .hash = th.hash, + }}); + }, + [&](FixedOutputInfo fsh) { + result += "fixed:"; + dumpRefs(fsh.references.references, fsh.references.hasSelfReference); + result += ":" + renderContentAddress(ContentAddress {FixedOutputHash { + .method = fsh.method, + .hash = fsh.hash + }}); + }, + }, ca.info); + + return result; +} + + +StorePathDescriptor parseStorePathDescriptor(std::string_view rawCa) +{ + auto rest = rawCa; + + std::string_view name; + std::string_view tag; + { + auto optName = splitPrefixTo(rest, ':'); + auto optTag = splitPrefixTo(rest, ':'); + if (!(optTag && optName)) + throw UsageError("not a content address because it is not in the form \"::\": %s", rawCa); + tag = *optTag; + name = *optName; + } + + auto parseRefs = [&]() -> PathReferences { + if (!splitPrefix(rest, "refs,")) + throw Error("Invalid CA \"%s\", \"%s\" should begin with \"refs:\"", rawCa, rest); + PathReferences ret; + size_t numReferences = 0; + { + auto countRaw = splitPrefixTo(rest, ':'); + if (!countRaw) + throw UsageError("Invalid count"); + numReferences = std::stoi(std::string { *countRaw }); + } + for (size_t i = 0; i < numReferences; i++) { + auto s = splitPrefixTo(rest, ':'); + if (!s) + throw UsageError("Missing reference no. %d", i); + ret.references.insert(StorePath(*s)); + } + if (splitPrefix(rest, "self:")) + ret.hasSelfReference = true; + return ret; + }; + + // Dummy value + ContentAddressWithReferences info = TextInfo { Hash(htSHA256), {} }; + + // Switch on tag + if (tag == "text") { + auto refs = parseRefs(); + if (refs.hasSelfReference) + throw UsageError("Text content addresses cannot have self references"); + auto hashType = parseHashType_(rest); + if (hashType != htSHA256) + throw Error("Text content address hash should use %s, but instead uses %s", + printHashType(htSHA256), printHashType(hashType)); + info = TextInfo { + { + .hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)), + }, + refs.references, + }; + } else if (tag == "fixed") { + auto refs = parseRefs(); + auto method = parseFileIngestionMethod_(rest); + auto hashType = parseHashType_(rest); + info = FixedOutputInfo { + { + .method = method, + .hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)), + }, + refs, + }; + } else + throw UsageError("content address tag \"%s\" is unrecognized. Recogonized tages are \"text\", \"fixed\", or \"ipfs\"", tag); + + return StorePathDescriptor { + .name = std::string { name }, + .info = info, + }; +} + + Hash getContentAddressHash(const ContentAddress & ca) { return std::visit(overloaded { @@ -96,7 +211,7 @@ Hash getContentAddressHash(const ContentAddress & ca) }, [](FixedOutputHash fsh) { return fsh.hash; - } + }, }, ca); } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 7d15a04fa4e9..7614362b7cbe 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -11,10 +11,11 @@ namespace nix { */ enum struct FileIngestionMethod : uint8_t { - Flat = false, - Recursive = true + Flat, + Recursive, }; + struct TextHash { Hash hash; }; @@ -66,6 +67,12 @@ struct PathReferences std::set references; bool hasSelfReference = false; + bool operator == (const PathReferences & other) const + { + return references == other.references + && hasSelfReference == other.hasSelfReference; + } + /* Functions to view references + hasSelfReference as one set, mainly for compatibility's sake. */ StorePathSet referencesPossiblyToSelf(const Ref & self) const; @@ -118,18 +125,24 @@ struct FixedOutputInfo : FixedOutputHash { PathReferences references; }; +typedef std::variant< + TextInfo, + FixedOutputInfo +> ContentAddressWithReferences; + struct StorePathDescriptor { std::string name; - std::variant< - TextInfo, - FixedOutputInfo - > info; + ContentAddressWithReferences info; bool operator < (const StorePathDescriptor & other) const { return name < other.name; + // FIXME second field } - }; +std::string renderStorePathDescriptor(StorePathDescriptor ca); + +StorePathDescriptor parseStorePathDescriptor(std::string_view rawCa); + } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 4e4e175366bf..b8019234bcbd 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -375,21 +375,24 @@ static void performOp(TunnelLogger * logger, ref store, } case wopAddToStore: { - std::string s, baseName; + HashType hashAlgo; + std::string baseName; FileIngestionMethod method; { - bool fixed; uint8_t recursive; - from >> baseName >> fixed /* obsolete */ >> recursive >> s; + bool fixed; + uint8_t recursive; + std::string hashAlgoRaw; + from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw; if (recursive > (uint8_t) FileIngestionMethod::Recursive) throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive); method = FileIngestionMethod { recursive }; /* Compatibility hack. */ if (!fixed) { - s = "sha256"; + hashAlgoRaw = "sha256"; method = FileIngestionMethod::Recursive; } + hashAlgo = parseHashType(hashAlgoRaw); } - HashType hashAlgo = parseHashType(s); StringSink savedNAR; TeeSource savedNARSource(from, savedNAR); @@ -407,8 +410,11 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); if (!savedRegular.regular) throw Error("regular file expected"); + StringSource dumpSource { + method == FileIngestionMethod::Recursive ? *savedNAR.s : savedRegular.s + }; auto path = store->addToStoreFromDump( - method == FileIngestionMethod::Recursive ? *savedNAR.s : savedRegular.s, + dumpSource, baseName, method, hashAlgo); @@ -735,10 +741,11 @@ static void performOp(TunnelLogger * logger, ref store, if (GET_PROTOCOL_MINOR(clientVersion) >= 21) source = std::make_unique(from, to); else { - TeeParseSink tee(from); - parseDump(tee, tee.source); - saved = std::move(*tee.saved.s); - source = std::make_unique(saved); + StringSink saved; + TeeSource tee { from, saved }; + ParseSink sink; + parseDump(sink, tee); + source = std::make_unique(std::move(*saved.s)); } logger->startWork(); diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 4c1bf5673090..22ab3823353c 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -60,8 +60,10 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'"); /* Extract the NAR from the source. */ - TeeParseSink tee(source); - parseDump(tee, tee.source); + StringSink saved; + TeeSource tee { source, saved }; + ParseSink ether; + parseDump(ether, tee); uint32_t magic = readInt(source); if (magic != exportMagic) @@ -77,15 +79,15 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) if (deriver != "") info.deriver = parseStorePath(deriver); - info.narHash = hashString(htSHA256, *tee.saved.s); - info.narSize = tee.saved.s->size(); + info.narHash = hashString(htSHA256, *saved.s); + info.narSize = saved.s->size(); // Ignore optional legacy signature. if (readInt(source) == 1) readString(source); // Can't use underlying source, which would have been exhausted - auto source = StringSource { *tee.saved.s }; + auto source = StringSource { *saved.s }; addToStore(info, source, NoRepair, checkSigs); res.push_back(info.path); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 683fa5196d73..72e2bce2e703 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -2,13 +2,14 @@ #include "util.hh" #include "archive.hh" #include "args.hh" +#include "sync.hh" #include #include #include #include #include - +#include namespace nix { @@ -129,8 +130,14 @@ bool Settings::isExperimentalFeatureEnabled(const std::string & name) void Settings::requireExperimentalFeature(const std::string & name) { - if (!isExperimentalFeatureEnabled(name)) - throw Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", name); + if (!isExperimentalFeatureEnabled(name)) { + if (allowExperimentalFeatures) { + static Sync> warned; + if (warned.lock()->insert(name).second) + warn("feature '%s' is experimental", name); + } else + throw Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", name); + } } bool Settings::isWSL1() diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 36f00033a598..98c97cd07db3 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -350,6 +350,10 @@ public: Setting githubAccessToken{this, "", "github-access-token", "GitHub access token to get access to GitHub data through the GitHub API for github:<..> flakes."}; + Setting allowExperimentalFeatures{this, true, "allow-experimental-features", + "Whether the use of experimental features other than those listed in " + "the option 'experimental-features' gives a warning rather than fatal error."}; + Setting experimentalFeatures{this, {}, "experimental-features", "Experimental Nix features to enable."}; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 5b7de70af0cd..20fef3299384 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -87,9 +87,10 @@ struct LegacySSHStore : public Store return uriScheme + host; } - void queryPathInfoUncached(const StorePath & path, + void queryPathInfoUncached(StorePathOrDesc pathOrDesc, Callback> callback) noexcept override { + auto path = this->bakeCaIfNeeded(pathOrDesc); try { auto conn(connections->get()); @@ -180,8 +181,9 @@ struct LegacySSHStore : public Store throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host); } - void narFromPath(const StorePath & path, Sink & sink) override + void narFromPath(StorePathOrDesc pathOrDesc, Sink & sink) override { + auto path = this->bakeCaIfNeeded(pathOrDesc); auto conn(connections->get()); conn->to << cmdDumpStorePath << printStorePath(path); @@ -233,7 +235,7 @@ struct LegacySSHStore : public Store return status; } - void ensurePath(const StorePath & path) override + void ensurePath(StorePathOrDesc desc) override { unsupported("ensurePath"); } void computeFSClosure(const StorePathSet & paths, diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 2f1d9663ad80..0e1f053b0c95 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -77,11 +77,12 @@ ref LocalFSStore::getFSAccessor() std::dynamic_pointer_cast(shared_from_this()))); } -void LocalFSStore::narFromPath(const StorePath & path, Sink & sink) +void LocalFSStore::narFromPath(const StorePathOrDesc pathOrDesc, Sink & sink) { - if (!isValidPath(path)) - throw Error("path '%s' is not valid", printStorePath(path)); - dumpPath(getRealStoreDir() + std::string(printStorePath(path), storeDir.size()), sink); + auto p = this->bakeCaIfNeeded(pathOrDesc); + if (!isValidPath(pathOrDesc)) + throw Error("path '%s' is not valid", printStorePath(p)); + dumpPath(getRealStoreDir() + std::string(printStorePath(p), storeDir.size()), sink); } const string LocalFSStore::drvsLogDir = "drvs"; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c0f34b6a4e6f..3a469de52bb9 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -629,9 +629,10 @@ uint64_t LocalStore::addValidPath(State & state, } -void LocalStore::queryPathInfoUncached(const StorePath & path, +void LocalStore::queryPathInfoUncached(StorePathOrDesc pathOrDesc, Callback> callback) noexcept { + auto path = bakeCaIfNeeded(pathOrDesc); try { auto info = std::make_shared(path); @@ -712,8 +713,9 @@ bool LocalStore::isValidPath_(State & state, const StorePath & path) } -bool LocalStore::isValidPathUncached(const StorePath & path) +bool LocalStore::isValidPathUncached(StorePathOrDesc pathOrDesc) { + auto path = bakeCaIfNeeded(pathOrDesc); return retrySQLite([&]() { auto state(_state.lock()); return isValidPath_(*state, path); @@ -1054,88 +1056,26 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, } -StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) -{ - Hash h = hashString(hashAlgo, dump); - - auto dstPath = makeFixedOutputPath(name, FixedOutputInfo { - { - .method = method, - .hash = h, - }, - {}, - }); - - addTempRoot(dstPath); - - if (repair || !isValidPath(dstPath)) { - - /* The first check above is an optimisation to prevent - unnecessary lock acquisition. */ - - auto realPath = Store::toRealPath(dstPath); - - PathLocks outputLock({realPath}); - - if (repair || !isValidPath(dstPath)) { - - deletePath(realPath); - - autoGC(); - - if (method == FileIngestionMethod::Recursive) { - StringSource source(dump); - restorePath(realPath, source); - } else - writeFile(realPath, dump); - - canonicalisePathMetaData(realPath, -1); - - /* Register the SHA-256 hash of the NAR serialisation of - the path in the database. We may just have computed it - above (if called with recursive == true and hashAlgo == - sha256); otherwise, compute it here. */ - HashResult hash = method == FileIngestionMethod::Recursive - ? HashResult { - hashAlgo == htSHA256 ? h : hashString(htSHA256, dump), - dump.size(), - } - : hashPath(htSHA256, realPath); - - optimisePath(realPath); // FIXME: combine with hashPath() - - ValidPathInfo info(dstPath); - info.narHash = hash.first; - info.narSize = hash.second; - info.ca = FixedOutputHash { .method = method, .hash = h }; - registerValidPath(info); - } - - outputLock.setDeletion(true); - } - - return dstPath; -} - - StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { Path srcPath(absPath(_srcPath)); + auto source = sinkToSource([&](Sink & sink) { + if (method == FileIngestionMethod::Recursive) + dumpPath(srcPath, sink, filter); + else + readFile(srcPath, sink); + }); + return addToStoreFromDump(*source, name, method, hashAlgo, repair); +} - if (method != FileIngestionMethod::Recursive) - return addToStoreFromDump(readFile(srcPath), name, method, hashAlgo, repair); - - /* For computing the NAR hash. */ - auto sha256Sink = std::make_unique(htSHA256); - /* For computing the store path. In recursive SHA-256 mode, this - is the same as the NAR hash, so no need to do it again. */ - std::unique_ptr hashSink = - hashAlgo == htSHA256 - ? nullptr - : std::make_unique(hashAlgo); +StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) +{ + /* For computing the store path. */ + auto hashSink = std::make_unique(hashAlgo); + TeeSource source { source0, *hashSink }; /* Read the source path into memory, but only if it's up to narBufferSize bytes. If it's larger, write it to a temporary @@ -1143,55 +1083,49 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, destination store path is already valid, we just delete the temporary path. Otherwise, we move it to the destination store path. */ - bool inMemory = true; - std::string nar; - - auto source = sinkToSource([&](Sink & sink) { - - LambdaSink sink2([&](const unsigned char * buf, size_t len) { - (*sha256Sink)(buf, len); - if (hashSink) (*hashSink)(buf, len); - - if (inMemory) { - if (nar.size() + len > settings.narBufferSize) { - inMemory = false; - sink << 1; - sink((const unsigned char *) nar.data(), nar.size()); - nar.clear(); - } else { - nar.append((const char *) buf, len); - } - } - - if (!inMemory) sink(buf, len); - }); - - dumpPath(srcPath, sink2, filter); - }); + bool inMemory = false; + + std::string dump; + + /* Fill out buffer, and decide whether we are working strictly in + memory based on whether we break out because the buffer is full + or the original source is empty */ + while (dump.size() < settings.narBufferSize) { + auto oldSize = dump.size(); + constexpr size_t chunkSize = 1024; + auto want = std::min(chunkSize, settings.narBufferSize - oldSize); + dump.resize(oldSize + want); + auto got = 0; + try { + got = source.read((uint8_t *) dump.data() + oldSize, want); + } catch (EndOfFile &) { + inMemory = true; + break; + } + dump.resize(oldSize + got); + } std::unique_ptr delTempDir; Path tempPath; - try { - /* Wait for the source coroutine to give us some dummy - data. This is so that we don't create the temporary - directory if the NAR fits in memory. */ - readInt(*source); + if (!inMemory) { + /* Drain what we pulled so far, and then keep on pulling */ + StringSource dumpSource { dump }; + ChainSource bothSource { dumpSource, source }; auto tempDir = createTempDir(realStoreDir, "add"); delTempDir = std::make_unique(tempDir); tempPath = tempDir + "/x"; - restorePath(tempPath, *source); + if (method == FileIngestionMethod::Recursive) + restorePath(tempPath, bothSource); + else + writeFile(tempPath, bothSource); - } catch (EndOfFile &) { - if (!inMemory) throw; - /* The NAR fits in memory, so we didn't do restorePath(). */ + dump.clear(); } - auto sha256 = sha256Sink->finish(); - - Hash hash = hashSink ? hashSink->finish().first : sha256.first; + auto [hash, size] = hashSink->finish(); ValidPathInfo info { *this, @@ -1206,8 +1140,6 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, }, }, }; - info.narHash = sha256.first; - info.narSize = sha256.second; addTempRoot(info.path); @@ -1227,19 +1159,33 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, autoGC(); if (inMemory) { + StringSource dumpSource { dump }; /* Restore from the NAR in memory. */ - StringSource source(nar); - restorePath(realPath, source); + if (method == FileIngestionMethod::Recursive) + restorePath(realPath, dumpSource); + else + writeFile(realPath, dumpSource); } else { /* Move the temporary path we restored above. */ if (rename(tempPath.c_str(), realPath.c_str())) throw Error("renaming '%s' to '%s'", tempPath, realPath); } + /* For computing the nar hash. In recursive SHA-256 mode, this + is the same as the store hash, so no need to do it again. */ + auto narHash = std::pair { hash, size }; + if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) { + HashSink narSink { htSHA256 }; + dumpPath(realPath, narSink); + narHash = narSink.finish(); + } + canonicalisePathMetaData(realPath, -1); // FIXME: merge into restorePath optimisePath(realPath); + info.narHash = narHash.first; + info.narSize = narHash.second; registerValidPath(info); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index aa72f203848e..39e633a25708 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -119,14 +119,14 @@ public: std::string getUri() override; - bool isValidPathUncached(const StorePath & path) override; + bool isValidPathUncached(StorePathOrDesc path) override; StorePathSet queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute = NoSubstitute) override; StorePathSet queryAllValidPaths() override; - void queryPathInfoUncached(const StorePath & path, + void queryPathInfoUncached(StorePathOrDesc, Callback> callback) noexcept override; void queryReferrers(const StorePath & path, StorePathSet & referrers) override; @@ -154,7 +154,7 @@ public: in `dump', which is either a NAR serialisation (if recursive == true) or simply the contents of a regular file (if recursive == false). */ - StorePath addToStoreFromDump(const string & dump, const string & name, + StorePath addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; StorePath addTextToStore(const string & name, const string & s, @@ -167,7 +167,7 @@ public: BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; - void ensurePath(const StorePath & path) override; + void ensurePath(StorePathOrDesc path) override; void addTempRoot(const StorePath & path) override; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 13823a933c35..7cc04f7f9765 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -33,7 +33,8 @@ void Store::computeFSClosure(const StorePathSet & startPaths, state->pending++; } - queryPathInfo(parseStorePath(path), {[&, pathS(path)](std::future> fut) { + auto p = parseStorePath(path); + queryPathInfo(p, {[&, pathS(path)](std::future> fut) { // FIXME: calls to isValidPath() should be async try { diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 58235a7fc20f..9709562e24b2 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -15,6 +15,8 @@ struct NarInfo : ValidPathInfo std::string system; NarInfo() = delete; + NarInfo(const Store & store, StorePathDescriptor && ca) + : ValidPathInfo(store, std::move(ca)) { } NarInfo(StorePath && path) : ValidPathInfo(std::move(path)) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const Store & store, const std::string & s, const std::string & whence); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index f7c91b8fbc2e..f67c1f811aa4 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -297,8 +297,9 @@ ConnectionHandle RemoteStore::getConnection() } -bool RemoteStore::isValidPathUncached(const StorePath & path) +bool RemoteStore::isValidPathUncached(StorePathOrDesc pathOrDesc) { + auto path = bakeCaIfNeeded(pathOrDesc); auto conn(getConnection()); conn->to << wopIsValidPath << printStorePath(path); conn.processStderr(); @@ -407,9 +408,10 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, const } -void RemoteStore::queryPathInfoUncached(const StorePath & path, +void RemoteStore::queryPathInfoUncached(StorePathOrDesc pathOrDesc, Callback> callback) noexcept { + auto path = bakeCaIfNeeded(pathOrDesc); try { std::shared_ptr info; { @@ -632,8 +634,9 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD } -void RemoteStore::ensurePath(const StorePath & path) +void RemoteStore::ensurePath(StorePathOrDesc pathOrDesc) { + auto path = bakeCaIfNeeded(pathOrDesc); auto conn(getConnection()); conn->to << wopEnsurePath << printStorePath(path); conn.processStderr(); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index c96ad48e7456..dc6fa18ac6c7 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -35,14 +35,14 @@ public: /* Implementations of abstract store API methods. */ - bool isValidPathUncached(const StorePath & path) override; + bool isValidPathUncached(StorePathOrDesc path) override; StorePathSet queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute = NoSubstitute) override; StorePathSet queryAllValidPaths() override; - void queryPathInfoUncached(const StorePath & path, + void queryPathInfoUncached(StorePathOrDesc, Callback> callback) noexcept override; void queryReferrers(const StorePath & path, StorePathSet & referrers) override; @@ -75,7 +75,7 @@ public: BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; - void ensurePath(const StorePath & path) override; + void ensurePath(StorePathOrDesc path) override; void addTempRoot(const StorePath & path) override; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 1b7dff085e6d..7b3365cce42f 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -226,8 +226,9 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore fetches the .narinfo file, rather than first checking for its existence via a HEAD request. Since .narinfos are small, doing a GET is unlikely to be slower than HEAD. */ - bool isValidPathUncached(const StorePath & storePath) override + bool isValidPathUncached(StorePathOrDesc storePathOrDesc) override { + auto storePath = bakeCaIfNeeded(storePathOrDesc); try { queryPathInfo(storePath); return true; diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index caae6b596c7a..3a973d93dd68 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -40,7 +40,7 @@ class SSHStore : public RemoteStore bool sameMachine() override { return false; } - void narFromPath(const StorePath & path, Sink & sink) override; + void narFromPath(StorePathOrDesc pathOrDesc, Sink & sink) override; ref getFSAccessor() override; @@ -68,8 +68,9 @@ class SSHStore : public RemoteStore }; }; -void SSHStore::narFromPath(const StorePath & path, Sink & sink) +void SSHStore::narFromPath(StorePathOrDesc pathOrDesc, Sink & sink) { + auto path = bakeCaIfNeeded(pathOrDesc); auto conn(connections->get()); conn->to << wopNarFromPath << printStorePath(path); conn->processStderr(); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index fa2614594339..3457678cd938 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -7,6 +7,7 @@ #include "json.hh" #include "derivations.hh" #include "url.hh" +#include "references.hh" #include "archive.hh" #include @@ -174,6 +175,17 @@ static std::string makeType( return std::move(type); } +StorePath Store::bakeCaIfNeeded(StorePathOrDesc path) const +{ + return std::visit(overloaded { + [this](std::reference_wrapper storePath) { + return StorePath {storePath}; + }, + [this](std::reference_wrapper ca) { + return makeFixedOutputPathFromCA(ca); + }, + }, path); +} StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { @@ -202,10 +214,6 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons } -// FIXME Put this somewhere? -template struct overloaded : Ts... { using Ts::operator()...; }; -template overloaded(Ts...) -> overloaded; - StorePath Store::makeFixedOutputPathFromCA(const StorePathDescriptor & info) const { // New template @@ -323,9 +331,9 @@ StorePathSet Store::queryDerivationOutputs(const StorePath & path) return outputPaths; } -bool Store::isValidPath(const StorePath & storePath) +bool Store::isValidPath(StorePathOrDesc storePath) { - std::string hashPart(storePath.hashPart()); + std::string hashPart { bakeCaIfNeeded(storePath).hashPart() }; { auto state_(state.lock()); @@ -359,7 +367,7 @@ bool Store::isValidPath(const StorePath & storePath) /* Default implementation for stores that only implement queryPathInfoUncached(). */ -bool Store::isValidPathUncached(const StorePath & path) +bool Store::isValidPathUncached(StorePathOrDesc path) { try { queryPathInfo(path); @@ -370,7 +378,7 @@ bool Store::isValidPathUncached(const StorePath & path) } -ref Store::queryPathInfo(const StorePath & storePath) +ref Store::queryPathInfo(StorePathOrDesc storePath) { std::promise> promise; @@ -395,11 +403,13 @@ static bool goodStorePath(const StorePath & expected, const StorePath & actual) } -void Store::queryPathInfo(const StorePath & storePath, +void Store::queryPathInfo(StorePathOrDesc pathOrCa, Callback> callback) noexcept { std::string hashPart; + auto storePath = bakeCaIfNeeded(pathOrCa); + try { hashPart = storePath.hashPart(); @@ -433,8 +443,8 @@ void Store::queryPathInfo(const StorePath & storePath, auto callbackPtr = std::make_shared(std::move(callback)); - queryPathInfoUncached(storePath, - {[this, storePathS{printStorePath(storePath)}, hashPart, callbackPtr](std::future> fut) { + queryPathInfoUncached(pathOrCa, + {[this, storePath, hashPart, callbackPtr](std::future> fut) { try { auto info = fut.get(); @@ -447,11 +457,9 @@ void Store::queryPathInfo(const StorePath & storePath, state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info }); } - auto storePath = parseStorePath(storePathS); - if (!info || !goodStorePath(storePath, info->path)) { stats.narInfoMissing++; - throw InvalidPath("path '%s' is not valid", storePathS); + throw InvalidPath("path '%s' is not valid", printStorePath(storePath)); } (*callbackPtr)(ref(info)); @@ -474,13 +482,13 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m std::condition_variable wakeup; ThreadPool pool; - auto doQuery = [&](const Path & path) { + auto doQuery = [&](const StorePath & path) { checkInterrupt(); - queryPathInfo(parseStorePath(path), {[path, this, &state_, &wakeup](std::future> fut) { + queryPathInfo(path, {[path, this, &state_, &wakeup](std::future> fut) { auto state(state_.lock()); try { auto info = fut.get(); - state->valid.insert(parseStorePath(path)); + state->valid.insert(path); } catch (InvalidPath &) { } catch (...) { state->exc = std::current_exception(); @@ -492,7 +500,7 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m }; for (auto & path : paths) - pool.enqueue(std::bind(doQuery, printStorePath(path))); // FIXME + pool.enqueue(std::bind(doQuery, path)); pool.process(); @@ -654,18 +662,22 @@ void Store::buildPaths(const std::vector & paths, BuildMod void copyStorePath(ref srcStore, ref dstStore, - const StorePath & storePath, RepairFlag repair, CheckSigsFlag checkSigs) + StorePathOrDesc storePath, RepairFlag repair, CheckSigsFlag checkSigs) { auto srcUri = srcStore->getUri(); auto dstUri = dstStore->getUri(); + // FIXME Use CA when we have it in messages below + + auto actualStorePath = srcStore->bakeCaIfNeeded(storePath); + Activity act(*logger, lvlInfo, actCopyPath, srcUri == "local" || srcUri == "daemon" - ? fmt("copying path '%s' to '%s'", srcStore->printStorePath(storePath), dstUri) + ? fmt("copying path '%s' to '%s'", srcStore->printStorePath(actualStorePath), dstUri) : dstUri == "local" || dstUri == "daemon" - ? fmt("copying path '%s' from '%s'", srcStore->printStorePath(storePath), srcUri) - : fmt("copying path '%s' from '%s' to '%s'", srcStore->printStorePath(storePath), srcUri, dstUri), - {srcStore->printStorePath(storePath), srcUri, dstUri}); + ? fmt("copying path '%s' from '%s'", srcStore->printStorePath(actualStorePath), srcUri) + : fmt("copying path '%s' from '%s' to '%s'", srcStore->printStorePath(actualStorePath), srcUri, dstUri), + {srcStore->printStorePath(actualStorePath), srcUri, dstUri}); PushActivity pact(act.id); auto info = srcStore->queryPathInfo(storePath); @@ -673,19 +685,36 @@ void copyStorePath(ref srcStore, ref dstStore, uint64_t total = 0; // recompute store path on the chance dstStore does it differently - if (info->ca && info->references.empty()) { - auto info2 = make_ref(*info); - info2->path = dstStore->makeFixedOutputPathFromCA(*info->fullStorePathDescriptorOpt()); - if (dstStore->storeDir == srcStore->storeDir) - assert(info->path == info2->path); - info = info2; + if (auto p = std::get_if>(&storePath)) { + auto ca = static_cast(*p); + // { + // ValidPathInfo srcInfoCA { *srcStore, StorePathDescriptor { ca } }; + // assert((PathReferences &)(*info) == (PathReferences &)srcInfoCA); + // } + if (info->references.empty()) { + auto info2 = make_ref(*info); + ValidPathInfo dstInfoCA { *dstStore, StorePathDescriptor { ca } }; + if (dstStore->storeDir == srcStore->storeDir) + assert(info2->path == info2->path); + info2->path = std::move(dstInfoCA.path); + info2->ca = std::move(dstInfoCA.ca); + info = info2; + } } if (!info->narHash) { StringSink sink; - srcStore->narFromPath({storePath}, sink); + srcStore->narFromPath(storePath, sink); auto info2 = make_ref(*info); - info2->narHash = hashString(htSHA256, *sink.s); + + std::unique_ptr hashSink; + if (!info->ca || !info->hasSelfReference) + hashSink = std::make_unique(htSHA256); + else + hashSink = std::make_unique(htSHA256, std::string(info->path.hashPart())); + (*hashSink)((unsigned char *) sink.s->data(), sink.s->size()); + info2->narHash = hashSink->finish().first; + if (!info->narSize) info2->narSize = sink.s->size(); if (info->ultimate) info2->ultimate = false; info = info2; @@ -709,7 +738,7 @@ void copyStorePath(ref srcStore, ref dstStore, }); srcStore->narFromPath(storePath, wrapperSink); }, [&]() { - throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", srcStore->printStorePath(storePath), srcStore->getUri()); + throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", srcStore->printStorePath(actualStorePath), srcStore->getUri()); }); dstStore->addToStore(*info, *source, repair, checkSigs); @@ -911,12 +940,12 @@ std::optional ValidPathInfo::fullStorePathDescriptorOpt() c TextInfo info { th }; assert(!hasSelfReference); info.references = references; - return std::variant { info }; + return ContentAddressWithReferences { info }; }, [&](FixedOutputHash foh) { FixedOutputInfo info { foh }; info.references = static_cast>(*this); - return std::variant { info }; + return ContentAddressWithReferences { info }; }, }, *ca), }; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index cdbc2d188fd6..6bd8104a0501 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -145,6 +145,12 @@ struct BuildResult } }; +/* Useful for many store functions which can take advantage of content + addresses or work with regular store paths */ +typedef std::variant< + std::reference_wrapper, + std::reference_wrapper +> StorePathOrDesc; class Store : public std::enable_shared_from_this, public Config { @@ -256,6 +262,8 @@ public: StorePath makeFixedOutputPathFromCA(const StorePathDescriptor & info) const; + StorePath bakeCaIfNeeded(StorePathOrDesc path) const; + /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store path and the cryptographic hash of the contents of srcPath. */ @@ -281,11 +289,11 @@ public: const StorePathSet & references) const; /* Check whether a path is valid. */ - bool isValidPath(const StorePath & path); + bool isValidPath(StorePathOrDesc desc); protected: - virtual bool isValidPathUncached(const StorePath & path); + virtual bool isValidPathUncached(StorePathOrDesc desc); public: @@ -307,15 +315,15 @@ public: /* Query information about a valid path. It is permitted to omit the name part of the store path. */ - ref queryPathInfo(const StorePath & path); + ref queryPathInfo(StorePathOrDesc path); /* Asynchronous version of queryPathInfo(). */ - void queryPathInfo(const StorePath & path, + void queryPathInfo(StorePathOrDesc path, Callback> callback) noexcept; protected: - virtual void queryPathInfoUncached(const StorePath & path, + virtual void queryPathInfoUncached(StorePathOrDesc path, Callback> callback) noexcept = 0; public: @@ -373,7 +381,7 @@ public: std::optional expectedCAHash = {}); // FIXME: remove? - virtual StorePath addToStoreFromDump(const string & dump, const string & name, + virtual StorePath addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) { throw Error("addToStoreFromDump() is not supported by this store"); @@ -385,7 +393,7 @@ public: const StorePathSet & references, RepairFlag repair = NoRepair) = 0; /* Write a NAR dump of a store path. */ - virtual void narFromPath(const StorePath & path, Sink & sink) = 0; + virtual void narFromPath(StorePathOrDesc desc, Sink & sink) = 0; /* For each path, if it's a derivation, build it. Building a derivation means ensuring that the output paths are valid. If @@ -408,7 +416,7 @@ public: /* Ensure that a path is valid. If it is not currently valid, it may be made valid by running a substitute (if defined for the path). */ - virtual void ensurePath(const StorePath & path) = 0; + virtual void ensurePath(StorePathOrDesc desc) = 0; /* Add a store path as a temporary root of the garbage collector. The root disappears as soon as we exit. */ @@ -627,7 +635,8 @@ public: LocalFSStore(const Params & params); - void narFromPath(const StorePath & path, Sink & sink) override; + void narFromPath(StorePathOrDesc path, Sink & sink) override; + ref getFSAccessor() override; /* Register a permanent GC root. */ @@ -648,7 +657,8 @@ public: /* Copy a path from one store to another. */ void copyStorePath(ref srcStore, ref dstStore, - const StorePath & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); + StorePathOrDesc storePath, + RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); /* Copy store paths from one store to another. The paths may be copied diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 302b1bb18a07..32d98a6104df 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -63,14 +63,6 @@ struct ParseSink virtual void createSymlink(const Path & path, const string & target) { }; }; -struct TeeParseSink : ParseSink -{ - StringSink saved; - TeeSource source; - - TeeParseSink(Source & source) : source(source, saved) { } -}; - void parseDump(ParseSink & sink, Source & source); void restorePath(const Path & path, Source & source); diff --git a/src/libutil/error.hh b/src/libutil/error.hh index f4b3f11bb77a..2671007a78fb 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -199,9 +199,8 @@ public: template SysError(const Args & ... args) - : Error("") + : Error(""), errNo(errno) { - errNo = errno; auto hf = hintfmt(args...); err.hint = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo)); } diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index c8b71188fe0f..ceb53d133c16 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -92,6 +92,10 @@ void Source::operator () (unsigned char * data, size_t len) } } +void Source::operator () (std::basic_string_view & data) +{ + (*this)((unsigned char *) data.data(), data.size()); +} std::string Source::drain() { @@ -322,5 +326,18 @@ void StringSink::operator () (const unsigned char * data, size_t len) s->append((const char *) data, len); } +size_t ChainSource::read(unsigned char * data, size_t len) +{ + if (useSecond) { + return source2.read(data, len); + } else { + try { + return source1.read(data, len); + } catch (EndOfFile &) { + useSecond = true; + return this->read(data, len); + } + } +} } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 8386a499124f..c2809b5a18d2 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -54,6 +54,7 @@ struct Source It blocks until all the requested data is available, or throws an error if it is not going to be available. */ void operator () (unsigned char * data, size_t len); + void operator () (std::basic_string_view & data); /* Store up to ‘len’ in the buffer pointed to by ‘data’, and return the number of bytes stored. It blocks until at least @@ -189,11 +190,12 @@ struct TeeSource : Source size_t read(unsigned char * data, size_t len) { size_t n = orig.read(data, len); - sink(data, len); + sink(data, n); return n; } }; + /* A reader that consumes the original Source until 'size'. */ struct SizedSource : Source { @@ -256,6 +258,19 @@ struct LambdaSource : Source } }; +/* Chain two sources together so after the first is exhausted, the second is + used */ +struct ChainSource : Source +{ + Source & source1, & source2; + bool useSecond = false; + ChainSource(Source & s1, Source & s2) + : source1(s1), source2(s2) + { } + + size_t read(unsigned char * data, size_t len) override; +}; + /* Convert a function that feeds data into a Sink into a Source. The Source executes the function as a coroutine. */ diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 3af485fa0feb..2170e4c93841 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -4,6 +4,7 @@ #include #include +#include #include #include diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 3641daaec8c7..d38657843ea7 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -597,4 +597,9 @@ constexpr auto enumerate(T && iterable) } +// C++17 std::visit boilerplate +template struct overloaded : Ts... { using Ts::operator()...; }; +template overloaded(Ts...) -> overloaded; + + } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index c992b7d740d2..592a10a1f7f9 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -703,7 +703,8 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) printMissing(globals.state->store, {{globals.state->store->parseStorePath(drv.queryOutPath())}}); if (globals.dryRun) return; - globals.state->store->ensurePath(globals.state->store->parseStorePath(drv.queryOutPath())); + auto path = globals.state->store->parseStorePath(drv.queryOutPath()); + globals.state->store->ensurePath(path); } debug(format("switching to new user environment")); diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 8e7f09e1244a..c143755c06f2 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -80,7 +80,8 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, /* This is only necessary when installing store paths, e.g., `nix-env -i /nix/store/abcd...-foo'. */ state.store->addTempRoot(state.store->parseStorePath(j.second)); - state.store->ensurePath(state.store->parseStorePath(j.second)); + auto path = state.store->parseStorePath(j.second); + state.store->ensurePath(path); references.insert(state.store->parseStorePath(j.second)); } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 887c1844518f..320e2c82c2e2 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -194,10 +194,10 @@ static void opAddFixed(Strings opFlags, Strings opArgs) /* Hack to support caching in `nix-prefetch-url'. */ static void opPrintFixedPath(Strings opFlags, Strings opArgs) { - auto recursive = FileIngestionMethod::Flat; + auto method = FileIngestionMethod::Flat; for (auto i : opFlags) - if (i == "--recursive") recursive = FileIngestionMethod::Recursive; + if (i == "--recursive") method = FileIngestionMethod::Recursive; else throw UsageError("unknown flag '%1%'", i); if (opArgs.size() != 3) @@ -210,7 +210,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputInfo { { - .method = recursive, + .method = method, .hash = Hash::parseAny(hash, hashAlgo), }, {}, @@ -355,7 +355,8 @@ static void opQuery(Strings opFlags, Strings opArgs) case qDeriver: for (auto & i : opArgs) { - auto info = store->queryPathInfo(store->followLinksToStorePath(i)); + auto path = store->followLinksToStorePath(i); + auto info = store->queryPathInfo(path); cout << fmt("%s\n", info->deriver ? store->printStorePath(*info->deriver) : "unknown-deriver"); } break; @@ -880,9 +881,11 @@ static void opServe(Strings opFlags, Strings opArgs) break; } - case cmdDumpStorePath: - store->narFromPath(store->parseStorePath(readString(in)), out); + case cmdDumpStorePath: { + auto path = store->parseStorePath(readString(in)); + store->narFromPath(path, out); break; + } case cmdImportPaths: { if (!writeAllowed) throw Error("importing paths is not allowed"); diff --git a/src/nix/command.cc b/src/nix/command.cc index 3651a9e9ca75..b4691e8adc87 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -28,7 +28,8 @@ ref StoreCommand::createStore() void StoreCommand::run() { - run(getStore()); + auto store = getStore(); + run(store); } StorePathsCommand::StorePathsCommand(bool recursive) diff --git a/src/nix/repl.cc b/src/nix/repl.cc index fdacf604b5c5..41cfc32aeec3 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -375,7 +375,8 @@ Path NixRepl::getDerivationPath(Value & v) { if (!drvInfo) throw Error("expression does not evaluate to a derivation, so I can't build it"); Path drvPath = drvInfo->queryDrvPath(); - if (drvPath == "" || !state->store->isValidPath(state->store->parseStorePath(drvPath))) + auto path = state->store->parseStorePath(drvPath); + if (drvPath == "" || !state->store->isValidPath(path)) throw Error("expression did not evaluate to a valid derivation"); return drvPath; } diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index a880bdae097f..c2439cc8aef8 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -133,7 +133,8 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand !hasSuffix(userEnv, "user-environment")) throw Error("directory '%s' does not appear to be part of a Nix profile", where); - if (!store->isValidPath(store->parseStorePath(userEnv))) + auto path = store->parseStorePath(userEnv); + if (!store->isValidPath(path)) throw Error("directory '%s' is not in the Nix store", userEnv); return profileDir; diff --git a/src/nix/verify.cc b/src/nix/verify.cc index fc7a9765c43e..397af8295cc2 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -73,14 +73,14 @@ struct CmdVerify : StorePathsCommand ThreadPool pool; - auto doPath = [&](const Path & storePath) { + auto doPath = [&](const StorePath & storePath) { try { checkInterrupt(); MaintainCount> mcActive(active); update(); - auto info = store->queryPathInfo(store->parseStorePath(storePath)); + auto info = store->queryPathInfo(storePath); // Note: info->path can be different from storePath // for binary cache stores when using --all (since we @@ -178,7 +178,7 @@ struct CmdVerify : StorePathsCommand }; for (auto & storePath : storePaths) - pool.enqueue(std::bind(doPath, store->printStorePath(storePath))); + pool.enqueue(std::bind(doPath, storePath)); pool.process(); From c9f83f25088222642373a086bba18b043004b634 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 21 Jul 2020 22:57:15 +0000 Subject: [PATCH 21/31] Fix issues with store path descriptor round trip --- src/libfetchers/cache.cc | 5 +++-- src/libstore/content-address.cc | 23 ++++++++++------------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index 527fc0b06a79..9ceee365ac37 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -51,13 +51,14 @@ struct CacheImpl : Cache ref store, const Attrs & inAttrs, const Attrs & infoAttrs, - const StorePathDescriptor & storePath, + const StorePathDescriptor & storePathDesc, bool immutable) override { _state.lock()->add.use() (attrsToJson(inAttrs).dump()) (attrsToJson(infoAttrs).dump()) - (store->printStorePath(store->makeFixedOutputPathFromCA(storePath))) + // FIXME should use JSON for store path descriptor + (renderStorePathDescriptor(storePathDesc)) (immutable) (time(0)).exec(); } diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index af0780e07035..bed155b888d0 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -15,8 +15,6 @@ std::string makeFileIngestionPrefix(const FileIngestionMethod m) { return ""; case FileIngestionMethod::Recursive: return "r:"; - default: - throw Error("impossible, caught both cases"); } abort(); } @@ -78,7 +76,7 @@ ContentAddress parseContentAddress(std::string_view rawCa) { .hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)), }; } else - throw UsageError("path-info content address prefix \"%s\" is unrecognized. Recogonized prefixes are \"text\", \"fixed\", or \"ipfs\"", prefix); + throw UsageError("path-info content address prefix \"%s\" is unrecognized. Recogonized prefixes are \"text\" or \"fixed\"", prefix); }; std::optional parseContentAddressOpt(std::string_view rawCaOpt) { @@ -93,7 +91,8 @@ std::string renderContentAddress(std::optional ca) { // FIXME Deduplicate with store-api.cc path computation std::string renderStorePathDescriptor(StorePathDescriptor ca) { - std::string result = ca.name; + std::string result { ca.name }; + result += ":"; auto dumpRefs = [&](auto references, bool hasSelfReference) { result += "refs:"; @@ -103,23 +102,20 @@ std::string renderStorePathDescriptor(StorePathDescriptor ca) result += i.to_string(); } if (hasSelfReference) result += ":self"; + result += ":"; }; std::visit(overloaded { [&](TextInfo th) { result += "text:"; dumpRefs(th.references, false); - result += ":" + renderContentAddress(ContentAddress {TextHash { - .hash = th.hash, - }}); + result += th.hash.to_string(Base32, true); }, [&](FixedOutputInfo fsh) { result += "fixed:"; dumpRefs(fsh.references.references, fsh.references.hasSelfReference); - result += ":" + renderContentAddress(ContentAddress {FixedOutputHash { - .method = fsh.method, - .hash = fsh.hash - }}); + result += makeFileIngestionPrefix(fsh.method); + result += fsh.hash.to_string(Base32, true); }, }, ca.info); @@ -129,6 +125,7 @@ std::string renderStorePathDescriptor(StorePathDescriptor ca) StorePathDescriptor parseStorePathDescriptor(std::string_view rawCa) { + warn("%s", rawCa); auto rest = rawCa; std::string_view name; @@ -143,7 +140,7 @@ StorePathDescriptor parseStorePathDescriptor(std::string_view rawCa) } auto parseRefs = [&]() -> PathReferences { - if (!splitPrefix(rest, "refs,")) + if (!splitPrefix(rest, "refs:")) throw Error("Invalid CA \"%s\", \"%s\" should begin with \"refs:\"", rawCa, rest); PathReferences ret; size_t numReferences = 0; @@ -194,7 +191,7 @@ StorePathDescriptor parseStorePathDescriptor(std::string_view rawCa) refs, }; } else - throw UsageError("content address tag \"%s\" is unrecognized. Recogonized tages are \"text\", \"fixed\", or \"ipfs\"", tag); + throw UsageError("content address tag \"%s\" is unrecognized. Recogonized tages are \"text\" or \"fixed\"", tag); return StorePathDescriptor { .name = std::string { name }, From 8d01e9fb343cb6eff58b5abf97d8de87e87fd999 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Tue, 4 Aug 2020 11:28:49 -0400 Subject: [PATCH 22/31] Move back code to minimize diff size --- src/libstore/path-info.hh | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index a9ee05dcb0d6..f8ba019cb25f 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -20,6 +20,16 @@ namespace nix { class Store; + +struct SubstitutablePathInfo : PathReferences +{ + std::optional deriver; + uint64_t downloadSize; /* 0 = unknown or inapplicable */ + uint64_t narSize; /* 0 = unknown */ +}; + +typedef std::map SubstitutablePathInfos; + struct ValidPathInfo : PathReferences { StorePath path; @@ -109,13 +119,4 @@ struct ValidPathInfo : PathReferences typedef list ValidPathInfos; - -struct SubstitutablePathInfo : PathReferences -{ - std::optional deriver; - uint64_t downloadSize; /* 0 = unknown or inapplicable */ - uint64_t narSize; /* 0 = unknown */ -}; - -typedef std::map SubstitutablePathInfos; } From 07f91346668774da9a201259fad135d924cf29fe Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Tue, 4 Aug 2020 11:36:27 -0400 Subject: [PATCH 23/31] Remove unused imports --- src/libstore/path-info.hh | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index f8ba019cb25f..9046dfc1a514 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -1,17 +1,9 @@ #pragma once -// TODO many of thes eare not needed. #include "path.hh" #include "hash.hh" #include "content-address.hh" -#include "serialise.hh" -#include "crypto.hh" -#include "lru-cache.hh" -#include "sync.hh" -#include "globals.hh" -#include "config.hh" - -#include + #include #include From 535ad25a58db36a1edb1696e91717a368cd61f5d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 7 Aug 2020 14:02:57 +0000 Subject: [PATCH 24/31] Fix perl bindings build --- perl/lib/Nix/Store.xs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index f5f0eea0d183..203b7eb09b93 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -59,7 +59,8 @@ void setVerbosity(int level) int isValidPath(char * path) CODE: try { - RETVAL = store()->isValidPath(store()->parseStorePath(path)); + auto storePath = store()->parseStorePath(path); + RETVAL = store()->isValidPath(storePath); } catch (Error & e) { croak("%s", e.what()); } @@ -70,7 +71,8 @@ int isValidPath(char * path) SV * queryReferences(char * path) PPCODE: try { - for (auto & i : store()->queryPathInfo(store()->parseStorePath(path))->references) + auto storePath = store()->parseStorePath(path); + for (auto & i : store()->queryPathInfo(storePath)->referencesPossiblyToSelf()) XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -80,7 +82,8 @@ SV * queryReferences(char * path) SV * queryPathHash(char * path) PPCODE: try { - auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash->to_string(Base32, true); + auto storePath = store()->parseStorePath(path); + auto s = store()->queryPathInfo(storePath)->narHash->to_string(Base32, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -90,7 +93,8 @@ SV * queryPathHash(char * path) SV * queryDeriver(char * path) PPCODE: try { - auto info = store()->queryPathInfo(store()->parseStorePath(path)); + auto storePath = store()->parseStorePath(path); + auto info = store()->queryPathInfo(storePath); if (!info->deriver) XSRETURN_UNDEF; XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0))); } catch (Error & e) { @@ -101,7 +105,8 @@ SV * queryDeriver(char * path) SV * queryPathInfo(char * path, int base32) PPCODE: try { - auto info = store()->queryPathInfo(store()->parseStorePath(path)); + auto storePath = store()->parseStorePath(path); + auto info = store()->queryPathInfo(storePath); if (!info->deriver) XPUSHs(&PL_sv_undef); else @@ -111,7 +116,7 @@ SV * queryPathInfo(char * path, int base32) mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); AV * arr = newAV(); - for (auto & i : info->references) + for (auto & i : info->referencesPossiblyToSelf()) av_push(arr, newSVpv(store()->printStorePath(i).c_str(), 0)); XPUSHs(sv_2mortal(newRV((SV *) arr))); } catch (Error & e) { From e04cf6b6bba4520c41353cd97545c90927ba6f43 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 9 Sep 2020 03:47:26 +0000 Subject: [PATCH 25/31] Allow `copyPaths` and `queryValidPaths` to work with StorePathDesc --- src/libstore/binary-cache-store.cc | 2 +- src/libstore/legacy-ssh-store.cc | 14 +++- src/libstore/local-store.cc | 6 +- src/libstore/local-store.hh | 2 +- src/libstore/remote-store.cc | 19 +++-- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.cc | 117 +++++++++++++++++++---------- src/libstore/store-api.hh | 16 +++- 8 files changed, 120 insertions(+), 58 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index bd4153e0b2cd..096a3c1e96b0 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -108,7 +108,7 @@ std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath) void BinaryCacheStore::writeNarInfo(ref narInfo) { - auto narInfoFile = narInfoFileFor(bakeCaIfNeeded(narInfo->path)); + auto narInfoFile = narInfoFileFor(narInfo->path); upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo"); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 07cf2af22a69..63a861c631f3 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -309,19 +309,27 @@ struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig out.insert(i); } - StorePathSet queryValidPaths(const StorePathSet & paths, + std::set queryValidPaths(const std::set & paths, SubstituteFlag maybeSubstitute = NoSubstitute) override { auto conn(connections->get()); + StorePathSet paths2; + for (auto & pathOrDesc : paths) + paths2.insert(bakeCaIfNeeded(pathOrDesc)); + conn->to << cmdQueryValidPaths << false // lock << maybeSubstitute; - WorkerProto::write(*this, conn->to, paths); + WorkerProto::write(*this, conn->to, paths2); conn->to.flush(); - return WorkerProto::read(*this, conn->from); + auto res = WorkerProto::read(*this, conn->from); + std::set res2; + for (auto & r : res) + res2.insert(r); + return res2; } void connect() override diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 6ba9c4b89cbb..302fb24f08b5 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -754,11 +754,11 @@ bool LocalStore::isValidPathUncached(StorePathOrDesc pathOrDesc) } -StorePathSet LocalStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) +std::set LocalStore::queryValidPaths(const std::set & paths, SubstituteFlag maybeSubstitute) { - StorePathSet res; + std::set res; for (auto & i : paths) - if (isValidPath(i)) res.insert(i); + if (isValidPath(borrowStorePathOrDesc(i))) res.insert(i); return res; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 17e388b1529c..355e8b2ef7c6 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -125,7 +125,7 @@ public: bool isValidPathUncached(StorePathOrDesc path) override; - StorePathSet queryValidPaths(const StorePathSet & paths, + std::set queryValidPaths(const std::set & paths, SubstituteFlag maybeSubstitute = NoSubstitute) override; StorePathSet queryAllValidPaths() override; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index f7742d1530c2..7d626820f661 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -302,19 +302,28 @@ bool RemoteStore::isValidPathUncached(StorePathOrDesc pathOrDesc) } -StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) +std::set RemoteStore::queryValidPaths(const std::set & paths, SubstituteFlag maybeSubstitute) { auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { - StorePathSet res; + std::set res; for (auto & i : paths) - if (isValidPath(i)) res.insert(i); + if (isValidPath(borrowStorePathOrDesc(i))) + res.insert(i); return res; } else { + StorePathSet paths2; + for (auto & pathOrDesc : paths) + paths2.insert(bakeCaIfNeeded(pathOrDesc)); + // FIXME make new version to take advantage of desc case conn->to << wopQueryValidPaths; - WorkerProto::write(*this, conn->to, paths); + WorkerProto::write(*this, conn->to, paths2); conn.processStderr(); - return WorkerProto::read(*this, conn->from); + auto res = WorkerProto::read(*this, conn->from); + std::set res2; + for (auto & r : res) + res2.insert(r); + return res2; } } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 04efceb8bcd6..7acff61f74ae 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -41,7 +41,7 @@ public: bool isValidPathUncached(StorePathOrDesc path) override; - StorePathSet queryValidPaths(const StorePathSet & paths, + std::set queryValidPaths(const std::set & paths, SubstituteFlag maybeSubstitute = NoSubstitute) override; StorePathSet queryAllValidPaths() override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 8a07b17f99c2..d5e4c33aa93e 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -180,6 +180,11 @@ static std::string makeType( return std::move(type); } +StorePath Store::bakeCaIfNeeded(const OwnedStorePathOrDesc & path) const +{ + return bakeCaIfNeeded(borrowStorePathOrDesc(path)); +} + StorePath Store::bakeCaIfNeeded(StorePathOrDesc path) const { return std::visit(overloaded { @@ -192,6 +197,15 @@ StorePath Store::bakeCaIfNeeded(StorePathOrDesc path) const }, path); } +StorePathOrDesc borrowStorePathOrDesc(const OwnedStorePathOrDesc & storePathOrDesc) { + // Can't use std::visit as it would copy :( + if (auto p = std::get_if(&storePathOrDesc)) + return *p; + if (auto p = std::get_if(&storePathOrDesc)) + return *p; + abort(); +} + StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) { @@ -552,22 +566,38 @@ void Store::queryPathInfo(StorePathOrDesc pathOrCa, StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) +{ + std::set paths2; + for (auto & p : paths) + paths2.insert(p); + auto res = queryValidPaths(paths2, maybeSubstitute); + StorePathSet res2; + for (auto & r : res) { + auto p = std::get_if(&r); + assert(p); + res2.insert(*p); + } + return res2; +} + +std::set Store::queryValidPaths(const std::set & paths, SubstituteFlag maybeSubstitute) { struct State { size_t left; - StorePathSet valid; + std::set valid; std::exception_ptr exc; }; - Sync state_(State{paths.size(), StorePathSet()}); + Sync state_(State{paths.size(), {}}); std::condition_variable wakeup; ThreadPool pool; - auto doQuery = [&](const StorePath & path) { + auto doQuery = [&](const OwnedStorePathOrDesc & path) { checkInterrupt(); - queryPathInfo(path, {[path, this, &state_, &wakeup](std::future> fut) { + auto path2 = borrowStorePathOrDesc(path); + queryPathInfo(path2, {[path, this, &state_, &wakeup](std::future> fut) { auto state(state_.lock()); try { auto info = fut.get(); @@ -806,20 +836,25 @@ void copyStorePath(ref srcStore, ref dstStore, } -std::map copyPaths(ref srcStore, ref dstStore, const StorePathSet & storePaths, +void copyPaths(ref srcStore, ref dstStore, const StorePathSet & storePaths, + RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) +{ + std::set storePaths2; + for (auto & p : storePaths) + storePaths2.insert(p); + return copyPaths(srcStore, dstStore, storePaths2, repair, checkSigs, substitute); +} + +void copyPaths(ref srcStore, ref dstStore, const std::set & storePaths, RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) { auto valid = dstStore->queryValidPaths(storePaths, substitute); - StorePathSet missing; + std::set missing; for (auto & path : storePaths) if (!valid.count(path)) missing.insert(path); - std::map pathsMap; - for (auto & path : storePaths) - pathsMap.insert_or_assign(path, path); - - if (missing.empty()) return pathsMap; + if (missing.empty()) return; Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); @@ -834,57 +869,57 @@ std::map copyPaths(ref srcStore, ref dstStor ThreadPool pool; - processGraph(pool, - StorePathSet(missing.begin(), missing.end()), - - [&](const StorePath & storePath) { - auto info = srcStore->queryPathInfo(storePath); - auto storePathForDst = storePath; - if (info->ca && info->references.empty() && !info->hasSelfReference) { - storePathForDst = dstStore->makeFixedOutputPathFromCA(*info->fullStorePathDescriptorOpt()); - if (dstStore->storeDir == srcStore->storeDir) - assert(storePathForDst == storePath); - if (storePathForDst != storePath) - debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); + processGraph(pool, + std::set(missing.begin(), missing.end()), + + [&](const OwnedStorePathOrDesc & storePathOrDesc) -> std::set { + auto info = srcStore->queryPathInfo(borrowStorePathOrDesc(storePathOrDesc)); + + /* If we can "upgrade" the node into a descriptor, do that. */ + if (auto storePathP = std::get_if(&storePathOrDesc)) { + if (auto descOpt = info->fullStorePathDescriptorOpt()) { + auto & desc = *descOpt; + debug("found CA description for path '%s'", srcStore->printStorePath(*storePathP)); + return { desc }; + } } - pathsMap.insert_or_assign(storePath, storePathForDst); - if (dstStore->isValidPath(storePath)) { + if (dstStore->isValidPath(borrowStorePathOrDesc(storePathOrDesc))) { nrDone++; showProgress(); - return StorePathSet(); + return {}; } bytesExpected += info->narSize; act.setExpected(actCopyPath, bytesExpected); - return info->references; + std::set res; + for (auto & i : info->references) + res.insert(i); + return res; }, - [&](const StorePath & storePath) { + [&](const OwnedStorePathOrDesc & storePathOrDesc) { checkInterrupt(); - auto info = srcStore->queryPathInfo(storePath); + auto storePathOrDescB = borrowStorePathOrDesc(storePathOrDesc); - auto storePathForDst = storePath; - if (info->ca && info->references.empty() && !info->hasSelfReference) { - storePathForDst = dstStore->makeFixedOutputPathFromCA(*info->fullStorePathDescriptorOpt()); - if (dstStore->storeDir == srcStore->storeDir) - assert(storePathForDst == storePath); - if (storePathForDst != storePath) - debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); - } - pathsMap.insert_or_assign(storePath, storePathForDst); + auto info = srcStore->queryPathInfo(storePathOrDescB); + + auto descOpt = info->fullStorePathDescriptorOpt(); + if (descOpt) + storePathOrDescB = *descOpt; - if (!dstStore->isValidPath(storePathForDst)) { + if (!dstStore->isValidPath(storePathOrDescB)) { MaintainCount mc(nrRunning); showProgress(); try { - copyStorePath(srcStore, dstStore, storePath, repair, checkSigs); + copyStorePath(srcStore, dstStore, storePathOrDescB, repair, checkSigs); } catch (Error &e) { nrFailed++; if (!settings.keepGoing) throw e; + auto storePath = dstStore->bakeCaIfNeeded(storePathOrDescB); logger->log(lvlError, fmt("could not copy %s: %s", dstStore->printStorePath(storePath), e.what())); showProgress(); return; @@ -895,7 +930,7 @@ std::map copyPaths(ref srcStore, ref dstStor showProgress(); }); - return pathsMap; + return; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 8aab56154698..6e1b368770eb 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -172,11 +172,14 @@ struct BuildResult /* Useful for many store functions which can take advantage of content addresses or work with regular store paths */ +typedef std::variant OwnedStorePathOrDesc; typedef std::variant< std::reference_wrapper, std::reference_wrapper > StorePathOrDesc; +StorePathOrDesc borrowStorePathOrDesc(const OwnedStorePathOrDesc &); + struct StoreConfig : public Config { using Config::Config; @@ -327,6 +330,7 @@ public: StorePath makeFixedOutputPathFromCA(const StorePathDescriptor & info) const; StorePath bakeCaIfNeeded(StorePathOrDesc path) const; + StorePath bakeCaIfNeeded(const OwnedStorePathOrDesc & path) const; /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store @@ -363,6 +367,8 @@ public: /* Query which of the given paths is valid. Optionally, try to substitute missing paths. */ + virtual std::set queryValidPaths(const std::set & paths, + SubstituteFlag maybeSubstitute = NoSubstitute); virtual StorePathSet queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute = NoSubstitute); @@ -741,9 +747,13 @@ void copyStorePath(ref srcStore, ref dstStore, in parallel. They are copied in a topologically sorted order (i.e. if A is a reference of B, then A is copied before B), but the set of store paths is not automatically closed; use copyClosure() for - that. Returns a map of what each path was copied to the dstStore - as. */ -std::map copyPaths(ref srcStore, ref dstStore, + that. */ +void copyPaths(ref srcStore, ref dstStore, + const std::set & storePaths, + RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs, + SubstituteFlag substitute = NoSubstitute); +void copyPaths(ref srcStore, ref dstStore, const StorePathSet & storePaths, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, From fdbaca87a4583ab9debfc7c20955dfb2a31ffe14 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 5 Oct 2020 15:17:55 +0000 Subject: [PATCH 26/31] Fix improperly-merged revert from upstream --- src/libstore/globals.cc | 13 +++---------- src/libstore/globals.hh | 4 ---- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 62ee9c7839d9..c5734852dbf0 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -2,7 +2,6 @@ #include "util.hh" #include "archive.hh" #include "args.hh" -#include "sync.hh" #include "abstract-setting-to-json.hh" #include @@ -10,10 +9,10 @@ #include #include #include -#include #include + namespace nix { @@ -134,14 +133,8 @@ bool Settings::isExperimentalFeatureEnabled(const std::string & name) void Settings::requireExperimentalFeature(const std::string & name) { - if (!isExperimentalFeatureEnabled(name)) { - if (allowExperimentalFeatures) { - static Sync> warned; - if (warned.lock()->insert(name).second) - warn("feature '%s' is experimental", name); - } else - throw Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", name); - } + if (!isExperimentalFeatureEnabled(name)) + throw Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", name); } bool Settings::isWSL1() diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 2a135c733945..8c63c5b34829 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -908,10 +908,6 @@ public: value. )"}; - Setting allowExperimentalFeatures{this, true, "allow-experimental-features", - "Whether the use of experimental features other than those listed in " - "the option 'experimental-features' gives a warning rather than fatal error."}; - Setting experimentalFeatures{this, {}, "experimental-features", "Experimental Nix features to enable."}; From f1bcb730a94b4a0546864762b04d5fc1bedd4550 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 25 Mar 2022 22:56:59 +0000 Subject: [PATCH 27/31] Fill in missing comparison operators for content addresses (cherry-pick of 2c21cb672043fcf3c3fd19f89618b37693c0dc62) --- src/libstore/content-address.hh | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 0618dcc11b2b..e895d844a3dc 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -3,6 +3,7 @@ #include #include "hash.hh" #include "path.hh" +#include "comparator.hh" namespace nix { @@ -46,6 +47,8 @@ std::string renderContentAddressMethod(ContentAddressMethod caMethod); struct TextHash { Hash hash; + + GENERATE_CMP(TextHash, me->hash); }; /// Pair of a hash, and how the file system was ingested @@ -53,6 +56,8 @@ struct FixedOutputHash { FileIngestionMethod method; Hash hash; std::string printMethodAlgo() const; + + GENERATE_CMP(FixedOutputHash, me->method, me->hash); }; /* @@ -91,17 +96,13 @@ struct PathReferences std::set references; bool hasSelfReference = false; - bool operator == (const PathReferences & other) const - { - return references == other.references - && hasSelfReference == other.hasSelfReference; - } - /* Functions to view references + hasSelfReference as one set, mainly for compatibility's sake. */ StorePathSet referencesPossiblyToSelf(const Ref & self) const; void insertReferencePossiblyToSelf(const Ref & self, Ref && ref); void setReferencesPossiblyToSelf(const Ref & self, std::set && refs); + + GENERATE_CMP(PathReferences, me->references, me->hasSelfReference); }; template @@ -142,11 +143,15 @@ void PathReferences::setReferencesPossiblyToSelf(const Ref & self, std::set struct TextInfo : TextHash { // References for the paths, self references disallowed StorePathSet references; + + GENERATE_CMP(TextInfo, *(const TextHash *)me, me->references); }; struct FixedOutputInfo : FixedOutputHash { // References for the paths PathReferences references; + + GENERATE_CMP(FixedOutputInfo, *(const FixedOutputHash *)me, me->references); }; typedef std::variant< @@ -158,17 +163,7 @@ struct StorePathDescriptor { std::string name; ContentAddressWithReferences info; - bool operator == (const StorePathDescriptor & other) const - { - return name == other.name; - // FIXME second field - } - - bool operator < (const StorePathDescriptor & other) const - { - return name < other.name; - // FIXME second field - } + GENERATE_CMP(StorePathDescriptor, me->name, me->info); }; std::string renderStorePathDescriptor(StorePathDescriptor ca); From 37ab1891b9de3b6f72032bd6d2fd0f2d3d602b51 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 26 Mar 2022 03:39:18 +0000 Subject: [PATCH 28/31] Fix bug in `copyPaths` References to unknown nodes are filtered. --- src/libstore/store-api.cc | 43 +++++++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 62ab892acac0..c8265ef6a7f6 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -899,9 +899,35 @@ void copyPaths(ref srcStore, ref dstStore, const std::setqueryValidPaths(storePaths, substitute); - std::set missing; - for (auto & path : storePaths) - if (!valid.count(path)) missing.insert(path); + std::set missing; + + /* If we can "upgrade" the node into a descriptor, do that. We will + need to have a node for this. */ + std::map upgraded; + + for (auto & path : storePaths) { + if (valid.count(path)) continue; + + missing.insert(path); + + auto info = srcStore->queryPathInfo(borrowStorePathOrDesc(path)); + + auto storePathP = std::get_if(&path); + if (!storePathP) continue; + auto & storePath = *storePathP; + + auto descOpt = info->fullStorePathDescriptorOpt(); + if (!descOpt) continue; + auto desc = *std::move(descOpt); + + debug("found CA description '%s' for path '%s'", + renderStorePathDescriptor(desc), + srcStore->printStorePath(storePath)); + + auto [it, _] = missing.insert(std::move(desc)); + const auto & descRef = std::get(*it); + upgraded.insert_or_assign(storePath, &descRef); + } Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); @@ -924,9 +950,14 @@ void copyPaths(ref srcStore, ref dstStore, const std::set(&storePathOrDesc)) { - if (auto descOpt = info->fullStorePathDescriptorOpt()) { - auto & desc = *descOpt; - debug("found CA description for path '%s'", srcStore->printStorePath(*storePathP)); + auto & storePath = *storePathP; + auto it = upgraded.find(storePath); + if (it != upgraded.end()) { + auto & [_, descP] = *it; + auto & desc = *descP; + debug("Using found CA description '%s' for path '%s'", + renderStorePathDescriptor(desc), + srcStore->printStorePath(storePath)); return { desc }; } } From 4a55c7b74b0de4f039fd135a86cfa44afc2a264f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 26 Mar 2022 05:07:48 +0000 Subject: [PATCH 29/31] Remove pointless copy --- src/libstore/store-api.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c8265ef6a7f6..bf4099d54502 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -943,7 +943,7 @@ void copyPaths(ref srcStore, ref dstStore, const std::set(pool, - std::set(missing.begin(), missing.end()), + missing, [&](const OwnedStorePathOrDesc & storePathOrDesc) -> std::set { auto info = srcStore->queryPathInfo(borrowStorePathOrDesc(storePathOrDesc)); From 6601ac86421fb05377f113aa52c55da7b9cbe658 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 14 Jan 2023 16:55:13 -0500 Subject: [PATCH 30/31] Revert "Make `ValidPathInfo` have plain `StorePathSet` references like before" This reverts commit b3d91239ae9f21a60057b278ceeff663fb786246. --- perl/lib/Nix/Store.xs | 4 +- src/libexpr/primops.cc | 5 +- src/libstore/binary-cache-store.cc | 8 +-- src/libstore/build/local-derivation-goal.cc | 11 ++-- src/libstore/build/substitution-goal.cc | 10 ++- src/libstore/content-address.cc | 10 --- src/libstore/content-address.hh | 11 +--- src/libstore/daemon.cc | 8 +-- src/libstore/export-import.cc | 4 +- src/libstore/legacy-ssh-store.cc | 6 +- src/libstore/local-store.cc | 10 +-- src/libstore/make-content-addressed.cc | 5 +- src/libstore/misc.cc | 11 ++-- src/libstore/nar-info-disk-cache.cc | 2 +- src/libstore/nar-info.cc | 2 +- src/libstore/path-info.cc | 48 +++++++++------ src/libstore/path-info.hh | 11 +++- src/libstore/remote-store.cc | 13 ++-- src/libstore/store-api.cc | 14 ++--- src/libutil/reference-set.hh | 68 +++++++++++++++++++++ src/nix-store/dotgraph.cc | 2 +- src/nix-store/graphml.cc | 2 +- src/nix-store/nix-store.cc | 8 +-- src/nix/why-depends.cc | 2 +- 24 files changed, 166 insertions(+), 109 deletions(-) create mode 100644 src/libutil/reference-set.hh diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index f19fb20bf1c8..bdb4fa6556ec 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -69,7 +69,7 @@ int isValidPath(char * path) SV * queryReferences(char * path) PPCODE: try { - for (auto & i : store()->queryPathInfo(store()->parseStorePath(path))->references) + for (auto & i : store()->queryPathInfo(store()->parseStorePath(path))->referencesPossiblyToSelf()) XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -110,7 +110,7 @@ SV * queryPathInfo(char * path, int base32) mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); AV * refs = newAV(); - for (auto & i : info->references) + for (auto & i : info->referencesPossiblyToSelf()) av_push(refs, newSVpv(store()->printStorePath(i).c_str(), 0)); XPUSHs(sv_2mortal(newRV((SV *) refs))); AV * sigs = newAV(); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 3b32625b1f7c..ae573cf4d759 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1544,7 +1544,8 @@ static void prim_readFile(EvalState & state, const PosIdx pos, Value * * args, V StorePathSet refs; if (state.store->isInStore(path)) { try { - refs = state.store->queryPathInfo(state.store->toStorePath(path).first)->references; + // FIXME: Are self references becoming non-self references OK? + refs = state.store->queryPathInfo(state.store->toStorePath(path).first)->referencesPossiblyToSelf(); } catch (Error &) { // FIXME: should be InvalidPathError } // Re-scan references to filter down to just the ones that actually occur in the file. @@ -1979,7 +1980,7 @@ static void addPath( try { auto [storePath, subPath] = state.store->toStorePath(path); // FIXME: we should scanForReferences on the path before adding it - refs = state.store->queryPathInfo(storePath)->references; + refs = state.store->queryPathInfo(storePath)->referencesPossiblyToSelf(); path = state.store->toRealPath(storePath) + subPath; } catch (Error &) { // FIXME: should be InvalidPathError } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index ac41add2c73f..087b376551af 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -180,11 +180,11 @@ ref BinaryCacheStore::addToStoreCommon( duration); /* Verify that all references are valid. This may do some .narinfo - reads, but typically they'll already be cached. */ - for (auto & ref : info.references) + reads, but typically they'll already be cached. Note that + self-references are always valid. */ + for (auto & ref : info.references.others) try { - if (ref != info.path) - queryPathInfo(ref); + queryPathInfo(ref); } catch (InvalidPath &) { throw Error("cannot add '%s' to the binary cache because the reference '%s' is not valid", printStorePath(info.path), printStorePath(ref)); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index bb4f929898e2..d96858fc076f 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2523,10 +2523,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() auto narHashAndSize = hashPath(htSHA256, actualPath); ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; - auto refs = rewriteRefs(); - newInfo0.references = std::move(refs.others); - if (refs.self) - newInfo0.references.insert(newInfo0.path); + newInfo0.references = rewriteRefs(); return newInfo0; }, @@ -2777,12 +2774,12 @@ void LocalDerivationGoal::checkOutputs(const std::mapsecond.narSize; - for (auto & ref : i->second.references) + for (auto & ref : i->second.referencesPossiblyToSelf()) pathsLeft.push(ref); } else { auto info = worker.store.queryPathInfo(path); closureSize += info->narSize; - for (auto & ref : info->references) + for (auto & ref : info->referencesPossiblyToSelf()) pathsLeft.push(ref); } } @@ -2822,7 +2819,7 @@ void LocalDerivationGoal::checkOutputs(const std::mapreferences) - if (i != storePath) /* ignore self-references */ - addWaitee(worker.makePathSubstitutionGoal(i)); + for (auto & i : info->references.others) + addWaitee(worker.makePathSubstitutionGoal(i)); if (waitees.empty()) /* to prevent hang (no wake-up event) */ referencesValid(); @@ -188,9 +187,8 @@ void PathSubstitutionGoal::referencesValid() return; } - for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - assert(worker.store.isValidPath(i)); + for (auto & i : info->references.others) + assert(worker.store.isValidPath(i)); state = &PathSubstitutionGoal::tryToRun; worker.wakeUp(shared_from_this()); diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index a98e34cb8a5b..3b8a773b74cd 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -151,16 +151,6 @@ Hash getContentAddressHash(const ContentAddress & ca) }, ca); } -bool StoreReferences::empty() const -{ - return !self && others.empty(); -} - -size_t StoreReferences::size() const -{ - return (self ? 1 : 0) + others.size(); -} - ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { return std::visit(overloaded { [&](const TextHash & h) -> ContentAddressWithReferences { diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 4a50bbee0f3d..f8a4d53701d3 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -4,6 +4,7 @@ #include "hash.hh" #include "path.hh" #include "comparator.hh" +#include "reference-set.hh" namespace nix { @@ -94,15 +95,7 @@ Hash getContentAddressHash(const ContentAddress & ca); * References set */ -struct StoreReferences { - StorePathSet others; - bool self = false; - - bool empty() const; - size_t size() const; - - GENERATE_CMP(StoreReferences, me->self, me->others); -}; +typedef References StoreReferences; /* * Full content address diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 12596ba49837..605f871fc20c 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -336,7 +336,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); StorePathSet paths; if (op == wopQueryReferences) - for (auto & i : store->queryPathInfo(path)->references) + for (auto & i : store->queryPathInfo(path)->referencesPossiblyToSelf()) paths.insert(i); else if (op == wopQueryReferrers) store->queryReferrers(path, paths); @@ -758,7 +758,7 @@ static void performOp(TunnelLogger * logger, ref store, else { to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); - worker_proto::write(*store, to, i->second.references); + worker_proto::write(*store, to, i->second.references.possiblyToSelf(path)); to << i->second.downloadSize << i->second.narSize; } @@ -781,7 +781,7 @@ static void performOp(TunnelLogger * logger, ref store, for (auto & i : infos) { to << store->printStorePath(i.first) << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); - worker_proto::write(*store, to, i.second.references); + worker_proto::write(*store, to, i.second.references.possiblyToSelf(i.first)); to << i.second.downloadSize << i.second.narSize; } break; @@ -863,7 +863,7 @@ static void performOp(TunnelLogger * logger, ref store, ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = worker_proto::read(*store, from, Phantom {}); + info.setReferencesPossiblyToSelf(worker_proto::read(*store, from, Phantom {})); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); info.ca = parseContentAddressOpt(readString(from)); diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 9875da909066..4adf51573386 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -45,7 +45,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) teeSink << exportMagic << printStorePath(path); - worker_proto::write(*this, teeSink, info->references); + worker_proto::write(*this, teeSink, info->referencesPossiblyToSelf()); teeSink << (info->deriver ? printStorePath(*info->deriver) : "") << 0; @@ -80,7 +80,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = references; + info.setReferencesPossiblyToSelf(std::move(references)); info.narSize = saved.s.size(); // Ignore optional legacy signature. diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 2c9dd26808ad..6a694f034270 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -137,7 +137,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = worker_proto::read(*this, conn->from, Phantom {}); + info->setReferencesPossiblyToSelf(worker_proto::read(*this, conn->from, Phantom {})); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -171,7 +171,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - worker_proto::write(*this, conn->to, info.references); + worker_proto::write(*this, conn->to, info.referencesPossiblyToSelf()); conn->to << info.registrationTime << info.narSize @@ -200,7 +200,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor conn->to << exportMagic << printStorePath(info.path); - worker_proto::write(*this, conn->to, info.references); + worker_proto::write(*this, conn->to, info.referencesPossiblyToSelf()); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2d03d2d8b6a6..b32953f3ff96 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -938,7 +938,8 @@ std::shared_ptr LocalStore::queryPathInfoInternal(State & s auto useQueryReferences(state.stmts->QueryReferences.use()(info->id)); while (useQueryReferences.next()) - info->references.insert(parseStorePath(useQueryReferences.getStr(0))); + info->insertReferencePossiblyToSelf( + parseStorePath(useQueryReferences.getStr(0))); return info; } @@ -1205,7 +1206,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) for (auto & [_, i] : infos) { auto referrer = queryValidPathId(*state, i.path); - for (auto & j : i.references) + for (auto & j : i.referencesPossiblyToSelf()) state->stmts->AddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); } @@ -1226,7 +1227,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) topoSort(paths, {[&](const StorePath & path) { auto i = infos.find(path); - return i == infos.end() ? StorePathSet() : i->second.references; + return i == infos.end() ? StorePathSet() : i->second.references.others; }}, {[&](const StorePath & path, const StorePath & parent) { return BuildError( @@ -1524,7 +1525,8 @@ StorePath LocalStore::addTextToStore( ValidPathInfo info { dstPath, narHash }; info.narSize = sink.s.size(); - info.references = references; + // No self reference allowed with text-hashing + info.references.others = references; info.ca = TextHash { .hash = hash }; registerValidPath(info); } diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 09f615439cbe..5d7945eb1c75 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -28,9 +28,8 @@ std::map makeContentAddressed( StringMap rewrites; StoreReferences refs; - for (auto & ref : oldInfo->references) { - if (ref == path) - refs.self = true; + refs.self = oldInfo->references.self; + for (auto & ref : oldInfo->references.others) { auto i = remappings.find(ref); auto replacement = i != remappings.end() ? i->second : ref; // FIXME: warn about unremapped paths? diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index da96dcebcd72..70e97569a9a5 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -40,9 +40,8 @@ void Store::computeFSClosure(const StorePathSet & startPaths, std::future> & fut) { StorePathSet res; auto info = fut.get(); - for (auto & ref : info->references) - if (ref != path) - res.insert(ref); + for (auto & ref : info->references.others) + res.insert(ref); if (includeOutputs && path.isDerivation()) for (auto & [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) @@ -224,7 +223,7 @@ void Store::queryMissing(const std::vector & targets, state->narSize += info->second.narSize; } - for (auto & ref : info->second.references) + for (auto & ref : info->second.references.others) pool.enqueue(std::bind(doPath, DerivedPath::Opaque { ref })); }, }, req.raw()); @@ -242,7 +241,7 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) return topoSort(paths, {[&](const StorePath & path) { try { - return queryPathInfo(path)->references; + return queryPathInfo(path)->references.others; } catch (InvalidPath &) { return StorePathSet(); } @@ -298,7 +297,7 @@ std::map drvOutputReferences( auto info = store.queryPathInfo(outputPath); - return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references); + return drvOutputReferences(Realisation::closure(store, inputRealisations), info->referencesPossiblyToSelf()); } OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_) diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 3e0689534b66..8d4a21daf815 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -248,7 +248,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache narInfo->fileSize = queryNAR.getInt(5); narInfo->narSize = queryNAR.getInt(7); for (auto & r : tokenizeString(queryNAR.getStr(8), " ")) - narInfo->references.insert(StorePath(r)); + narInfo->insertReferencePossiblyToSelf(StorePath(r)); if (!queryNAR.isNull(9)) narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString(queryNAR.getStr(10), " ")) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 071d8355e5c0..f54e8f1fc3f6 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -63,7 +63,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & auto refs = tokenizeString(value, " "); if (!references.empty()) throw corrupt(); for (auto & r : refs) - references.insert(StorePath(r)); + insertReferencePossiblyToSelf(StorePath(r)); } else if (name == "Deriver") { if (value != "unknown-deriver") diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 93f91e702fd0..2972c0bbeef0 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -12,7 +12,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const "1;" + store.printStorePath(path) + ";" + narHash.to_string(Base32, true) + ";" + std::to_string(narSize) + ";" - + concatStringsSep(",", store.printStorePathSet(references)); + + concatStringsSep(",", store.printStorePathSet(referencesPossiblyToSelf())); } @@ -30,25 +30,16 @@ std::optional ValidPathInfo::fullStorePathDescriptorOpt() c .name = std::string { path.name() }, .info = std::visit(overloaded { [&](const TextHash & th) -> ContentAddressWithReferences { - assert(references.count(path) == 0); + assert(!references.self); return TextInfo { th, - .references = references, + .references = references.others, }; }, [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { - auto refs = references; - bool hasSelfReference = false; - if (refs.count(path)) { - hasSelfReference = true; - refs.erase(path); - } return FixedOutputInfo { foh, - .references = { - .others = std::move(refs), - .self = hasSelfReference, - }, + .references = references, }; }, }, *ca), @@ -94,7 +85,7 @@ bool ValidPathInfo::checkSignature(const Store & store, const PublicKeys & publi Strings ValidPathInfo::shortRefs() const { Strings refs; - for (auto & r : references) + for (auto & r : referencesPossiblyToSelf()) refs.push_back(std::string(r.to_string())); return refs; } @@ -109,19 +100,36 @@ ValidPathInfo::ValidPathInfo( { std::visit(overloaded { [this](TextInfo && ti) { - this->references = std::move(ti.references); + this->references = { + .others = std::move(ti.references), + .self = false, + }; this->ca = std::move((TextHash &&) ti); }, [this](FixedOutputInfo && foi) { - this->references = std::move(foi.references.others); - if (foi.references.self) - this->references.insert(path); + this->references = std::move(foi.references); this->ca = std::move((FixedOutputHash &&) foi); }, }, std::move(info.info)); } +StorePathSet ValidPathInfo::referencesPossiblyToSelf() const +{ + return references.possiblyToSelf(path); +} + +void ValidPathInfo::insertReferencePossiblyToSelf(StorePath && ref) +{ + return references.insertPossiblyToSelf(path, std::move(ref)); +} + +void ValidPathInfo::setReferencesPossiblyToSelf(StorePathSet && refs) +{ + return references.setPossiblyToSelf(path, std::move(refs)); +} + + ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format) { return read(source, store, format, store.parseStorePath(readString(source))); @@ -133,7 +141,7 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned auto narHash = Hash::parseAny(readString(source), htSHA256); ValidPathInfo info(path, narHash); if (deriver != "") info.deriver = store.parseStorePath(deriver); - info.references = worker_proto::read(store, source, Phantom {}); + info.setReferencesPossiblyToSelf(worker_proto::read(store, source, Phantom {})); source >> info.registrationTime >> info.narSize; if (format >= 16) { source >> info.ultimate; @@ -154,7 +162,7 @@ void ValidPathInfo::write( sink << store.printStorePath(path); sink << (deriver ? store.printStorePath(*deriver) : "") << narHash.to_string(Base16, false); - worker_proto::write(store, sink, references); + worker_proto::write(store, sink, referencesPossiblyToSelf()); sink << registrationTime << narSize; if (format >= 16) { sink << ultimate diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 476df79c2afb..9254835b77dd 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -17,20 +17,19 @@ class Store; struct SubstitutablePathInfo { std::optional deriver; - StorePathSet references; + StoreReferences references; uint64_t downloadSize; /* 0 = unknown or inapplicable */ uint64_t narSize; /* 0 = unknown */ }; typedef std::map SubstitutablePathInfos; - struct ValidPathInfo { StorePath path; std::optional deriver; Hash narHash; - StorePathSet references; + StoreReferences references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown uint64_t id; // internal use only @@ -82,6 +81,12 @@ struct ValidPathInfo /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; + /* Functions to view references + hasSelfReference as one set, mainly for + compatibility's sake. */ + StorePathSet referencesPossiblyToSelf() const; + void insertReferencePossiblyToSelf(StorePath && ref); + void setReferencesPossiblyToSelf(StorePathSet && refs); + static const size_t maxSigs = std::numeric_limits::max(); /* Return the number of signatures on this .narinfo that were diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index ff57a77ca283..8ea126c65a5f 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -402,7 +402,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = worker_proto::read(*this, conn->from, Phantom {}); + info.references.setPossiblyToSelf(i.first, worker_proto::read(*this, conn->from, Phantom {})); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); infos.insert_or_assign(i.first, std::move(info)); @@ -421,11 +421,12 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S conn.processStderr(); size_t count = readNum(conn->from); for (size_t n = 0; n < count; n++) { - SubstitutablePathInfo & info(infos[parseStorePath(readString(conn->from))]); + auto path = parseStorePath(readString(conn->from)); + SubstitutablePathInfo & info { infos[path] }; auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = worker_proto::read(*this, conn->from, Phantom {}); + info.references.setPossiblyToSelf(path, worker_proto::read(*this, conn->from, Phantom {})); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } @@ -633,7 +634,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, sink << exportMagic << printStorePath(info.path); - worker_proto::write(*this, sink, info.references); + worker_proto::write(*this, sink, info.referencesPossiblyToSelf()); sink << (info.deriver ? printStorePath(*info.deriver) : "") << 0 // == no legacy signature @@ -644,7 +645,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn.processStderr(0, source2.get()); auto importedPaths = worker_proto::read(*this, conn->from, Phantom {}); - assert(importedPaths.size() <= 1); + assert(importedPaths.empty() == 0); // doesn't include possible self reference } else { @@ -652,7 +653,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - worker_proto::write(*this, conn->to, info.references); + worker_proto::write(*this, conn->to, info.referencesPossiblyToSelf()); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c39e50d14516..9446ad132f5f 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -200,10 +200,7 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons { assert(info.hash.type == htSHA256); return makeStorePath( - makeType(*this, "text", StoreReferences { - .others = info.references, - .self = false, - }), + makeType(*this, "text", StoreReferences { info.references }), info.hash, name); } @@ -313,7 +310,7 @@ void Store::addMultipleToStore( bytesExpected += info.narSize; act.setExpected(actCopyPath, bytesExpected); - return info.references; + return info.references.others; }, [&](const StorePath & path) { @@ -818,7 +815,7 @@ std::string Store::makeValidityRegistration(const StorePathSet & paths, s += (format("%1%\n") % info->references.size()).str(); - for (auto & j : info->references) + for (auto & j : info->referencesPossiblyToSelf()) s += printStorePath(j) + "\n"; } @@ -880,7 +877,7 @@ json Store::pathInfoToJSON(const StorePathSet & storePaths, { auto& jsonRefs = (jsonPath["references"] = json::array()); - for (auto & ref : info->references) + for (auto & ref : info->referencesPossiblyToSelf()) jsonRefs.emplace_back(printStorePath(ref)); } @@ -1208,7 +1205,7 @@ std::optional decodeValidPathInfo(const Store & store, std::istre if (!n) throw Error("number expected"); while ((*n)--) { getline(str, s); - info.references.insert(store.parseStorePath(s)); + info.insertReferencePossiblyToSelf(store.parseStorePath(s)); } if (!str || str.eof()) throw Error("missing input"); return std::optional(std::move(info)); @@ -1231,7 +1228,6 @@ std::string showPaths(const PathSet & paths) return concatStringsSep(", ", quoteStrings(paths)); } - Derivation Store::derivationFromPath(const StorePath & drvPath) { ensurePath(drvPath); diff --git a/src/libutil/reference-set.hh b/src/libutil/reference-set.hh new file mode 100644 index 000000000000..ac4a9994e5ac --- /dev/null +++ b/src/libutil/reference-set.hh @@ -0,0 +1,68 @@ +#pragma once + +#include "comparator.hh" + +#include + +namespace nix { + +template +struct References +{ + std::set others; + bool self = false; + + bool empty() const; + size_t size() const; + + /* Functions to view references + self as one set, mainly for + compatibility's sake. */ + std::set possiblyToSelf(const Ref & self) const; + void insertPossiblyToSelf(const Ref & self, Ref && ref); + void setPossiblyToSelf(const Ref & self, std::set && refs); + + GENERATE_CMP(References, me->others, me->self); +}; + +template +bool References::empty() const +{ + return !self && others.empty(); +} + +template +size_t References::size() const +{ + return (self ? 1 : 0) + others.size(); +} + +template +std::set References::possiblyToSelf(const Ref & selfRef) const +{ + std::set refs { others }; + if (self) + refs.insert(selfRef); + return refs; +} + +template +void References::insertPossiblyToSelf(const Ref & selfRef, Ref && ref) +{ + if (ref == selfRef) + self = true; + else + others.insert(std::move(ref)); +} + +template +void References::setPossiblyToSelf(const Ref & selfRef, std::set && refs) +{ + if (refs.count(selfRef)) { + self = true; + refs.erase(selfRef); + } + + others = refs; +} + +} diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc index 577cadceb313..36d774dca766 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/nix-store/dotgraph.cc @@ -56,7 +56,7 @@ void printDotGraph(ref store, StorePathSet && roots) cout << makeNode(std::string(path.to_string()), path.name(), "#ff0000"); - for (auto & p : store->queryPathInfo(path)->references) { + for (auto & p : store->queryPathInfo(path)->referencesPossiblyToSelf()) { if (p != path) { workList.insert(p); cout << makeEdge(std::string(p.to_string()), std::string(path.to_string())); diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc index 425d61e53eda..d2eebca7affa 100644 --- a/src/nix-store/graphml.cc +++ b/src/nix-store/graphml.cc @@ -71,7 +71,7 @@ void printGraphML(ref store, StorePathSet && roots) auto info = store->queryPathInfo(path); cout << makeNode(*info); - for (auto & p : info->references) { + for (auto & p : info->referencesPossiblyToSelf()) { if (p != path) { workList.insert(p); cout << makeEdge(path.to_string(), p.to_string()); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 5b261ecc6d3f..5cb5aa53afaa 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -263,7 +263,7 @@ static void printTree(const StorePath & path, closure(B). That is, if derivation A is an (possibly indirect) input of B, then A is printed first. This has the effect of flattening the tree, preventing deeply nested structures. */ - auto sorted = store->topoSortPaths(info->references); + auto sorted = store->topoSortPaths(info->referencesPossiblyToSelf()); reverse(sorted.begin(), sorted.end()); for (const auto &[n, i] : enumerate(sorted)) { @@ -344,7 +344,7 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & j : ps) { if (query == qRequisites) store->computeFSClosure(j, paths, false, includeOutputs); else if (query == qReferences) { - for (auto & p : store->queryPathInfo(j)->references) + for (auto & p : store->queryPathInfo(j)->referencesPossiblyToSelf()) paths.insert(p); } else if (query == qReferrers) { @@ -867,7 +867,7 @@ static void opServe(Strings opFlags, Strings opArgs) auto info = store->queryPathInfo(i); out << store->printStorePath(info->path) << (info->deriver ? store->printStorePath(*info->deriver) : ""); - worker_proto::write(*store, out, info->references); + worker_proto::write(*store, out, info->referencesPossiblyToSelf()); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; @@ -964,7 +964,7 @@ static void opServe(Strings opFlags, Strings opArgs) }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = worker_proto::read(*store, in, Phantom {}); + info.setReferencesPossiblyToSelf(worker_proto::read(*store, in, Phantom {})); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); info.ca = parseContentAddressOpt(readString(in)); diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 76125e5e43d3..33cd13600335 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -136,7 +136,7 @@ struct CmdWhyDepends : SourceExprCommand for (auto & path : closure) graph.emplace(path, Node { .path = path, - .refs = store->queryPathInfo(path)->references, + .refs = store->queryPathInfo(path)->references.others, .dist = path == dependencyPath ? 0 : inf }); From b21dd384235bc9863c89ef249bbc1ca58f4ec3fd Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 23 Jan 2023 16:38:54 -0500 Subject: [PATCH 31/31] Revert "Don't add `StorePathDescriptor` for now" This reverts commit 4540e7b940ca56db821fe7c7d7d79fafa488f55e. --- src/libfetchers/tarball.cc | 14 +++-- src/libstore/binary-cache-store.cc | 50 +++++++++------- src/libstore/build/local-derivation-goal.cc | 14 +++-- src/libstore/build/substitution-goal.cc | 7 ++- src/libstore/content-address.hh | 7 +++ src/libstore/local-store.cc | 35 ++++++----- src/libstore/make-content-addressed.cc | 14 +++-- src/libstore/nar-info.hh | 4 +- src/libstore/path-info.cc | 64 +++++++++++---------- src/libstore/path-info.hh | 4 +- src/libstore/store-api.cc | 28 ++++----- src/libstore/store-api.hh | 2 +- src/nix/add-to-store.cc | 14 +++-- src/nix/profile.cc | 20 ++++--- 14 files changed, 151 insertions(+), 126 deletions(-) diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 3020466105f8..155b86cc4a28 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -72,13 +72,15 @@ DownloadFileResult downloadFile( auto hash = hashString(htSHA256, res.data); ValidPathInfo info { *store, - name, - FixedOutputInfo { - { - .method = FileIngestionMethod::Flat, - .hash = hash, + { + .name = name, + .info = FixedOutputInfo { + { + .method = FileIngestionMethod::Flat, + .hash = hash, + }, + .references = {}, }, - .references = {}, }, hashString(htSHA256, sink.s), }; diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 9058bb8b119b..ac41add2c73f 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -307,15 +307,17 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { *this, - name, - FixedOutputInfo { - { - .method = method, - .hash = nar.first, - }, - .references = { - .others = references, - .self = false, + { + .name = std::string { name }, + .info = FixedOutputInfo { + { + .method = method, + .hash = nar.first, + }, + .references = { + .others = references, + .self = false, + }, }, }, nar.first, @@ -425,15 +427,17 @@ StorePath BinaryCacheStore::addToStore( return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { *this, - name, - FixedOutputInfo { - { - .method = method, - .hash = h, - }, - .references = { - .others = references, - .self = false, + { + .name = std::string { name }, + .info = FixedOutputInfo { + { + .method = method, + .hash = h, + }, + .references = { + .others = references, + .self = false, + }, }, }, nar.first, @@ -461,10 +465,12 @@ StorePath BinaryCacheStore::addTextToStore( return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { *this, - std::string { name }, - TextInfo { - { .hash = textHash }, - references, + { + .name = std::string { name }, + .info = TextInfo { + { .hash = textHash }, + references, + }, }, nar.first, }; diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 98f8cb06100b..bb4f929898e2 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2475,13 +2475,15 @@ DrvOutputs LocalDerivationGoal::registerOutputs() auto got = caSink.finish().first; ValidPathInfo newInfo0 { worker.store, - outputPathName(drv->name, outputName), - FixedOutputInfo { - { - .method = outputHash.method, - .hash = got, + { + .name = outputPathName(drv->name, outputName), + .info = FixedOutputInfo { + { + .method = outputHash.method, + .hash = got, + }, + .references = rewriteRefs(), }, - .references = rewriteRefs(), }, Hash::dummy, }; diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 87fed495c56f..994cb4ac2f66 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -95,9 +95,10 @@ void PathSubstitutionGoal::tryNext() subs.pop_front(); if (ca) { - subPath = sub->makeFixedOutputPathFromCA( - std::string { storePath.name() }, - caWithoutRefs(*ca)); + subPath = sub->makeFixedOutputPathFromCA({ + .name = std::string { storePath.name() }, + .info = caWithoutRefs(*ca), + }); if (sub->storeDir == worker.store.storeDir) assert(subPath == storePath); } else if (sub->storeDir != worker.store.storeDir) { diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index c49ab269f0fb..4a50bbee0f3d 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -132,4 +132,11 @@ typedef std::variant< ContentAddressWithReferences caWithoutRefs(const ContentAddress &); +struct StorePathDescriptor { + std::string name; + ContentAddressWithReferences info; + + GENERATE_CMP(StorePathDescriptor, me->name, me->info); +}; + } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e55ccab84a7c..2d03d2d8b6a6 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1136,9 +1136,10 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst // Recompute store path so that we can use a different store root. if (path.second) { - subPath = makeFixedOutputPathFromCA( - path.first.name(), - caWithoutRefs(*path.second)); + subPath = makeFixedOutputPathFromCA({ + .name = std::string { path.first.name() }, + .info = caWithoutRefs(*path.second), + }); if (sub->storeDir == storeDir) assert(subPath == path.first); if (subPath != path.first) @@ -1416,18 +1417,21 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name auto [hash, size] = hashSink->finish(); - ContentAddressWithReferences desc = FixedOutputInfo { - { - .method = method, - .hash = hash, - }, - .references = { - .others = references, - .self = false, + auto desc = StorePathDescriptor { + std::string { name }, + FixedOutputInfo { + { + .method = method, + .hash = hash, + }, + .references = { + .others = references, + .self = false, + }, }, }; - auto dstPath = makeFixedOutputPathFromCA(name, desc); + auto dstPath = makeFixedOutputPathFromCA(desc); addTempRoot(dstPath); @@ -1471,12 +1475,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name optimisePath(realPath, repair); - ValidPathInfo info { - *this, - name, - std::move(desc), - narHash.first - }; + ValidPathInfo info { *this, std::move(desc), narHash.first }; info.narSize = narHash.second; registerValidPath(info); } diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 3ee64c77a903..09f615439cbe 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -49,13 +49,15 @@ std::map makeContentAddressed( ValidPathInfo info { dstStore, - path.name(), - FixedOutputInfo { - { - .method = FileIngestionMethod::Recursive, - .hash = narModuloHash, + StorePathDescriptor { + .name = std::string { path.name() }, + .info = FixedOutputInfo { + { + .method = FileIngestionMethod::Recursive, + .hash = narModuloHash, + }, + .references = std::move(refs), }, - .references = std::move(refs), }, Hash::dummy, }; diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index a4dccb39710a..f1e3aabbda71 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -16,8 +16,8 @@ struct NarInfo : ValidPathInfo uint64_t fileSize = 0; NarInfo() = delete; - NarInfo(const Store & store, std::string && name, ContentAddressWithReferences && ca, Hash narHash) - : ValidPathInfo(store, std::move(name), std::move(ca), narHash) + NarInfo(const Store & store, StorePathDescriptor && ca, Hash narHash) + : ValidPathInfo(store, std::move(ca), narHash) { } NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 5944afd06890..93f91e702fd0 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -21,45 +21,48 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } -std::optional ValidPathInfo::contentAddressWithReferenences() const +std::optional ValidPathInfo::fullStorePathDescriptorOpt() const { if (! ca) return std::nullopt; - return std::visit(overloaded { - [&](const TextHash & th) -> ContentAddressWithReferences { - assert(references.count(path) == 0); - return TextInfo { - th, - .references = references, - }; - }, - [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { - auto refs = references; - bool hasSelfReference = false; - if (refs.count(path)) { - hasSelfReference = true; - refs.erase(path); - } - return FixedOutputInfo { - foh, - .references = { - .others = std::move(refs), - .self = hasSelfReference, - }, - }; - }, - }, *ca); + return StorePathDescriptor { + .name = std::string { path.name() }, + .info = std::visit(overloaded { + [&](const TextHash & th) -> ContentAddressWithReferences { + assert(references.count(path) == 0); + return TextInfo { + th, + .references = references, + }; + }, + [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { + auto refs = references; + bool hasSelfReference = false; + if (refs.count(path)) { + hasSelfReference = true; + refs.erase(path); + } + return FixedOutputInfo { + foh, + .references = { + .others = std::move(refs), + .self = hasSelfReference, + }, + }; + }, + }, *ca), + }; } bool ValidPathInfo::isContentAddressed(const Store & store) const { - auto fullCaOpt = contentAddressWithReferenences(); + auto fullCaOpt = fullStorePathDescriptorOpt(); if (! fullCaOpt) return false; - auto caPath = store.makeFixedOutputPathFromCA(path.name(), *fullCaOpt); + auto caPath = store.makeFixedOutputPathFromCA(*fullCaOpt); bool res = caPath == path; @@ -99,10 +102,9 @@ Strings ValidPathInfo::shortRefs() const ValidPathInfo::ValidPathInfo( const Store & store, - std::string_view name, - ContentAddressWithReferences && ca, + StorePathDescriptor && info, Hash narHash) - : path(store.makeFixedOutputPathFromCA(name, ca)) + : path(store.makeFixedOutputPathFromCA(info)) , narHash(narHash) { std::visit(overloaded { @@ -116,7 +118,7 @@ ValidPathInfo::ValidPathInfo( this->references.insert(path); this->ca = std::move((FixedOutputHash &&) foi); }, - }, std::move(ca)); + }, std::move(info.info)); } diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 663d9454000d..476df79c2afb 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -77,7 +77,7 @@ struct ValidPathInfo void sign(const Store & store, const SecretKey & secretKey); - std::optional contentAddressWithReferenences() const; + std::optional fullStorePathDescriptorOpt() const; /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; @@ -100,7 +100,7 @@ struct ValidPathInfo ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { }; ValidPathInfo(const Store & store, - std::string_view name, ContentAddressWithReferences && ca, Hash narHash); + StorePathDescriptor && ca, Hash narHash); virtual ~ValidPathInfo() { } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 3c0c26706568..c39e50d14516 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -209,17 +209,17 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons } -StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const +StorePath Store::makeFixedOutputPathFromCA(const StorePathDescriptor & desc) const { // New template return std::visit(overloaded { [&](const TextInfo & ti) { - return makeTextPath(name, ti); + return makeTextPath(desc.name, ti); }, [&](const FixedOutputInfo & foi) { - return makeFixedOutputPath(name, foi); + return makeFixedOutputPath(desc.name, foi); } - }, ca); + }, desc.info); } @@ -437,13 +437,15 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, ValidPathInfo info { *this, - name, - FixedOutputInfo { - { - .method = method, - .hash = hash, + StorePathDescriptor { + std::string { name }, + FixedOutputInfo { + { + .method = method, + .hash = hash, + }, + .references = {}, }, - .references = {}, }, narHash, }; @@ -995,8 +997,7 @@ void copyStorePath( if (info->ca && info->references.empty()) { auto info2 = make_ref(*info); info2->path = dstStore.makeFixedOutputPathFromCA( - info->path.name(), - info->contentAddressWithReferenences().value()); + info->fullStorePathDescriptorOpt().value()); if (dstStore.storeDir == srcStore.storeDir) assert(info->path == info2->path); info = info2; @@ -1109,8 +1110,7 @@ std::map copyPaths( auto storePathForDst = storePathForSrc; if (currentPathInfo.ca && currentPathInfo.references.empty()) { storePathForDst = dstStore.makeFixedOutputPathFromCA( - currentPathInfo.path.name(), - currentPathInfo.contentAddressWithReferenences().value()); + currentPathInfo.fullStorePathDescriptorOpt().value()); if (dstStore.storeDir == srcStore.storeDir) assert(storePathForDst == storePathForSrc); if (storePathForDst != storePathForSrc) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 2d252db843a3..d77aea338aac 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -216,7 +216,7 @@ public: StorePath makeTextPath(std::string_view name, const TextInfo & info) const; - StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const; + StorePath makeFixedOutputPathFromCA(const StorePathDescriptor & info) const; /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 81dbc09a6f23..0b58818c373e 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -43,13 +43,15 @@ struct CmdAddToStore : MixDryRun, StoreCommand ValidPathInfo info { *store, - std::move(*namePart), - FixedOutputInfo { - { - .method = std::move(ingestionMethod), - .hash = std::move(hash), + StorePathDescriptor { + .name = *namePart, + .info = FixedOutputInfo { + { + .method = std::move(ingestionMethod), + .hash = std::move(hash), + }, + .references = {}, }, - .references = {}, }, narHash, }; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 345505532c4f..aac8e5c81c4d 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -200,15 +200,17 @@ struct ProfileManifest ValidPathInfo info { *store, - "profile", - FixedOutputInfo { - { - .method = FileIngestionMethod::Recursive, - .hash = narHash, - }, - .references = { - .others = std::move(references), - .self = false, + StorePathDescriptor { + "profile", + FixedOutputInfo { + { + .method = FileIngestionMethod::Recursive, + .hash = narHash, + }, + .references = { + .others = std::move(references), + .self = false, + }, }, }, narHash,