Skip to content

Commit

Permalink
Merge pull request #7 from obsidiansystems/git+validPathInfo-ca-prope…
Browse files Browse the repository at this point in the history
…r-datatype

valid path info ca proper datatype
  • Loading branch information
Ericson2314 authored Jun 4, 2020
2 parents 8bbe10e + 010013e commit 8df800c
Show file tree
Hide file tree
Showing 24 changed files with 243 additions and 156 deletions.
2 changes: 1 addition & 1 deletion src/libexpr/get-drvs.cc
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#include "get-drvs.hh"
#include "util.hh"
#include "eval-inline.hh"
#include "derivations.hh"
#include "store-api.hh"

#include <cstring>
#include <regex>
Expand Down
2 changes: 1 addition & 1 deletion src/libexpr/primops/context.cc
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#include "primops.hh"
#include "eval-inline.hh"
#include "derivations.hh"
#include "store-api.hh"

namespace nix {

Expand Down
5 changes: 4 additions & 1 deletion src/libfetchers/tarball.cc
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,10 @@ DownloadFileResult downloadFile(
ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name));
info.narHash = hashString(HashType::SHA256, *sink.s);
info.narSize = sink.s->size();
info.ca = makeFixedOutputCA(FileIngestionMethod::Flat, hash);
info.ca = FileSystemHash {
FileIngestionMethod::Flat,
hash,
};
auto source = StringSource { *sink.s };
store->addToStore(info, source, NoRepair, NoCheckSigs);
storePath = std::move(info.path);
Expand Down
9 changes: 6 additions & 3 deletions src/libstore/build.cc
Original file line number Diff line number Diff line change
Expand Up @@ -3708,7 +3708,7 @@ void DerivationGoal::registerOutputs()
/* Check that fixed-output derivations produced the right
outputs (i.e., the content hash should match the specified
hash). */
std::string ca;
std::optional<ContentAddress> ca;

if (fixedOutput) {

Expand Down Expand Up @@ -3757,7 +3757,7 @@ void DerivationGoal::registerOutputs()
else
assert(worker.store.parseStorePath(path) == dest);

ca = makeFixedOutputCA(i.second.hash->method, h2);
ca = FileSystemHash { i.second.hash->method, h2 };
}

/* Get rid of all weird permissions. This also checks that
Expand Down Expand Up @@ -3830,7 +3830,10 @@ void DerivationGoal::registerOutputs()
info.ca = ca;
worker.store.signPathInfo(info);

if (!info.references.empty()) info.ca.clear();
if (!info.references.empty()) {
// FIXME don't we have an experimental feature for fixed output with references?
info.ca = {};
}

infos.emplace(i.first, std::move(info));
}
Expand Down
83 changes: 83 additions & 0 deletions src/libstore/content-address.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
#include "content-address.hh"

namespace nix {

std::string FileSystemHash::printMethodAlgo() const {
return makeFileIngestionPrefix(method) + printHashType(*hash.type);
}

std::string makeFileIngestionPrefix(const FileIngestionMethod m) {
switch (m) {
case FileIngestionMethod::Flat:
return "";
case FileIngestionMethod::Recursive:
return "r:";
case FileIngestionMethod::Git:
return "git:";
}
abort();
}

std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
{
return "fixed:"
+ makeFileIngestionPrefix(method)
+ hash.to_string();
}

// FIXME Put this somewhere?
template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;

std::string renderContentAddress(ContentAddress ca) {
return std::visit(overloaded {
[](TextHash th) {
return "text:" + th.hash.to_string();
},
[](FileSystemHash fsh) {
return makeFixedOutputCA(fsh.method, fsh.hash);
}
}, ca);
}

ContentAddress parseContentAddress(std::string_view rawCa) {
auto prefixSeparator = rawCa.find(':');
if (prefixSeparator != string::npos) {
auto prefix = string(rawCa, 0, prefixSeparator);
if (prefix == "text") {
auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos);
Hash hash = Hash(string(hashTypeAndHash));
if (*hash.type != HashType::SHA256) {
throw Error("parseContentAddress: the text hash should have type SHA256");
}
return TextHash { hash };
} else if (prefix == "fixed") {
// This has to be an inverse of makeFixedOutputCA
auto methodAndHash = rawCa.substr(prefixSeparator+1, string::npos);
if (methodAndHash.substr(0, 2) == "r:") {
std::string_view hashRaw = methodAndHash.substr(2, string::npos);
return FileSystemHash { FileIngestionMethod::Recursive, Hash(string(hashRaw)) };
} else if (methodAndHash.substr(0, 4) == "git:") {
std::string_view hashRaw = methodAndHash.substr(4, string::npos);
return FileSystemHash { FileIngestionMethod::Git, Hash(string(hashRaw)) };
} else {
std::string_view hashRaw = methodAndHash;
return FileSystemHash { FileIngestionMethod::Flat, Hash(string(hashRaw)) };
}
} else {
throw Error("parseContentAddress: format not recognized; has to be text or fixed");
}
} else {
throw Error("Not a content address because it lacks an appropriate prefix");
}
};

std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt) {
return rawCaOpt == "" ? std::optional<ContentAddress> {} : parseContentAddress(rawCaOpt);
};

std::string renderContentAddress(std::optional<ContentAddress> ca) {
return ca ? renderContentAddress(*ca) : "";
}

}
68 changes: 68 additions & 0 deletions src/libstore/content-address.hh
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
#pragma once

#include <variant>
#include "hash.hh"

namespace nix {

enum struct FileIngestionMethod : uint8_t {
Flat,
Recursive,
Git,
};


struct TextHash {
Hash hash;
TextHash(const TextHash &) = default;
TextHash(TextHash &&) = default;
TextHash & operator = (const TextHash &) = default;
};

/// Pair of a hash, and how the file system was ingested
struct FileSystemHash {
FileIngestionMethod method;
Hash hash;
FileSystemHash(FileIngestionMethod method, Hash hash)
: method(std::move(method))
, hash(std::move(hash))
{ }
FileSystemHash(const FileSystemHash &) = default;
FileSystemHash(FileSystemHash &&) = default;
FileSystemHash & operator = (const FileSystemHash &) = default;
std::string printMethodAlgo() const;
};

/*
We've accumulated several types of content-addressed paths over the years;
fixed-output derivations support multiple hash algorithms and serialisation
methods (flat file vs NAR). Thus, ‘ca’ has one of the following forms:
* ‘text:sha256:<sha256 hash of file contents>’: For paths
computed by makeTextPath() / addTextToStore().
* ‘fixed:<r?>:<ht>:<h>’: For paths computed by
makeFixedOutputPath() / addToStore().
*/
typedef std::variant<
TextHash, // for paths computed by makeTextPath() / addTextToStore
FileSystemHash // for path computed by makeFixedOutputPath
> ContentAddress;

/* Compute the prefix to the hash algorithm which indicates how the files were
ingested. */
std::string makeFileIngestionPrefix(const FileIngestionMethod m);

/* Compute the content-addressability assertion (ValidPathInfo::ca)
for paths created by makeFixedOutputPath() / addToStore(). */
std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash);

std::string renderContentAddress(ContentAddress ca);

std::string renderContentAddress(std::optional<ContentAddress> ca);

ContentAddress parseContentAddress(std::string_view rawCa);

std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt);

}
5 changes: 3 additions & 2 deletions src/libstore/daemon.cc
Original file line number Diff line number Diff line change
Expand Up @@ -663,7 +663,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
to << info->ultimate
<< info->sigs
<< info->ca;
<< renderContentAddress(info->ca);
}
} else {
assert(GET_PROTOCOL_MINOR(clientVersion) >= 17);
Expand Down Expand Up @@ -721,7 +721,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
info.references = readStorePaths<StorePathSet>(*store, from);
from >> info.registrationTime >> info.narSize >> info.ultimate;
info.sigs = readStrings<StringSet>(from);
from >> info.ca >> repair >> dontCheckSigs;
info.ca = parseContentAddressOpt(readString(from));
from >> repair >> dontCheckSigs;
if (!trusted && dontCheckSigs)
dontCheckSigs = false;
if (!trusted)
Expand Down
5 changes: 0 additions & 5 deletions src/libstore/derivations.cc
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,6 @@

namespace nix {

std::string FileSystemHash::printMethodAlgo() const {
return makeFileIngestionPrefix(method) + printHashType(*hash.type);
}


BasicDerivation::BasicDerivation(const BasicDerivation & other)
: platform(other.platform)
, builder(other.builder)
Expand Down
18 changes: 3 additions & 15 deletions src/libstore/derivations.hh
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
#pragma once

#include "path.hh"
#include "types.hh"
#include "hash.hh"
#include "store-api.hh"
#include "content-address.hh"

#include <map>

Expand All @@ -12,20 +13,6 @@ namespace nix {

/* Abstract syntax of derivations. */

/// Pair of a hash, and how the file system was ingested
struct FileSystemHash {
FileIngestionMethod method;
Hash hash;
FileSystemHash(FileIngestionMethod method, Hash hash)
: method(std::move(method))
, hash(std::move(hash))
{ }
FileSystemHash(const FileSystemHash &) = default;
FileSystemHash(FileSystemHash &&) = default;
FileSystemHash & operator = (const FileSystemHash &) = default;
std::string printMethodAlgo() const;
};

struct DerivationOutput
{
StorePath path;
Expand Down Expand Up @@ -90,6 +77,7 @@ struct Derivation : BasicDerivation

class Store;

enum RepairFlag : bool { NoRepair = false, Repair = true };

/* Write a derivation to the Nix store, and return its path. */
StorePath writeDerivation(ref<Store> store,
Expand Down
4 changes: 2 additions & 2 deletions src/libstore/legacy-ssh-store.cc
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ struct LegacySSHStore : public Store
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) {
auto s = readString(conn->from);
info->narHash = s.empty() ? Hash() : Hash(s);
conn->from >> info->ca;
info->ca = parseContentAddressOpt(readString(conn->from));
info->sigs = readStrings<StringSet>(conn->from);
}

Expand Down Expand Up @@ -146,7 +146,7 @@ struct LegacySSHStore : public Store
<< info.narSize
<< info.ultimate
<< info.sigs
<< info.ca;
<< renderContentAddress(info.ca);
try {
copyNAR(source, conn->to);
} catch (...) {
Expand Down
25 changes: 13 additions & 12 deletions src/libstore/local-store.cc
Original file line number Diff line number Diff line change
Expand Up @@ -577,7 +577,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
uint64_t LocalStore::addValidPath(State & state,
const ValidPathInfo & info, bool checkOutputs)
{
if (info.ca != "" && !info.isContentAddressed(*this))
if (info.ca.has_value() && !info.isContentAddressed(*this))
throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't",
printStorePath(info.path));

Expand All @@ -589,7 +589,7 @@ uint64_t LocalStore::addValidPath(State & state,
(info.narSize, info.narSize != 0)
(info.ultimate ? 1 : 0, info.ultimate)
(concatStringsSep(" ", info.sigs), !info.sigs.empty())
(info.ca, !info.ca.empty())
(renderContentAddress(info.ca), (bool) info.ca)
.exec();
uint64_t id = sqlite3_last_insert_rowid(state.db);

Expand Down Expand Up @@ -663,7 +663,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
if (s) info->sigs = tokenizeString<StringSet>(s, " ");

s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 7);
if (s) info->ca = s;
if (s) info->ca = parseContentAddressOpt(s);

/* Get the references. */
auto useQueryReferences(state->stmtQueryReferences.use()(info->id));
Expand All @@ -686,7 +686,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
(info.narHash.to_string(Base::Base16))
(info.ultimate ? 1 : 0, info.ultimate)
(concatStringsSep(" ", info.sigs), !info.sigs.empty())
(info.ca, !info.ca.empty())
(renderContentAddress(info.ca), (bool) info.ca)
(printStorePath(info.path))
.exec();
}
Expand Down Expand Up @@ -1000,15 +1000,15 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,

deletePath(realPath);

if (info.ca != "" &&
!((hasPrefix(info.ca, "text:") && !info.references.count(info.path))
|| info.references.empty()))
// text hashing has long been allowed to have non-self-references because it is used for drv files.
bool refersToSelf = info.references.count(info.path) > 0;
if (info.ca.has_value() && !info.references.empty() && !(std::holds_alternative<TextHash>(*info.ca) && !refersToSelf))
settings.requireExperimentalFeature("ca-references");

/* While restoring the path from the NAR, compute the hash
of the NAR. */
std::unique_ptr<AbstractHashSink> hashSink;
if (info.ca == "" || !info.references.count(info.path))
if (!info.ca.has_value() || !info.references.count(info.path))
hashSink = std::make_unique<HashSink>(HashType::SHA256);
else
hashSink = std::make_unique<HashModuloSink>(HashType::SHA256, storePathToHash(printStorePath(info.path)));
Expand All @@ -1019,7 +1019,8 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
return n;
});

if (hasPrefix(info.ca, "fixed:git:"))
auto p = info.ca ? std::get_if<FileSystemHash>(&*info.ca) : NULL;
if (p && p->method == FileIngestionMethod::Git)
restoreGit(realPath, wrapperSource, realStoreDir, storeDir);
else
restorePath(realPath, wrapperSource);
Expand Down Expand Up @@ -1110,7 +1111,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
ValidPathInfo info(dstPath.clone());
info.narHash = hash.first;
info.narSize = hash.second;
info.ca = makeFixedOutputCA(method, h);
info.ca = FileSystemHash { method, h };
registerValidPath(info);
}

Expand Down Expand Up @@ -1195,7 +1196,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
info.narHash = narHash;
info.narSize = sink.s->size();
info.references = cloneStorePathSet(references);
info.ca = "text:" + hash.to_string();
info.ca = TextHash { .hash = hash };
registerValidPath(info);
}

Expand Down Expand Up @@ -1303,7 +1304,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
printMsg(Verbosity::Talkative, "checking contents of '%s'", printStorePath(i));

std::unique_ptr<AbstractHashSink> hashSink;
if (info->ca == "" || !info->references.count(info->path))
if (!info->ca || !info->references.count(info->path))
hashSink = std::make_unique<HashSink>(*info->narHash.type);
else
hashSink = std::make_unique<HashModuloSink>(*info->narHash.type, storePathToHash(printStorePath(info->path)));
Expand Down
Loading

0 comments on commit 8df800c

Please sign in to comment.