Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 9 additions & 8 deletions src/libexpr/primops.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1063,8 +1063,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
drv.env[i] = hashPlaceholder(i);
drv.outputs.insert_or_assign(i, DerivationOutput {
.output = DerivationOutputCAFloating {
.method = ingestionMethod,
.hashType = std::move(ht),
.hashMethod = CAPathHashMethod {
.fileIngestionMethod = ingestionMethod,
.hashType = std::move(ht),
},
},
});
}
Expand Down Expand Up @@ -1129,12 +1131,11 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
/* Optimisation, but required in read-only mode! because in that
case we don't actually write store derivations, so we can't
read them later.

However, we don't bother doing this for floating CA derivations because
their "hash modulo" is indeterminate until built. */
if (drv.type() != DerivationType::CAFloating)
drvHashes.insert_or_assign(drvPath,
hashDerivationModulo(*state.store, Derivation(drv), false));
*/
drvHashes.insert_or_assign(
drvPath,
hashDerivationModulo(*state.store, Derivation(drv), false)
);

state.mkAttrs(v, 1 + drv.outputs.size());
mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS});
Expand Down
19 changes: 10 additions & 9 deletions src/libstore/build/derivation-goal.cc
Original file line number Diff line number Diff line change
Expand Up @@ -493,8 +493,7 @@ void DerivationGoal::inputsRealised()
if (useDerivation) {
auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());

if ((!fullDrv.inputDrvs.empty() &&
fullDrv.type() == DerivationType::CAFloating) || fullDrv.type() == DerivationType::DeferredInputAddressed) {
if (fullDrv.needsRewriting()) {
/* We are be able to resolve this derivation based on the
now-known results of dependencies. If so, we become a stub goal
aliasing that resolved derivation goal */
Expand Down Expand Up @@ -3079,7 +3078,7 @@ void DerivationGoal::registerOutputs()

auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo {
auto & st = outputStats.at(outputName);
if (outputHash.method == FileIngestionMethod::Flat) {
if (outputHash.hashMethod.fileIngestionMethod == FileIngestionMethod::Flat) {
/* The output path should be a regular file without execute permission. */
if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
throw BuildError(
Expand All @@ -3090,8 +3089,8 @@ void DerivationGoal::registerOutputs()
rewriteOutput();
/* FIXME optimize and deduplicate with addToStore */
std::string oldHashPart { scratchPath.hashPart() };
HashModuloSink caSink { outputHash.hashType, oldHashPart };
switch (outputHash.method) {
HashModuloSink caSink { outputHash.hashMethod.hashType, oldHashPart };
switch (outputHash.hashMethod.fileIngestionMethod) {
case FileIngestionMethod::Recursive:
dumpPath(actualPath, caSink);
break;
Expand All @@ -3106,7 +3105,7 @@ void DerivationGoal::registerOutputs()
auto narHashAndSize = narSink.finish();
ValidPathInfo newInfo0 {
worker.store.makeFixedOutputPath(
outputHash.method,
outputHash.hashMethod.fileIngestionMethod,
got,
outputPathName(drv->name, outputName),
refs.second,
Expand All @@ -3115,7 +3114,7 @@ void DerivationGoal::registerOutputs()
};
newInfo0.narSize = narHashAndSize.second;
newInfo0.ca = FixedOutputHash {
.method = outputHash.method,
.method = outputHash.hashMethod.fileIngestionMethod,
.hash = got,
};
newInfo0.references = refs.second;
Expand Down Expand Up @@ -3148,8 +3147,10 @@ void DerivationGoal::registerOutputs()
},
[&](DerivationOutputCAFixed dof) {
auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating {
.method = dof.hash.method,
.hashType = dof.hash.hash.type,
.hashMethod = CAPathHashMethod {
.fileIngestionMethod = dof.hash.method,
.hashType = dof.hash.hash.type,
}
});

/* Check wanted hash */
Expand Down
21 changes: 17 additions & 4 deletions src/libstore/content-address.cc
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,22 @@

namespace nix {

std::string CAPathHashMethod::print() const
{
return makeFileIngestionPrefix(fileIngestionMethod) + printHashType(hashType);
}

CAPathHashMethod FixedOutputHash::hashMethod() const
{
return CAPathHashMethod {
.fileIngestionMethod = method,
.hashType = hash.type,
};
}

std::string FixedOutputHash::printMethodAlgo() const
{
return makeFileIngestionPrefix(method) + printHashType(hash.type);
return hashMethod().print();
}

std::string makeFileIngestionPrefix(const FileIngestionMethod m)
Expand Down Expand Up @@ -46,7 +59,7 @@ std::string renderContentAddressMethod(ContentAddressMethod cam)
[](TextHashMethod &th) {
return std::string{"text:"} + printHashType(htSHA256);
},
[](FixedOutputHashMethod &fshm) {
[](CAPathHashMethod &fshm) {
return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType);
}
}, cam);
Expand Down Expand Up @@ -89,7 +102,7 @@ static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & r
if (splitPrefix(rest, "r:"))
method = FileIngestionMethod::Recursive;
HashType hashType = parseHashType_();
return FixedOutputHashMethod {
return CAPathHashMethod {
.fileIngestionMethod = method,
.hashType = std::move(hashType),
};
Expand All @@ -109,7 +122,7 @@ ContentAddress parseContentAddress(std::string_view rawCa) {
.hash = Hash::parseNonSRIUnprefixed(rest, htSHA256)
});
},
[&](FixedOutputHashMethod fohMethod) {
[&](CAPathHashMethod fohMethod) {
return ContentAddress(FixedOutputHash {
.method = fohMethod.fileIngestionMethod,
.hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)),
Expand Down
24 changes: 13 additions & 11 deletions src/libstore/content-address.hh
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,17 @@ enum struct FileIngestionMethod : uint8_t {
Recursive = true
};

/*
We only have one way to hash text with references, so this is single-value
type is only useful in std::variant.
*/
struct TextHashMethod { };
struct CAPathHashMethod {
FileIngestionMethod fileIngestionMethod;
HashType hashType;
std::string print() const;
};

struct TextHash {
Hash hash;
};
Expand All @@ -19,6 +30,7 @@ struct FixedOutputHash {
FileIngestionMethod method;
Hash hash;
std::string printMethodAlgo() const;
CAPathHashMethod hashMethod() const;
};

/*
Expand Down Expand Up @@ -55,19 +67,9 @@ std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt);

Hash getContentAddressHash(const ContentAddress & ca);

/*
We only have one way to hash text with references, so this is single-value
type is only useful in std::variant.
*/
struct TextHashMethod { };
struct FixedOutputHashMethod {
FileIngestionMethod fileIngestionMethod;
HashType hashType;
};

typedef std::variant<
TextHashMethod,
FixedOutputHashMethod
CAPathHashMethod
> ContentAddressMethod;

ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod);
Expand Down
2 changes: 1 addition & 1 deletion src/libstore/daemon.cc
Original file line number Diff line number Diff line change
Expand Up @@ -388,7 +388,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
auto path = store->addTextToStore(name, contents, refs, repair);
return store->queryPathInfo(path);
},
[&](FixedOutputHashMethod &fohm) {
[&](CAPathHashMethod &fohm) {
if (!refs.empty())
throw UnimplementedError("cannot yet have refs with flat or nar-hashed data");
auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair);
Expand Down
133 changes: 66 additions & 67 deletions src/libstore/derivations.cc
Original file line number Diff line number Diff line change
Expand Up @@ -9,24 +9,35 @@ namespace nix {

std::optional<StorePath> DerivationOutput::path(const Store & store, std::string_view drvName, std::string_view outputName) const
{
return std::visit(overloaded {
[](DerivationOutputInputAddressed doi) -> std::optional<StorePath> {
return { doi.path };
},
[&](DerivationOutputCAFixed dof) -> std::optional<StorePath> {
return {
dof.path(store, drvName, outputName)
};
},
[](DerivationOutputCAFloating dof) -> std::optional<StorePath> {
return std::nullopt;
},
[](DerivationOutputDeferred) -> std::optional<StorePath> {
return std::nullopt;
},
}, output);
if (auto dof = std::get_if<DerivationOutputCAFixed>(&output))
return { dof->path(store, drvName, outputName) };
if (auto doi = std::get_if<DerivationOutputInputAddressed>(&output))
return { doi->path };
return std::nullopt;
}

std::optional<CAPathHashMethod> DerivationOutput::caHashMethod() const
{
if (auto dof = std::get_if<DerivationOutputCAFixed>(&output))
return { dof->hash.hashMethod() };
if (auto dof = std::get_if<DerivationOutputCAFloating>(&output))
return { dof->hashMethod };
return std::nullopt;
}

std::optional<Hash> DerivationOutput::hash() const
{
if (auto dof = std::get_if<DerivationOutputCAFixed>(&output))
return { dof->hash.hash };
return std::nullopt;
}

std::optional<std::string> DerivationOutput::hashMetaData() const
{
if (auto hashMethod = caHashMethod())
return { hashMethod->print() };
return std::nullopt;
}

StorePath DerivationOutputCAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const {
return store.makeFixedOutputPath(
Expand Down Expand Up @@ -180,8 +191,10 @@ static DerivationOutput parseDerivationOutput(const Store & store,
assert(pathS == "");
return DerivationOutput {
.output = DerivationOutputCAFloating {
.method = std::move(method),
.hashType = std::move(hashType),
.hashMethod = CAPathHashMethod {
.fileIngestionMethod = std::move(method),
.hashType = std::move(hashType),
},
},
};
}
Expand Down Expand Up @@ -320,28 +333,18 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
for (auto & i : outputs) {
if (first) first = false; else s += ',';
s += '('; printUnquotedString(s, i.first);
std::visit(overloaded {
[&](DerivationOutputInputAddressed doi) {
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(doi.path));
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, "");
},
[&](DerivationOutputCAFixed dof) {
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first)));
s += ','; printUnquotedString(s, dof.hash.printMethodAlgo());
s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false));
},
[&](DerivationOutputCAFloating dof) {
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
s += ','; printUnquotedString(s, "");
},
[&](DerivationOutputDeferred) {
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, "");
}
}, i.second.output);
s += ',';
if (auto outPath = i.second.path(store, name, i.first))
printUnquotedString(s, maskOutputs ? "" : store.printStorePath(*outPath));
else
printUnquotedString(s, "");
s += ',';
printUnquotedString(s, i.second.hashMetaData().value_or(""));
s += ',';
if (auto hash = i.second.hash())
printUnquotedString(s, hash->to_string(Base16, false));
else
printUnquotedString(s, "");
s += ')';
}

Expand Down Expand Up @@ -418,9 +421,9 @@ DerivationType BasicDerivation::type() const
[&](DerivationOutputCAFloating dof) {
floatingCAOutputs.insert(i.first);
if (!floatingHashType) {
floatingHashType = dof.hashType;
floatingHashType = dof.hashMethod.hashType;
} else {
if (*floatingHashType != dof.hashType)
if (*floatingHashType != dof.hashMethod.hashType)
throw Error("All floating outputs must use the same hash type");
}
},
Expand Down Expand Up @@ -450,7 +453,6 @@ DerivationType BasicDerivation::type() const
}
}


DrvHashes drvHashes;

/* pathDerivationModulo and hashDerivationModulo are mutually recursive
Expand Down Expand Up @@ -494,7 +496,6 @@ static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath
*/
DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
{
/* Return a fixed hash for fixed-output derivations. */
switch (drv.type()) {
case DerivationType::CAFloating:
return UnknownHashes {};
Expand Down Expand Up @@ -628,34 +629,22 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv,
return in;
}


void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv)
{
out << drv.outputs.size();
for (auto & i : drv.outputs) {
out << i.first;
std::visit(overloaded {
[&](DerivationOutputInputAddressed doi) {
out << store.printStorePath(doi.path)
<< ""
<< "";
},
[&](DerivationOutputCAFixed dof) {
out << store.printStorePath(dof.path(store, drv.name, i.first))
<< dof.hash.printMethodAlgo()
<< dof.hash.hash.to_string(Base16, false);
},
[&](DerivationOutputCAFloating dof) {
out << ""
<< (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
<< "";
},
[&](DerivationOutputDeferred) {
out << ""
<< ""
<< "";
},
}, i.second.output);
if (auto outPath = i.second.path(store, drv.name, i.first))
out << store.printStorePath(*outPath);
else
out << "";

out << i.second.hashMetaData().value_or("");

if (auto hash = i.second.hash())
out << hash->to_string(Base16, false);
else
out << "";
}
worker_proto::write(store, out, drv.inputSrcs);
out << drv.platform << drv.builder << drv.args;
Expand Down Expand Up @@ -720,6 +709,16 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String

Sync<DrvPathResolutions> drvPathResolutions;

bool Derivation::needsRewriting() const
{
auto myType = type();
if (myType == DerivationType::CAFloating && !inputDrvs.empty())
return true;
if (myType == DerivationType::DeferredInputAddressed)
return true;
return false;
}

std::optional<BasicDerivation> Derivation::tryResolve(Store & store) {
BasicDerivation resolved { *this };

Expand Down
Loading