mirror of
https://github.com/NixOS/nix
synced 2025-06-28 13:41:15 +02:00
Merge branch 'path-info' into ca-drv-exotic
This commit is contained in:
John Ericson
commit
08b8657978
85 changed files with 1319 additions and 590 deletions
|
|
@ -1,6 +1,7 @@
|
|||
#pragma once
|
||||
|
||||
#include "realisation.hh"
|
||||
#include "derived-path.hh"
|
||||
|
||||
#include <string>
|
||||
#include <chrono>
|
||||
|
|
@ -30,6 +31,8 @@ struct BuildResult
|
|||
ResolvesToAlreadyValid,
|
||||
NoSubstituters,
|
||||
} status = MiscFailure;
|
||||
|
||||
// FIXME: include entire ErrorInfo object.
|
||||
std::string errorMsg;
|
||||
|
||||
std::string toString() const {
|
||||
|
|
|
|||
|
|
@ -204,9 +204,34 @@ void DerivationGoal::haveDerivation()
|
|||
{
|
||||
trace("have derivation");
|
||||
|
||||
parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
|
||||
|
||||
if (!drv->type().hasKnownOutputPaths())
|
||||
settings.requireExperimentalFeature(Xp::CaDerivations);
|
||||
|
||||
if (!drv->type().isPure()) {
|
||||
settings.requireExperimentalFeature(Xp::ImpureDerivations);
|
||||
|
||||
for (auto & [outputName, output] : drv->outputs) {
|
||||
auto randomPath = StorePath::random(outputPathName(drv->name, outputName));
|
||||
assert(!worker.store.isValidPath(randomPath));
|
||||
initialOutputs.insert({
|
||||
outputName,
|
||||
InitialOutput {
|
||||
.wanted = true,
|
||||
.outputHash = impureOutputHash,
|
||||
.known = InitialOutputStatus {
|
||||
.path = randomPath,
|
||||
.status = PathStatus::Absent
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
gaveUpOnSubstitution();
|
||||
return;
|
||||
}
|
||||
|
||||
for (auto & i : drv->outputsAndOptPaths(worker.store))
|
||||
if (i.second.second)
|
||||
worker.store.addTempRoot(*i.second.second);
|
||||
|
|
@ -230,9 +255,6 @@ void DerivationGoal::haveDerivation()
|
|||
return;
|
||||
}
|
||||
|
||||
parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
|
||||
|
||||
|
||||
/* We are first going to try to create the invalid output paths
|
||||
through substitutes. If that doesn't work, we'll build
|
||||
them. */
|
||||
|
|
@ -266,6 +288,8 @@ void DerivationGoal::outputsSubstitutionTried()
|
|||
{
|
||||
trace("all outputs substituted (maybe)");
|
||||
|
||||
assert(drv->type().isPure());
|
||||
|
||||
if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) {
|
||||
done(BuildResult::TransientFailure, {},
|
||||
Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
|
||||
|
|
@ -315,9 +339,21 @@ void DerivationGoal::outputsSubstitutionTried()
|
|||
void DerivationGoal::gaveUpOnSubstitution()
|
||||
{
|
||||
/* The inputs must be built before we can build this goal. */
|
||||
inputDrvOutputs.clear();
|
||||
if (useDerivation)
|
||||
for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs)
|
||||
for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs) {
|
||||
/* Ensure that pure, non-fixed-output derivations don't
|
||||
depend on impure derivations. */
|
||||
if (drv->type().isPure() && !drv->type().isFixed()) {
|
||||
auto inputDrv = worker.evalStore.readDerivation(i.first);
|
||||
if (!inputDrv.type().isPure())
|
||||
throw Error("pure derivation '%s' depends on impure derivation '%s'",
|
||||
worker.store.printStorePath(drvPath),
|
||||
worker.store.printStorePath(i.first));
|
||||
}
|
||||
|
||||
addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal));
|
||||
}
|
||||
|
||||
/* Copy the input sources from the eval store to the build
|
||||
store. */
|
||||
|
|
@ -345,6 +381,8 @@ void DerivationGoal::gaveUpOnSubstitution()
|
|||
|
||||
void DerivationGoal::repairClosure()
|
||||
{
|
||||
assert(drv->type().isPure());
|
||||
|
||||
/* If we're repairing, we now know that our own outputs are valid.
|
||||
Now check whether the other paths in the outputs closure are
|
||||
good. If not, then start derivation goals for the derivations
|
||||
|
|
@ -452,22 +490,24 @@ void DerivationGoal::inputsRealised()
|
|||
drvs. */
|
||||
: true);
|
||||
},
|
||||
[&](const DerivationType::Impure &) {
|
||||
return true;
|
||||
}
|
||||
}, drvType.raw());
|
||||
|
||||
if (resolveDrv)
|
||||
{
|
||||
if (resolveDrv && !fullDrv.inputDrvs.empty()) {
|
||||
settings.requireExperimentalFeature(Xp::CaDerivations);
|
||||
|
||||
/* We are be able to resolve this derivation based on the
|
||||
now-known results of dependencies. If so, we become a stub goal
|
||||
aliasing that resolved derivation goal */
|
||||
std::optional attempt = fullDrv.tryResolve(worker.store);
|
||||
now-known results of dependencies. If so, we become a
|
||||
stub goal aliasing that resolved derivation goal. */
|
||||
std::optional attempt = fullDrv.tryResolve(worker.store, inputDrvOutputs);
|
||||
assert(attempt);
|
||||
Derivation drvResolved { *std::move(attempt) };
|
||||
|
||||
auto pathResolved = writeDerivation(worker.store, drvResolved);
|
||||
|
||||
auto msg = fmt("Resolved derivation: '%s' -> '%s'",
|
||||
auto msg = fmt("resolved derivation: '%s' -> '%s'",
|
||||
worker.store.printStorePath(drvPath),
|
||||
worker.store.printStorePath(pathResolved));
|
||||
act = std::make_unique<Activity>(*logger, lvlInfo, actBuildWaiting, msg,
|
||||
|
|
@ -488,21 +528,13 @@ void DerivationGoal::inputsRealised()
|
|||
/* Add the relevant output closures of the input derivation
|
||||
`i' as input paths. Only add the closures of output paths
|
||||
that are specified as inputs. */
|
||||
assert(worker.evalStore.isValidPath(drvPath));
|
||||
auto outputs = worker.evalStore.queryPartialDerivationOutputMap(depDrvPath);
|
||||
for (auto & j : wantedDepOutputs) {
|
||||
if (outputs.count(j) > 0) {
|
||||
auto optRealizedInput = outputs.at(j);
|
||||
if (!optRealizedInput)
|
||||
throw Error(
|
||||
"derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output",
|
||||
worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
|
||||
worker.store.computeFSClosure(*optRealizedInput, inputPaths);
|
||||
} else
|
||||
for (auto & j : wantedDepOutputs)
|
||||
if (auto outPath = get(inputDrvOutputs, { depDrvPath, j }))
|
||||
worker.store.computeFSClosure(*outPath, inputPaths);
|
||||
else
|
||||
throw Error(
|
||||
"derivation '%s' requires non-existent output '%s' from input derivation '%s'",
|
||||
worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -923,7 +955,7 @@ void DerivationGoal::buildDone()
|
|||
st =
|
||||
dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic :
|
||||
statusOk(status) ? BuildResult::OutputRejected :
|
||||
derivationType.isImpure() || diskFull ? BuildResult::TransientFailure :
|
||||
!derivationType.isSandboxed() || diskFull ? BuildResult::TransientFailure :
|
||||
BuildResult::PermanentFailure;
|
||||
}
|
||||
|
||||
|
|
@ -934,60 +966,53 @@ void DerivationGoal::buildDone()
|
|||
|
||||
void DerivationGoal::resolvedFinished()
|
||||
{
|
||||
trace("resolved derivation finished");
|
||||
|
||||
assert(resolvedDrvGoal);
|
||||
auto resolvedDrv = *resolvedDrvGoal->drv;
|
||||
|
||||
auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv);
|
||||
|
||||
StorePathSet outputPaths;
|
||||
|
||||
// `wantedOutputs` might be empty, which means “all the outputs”
|
||||
auto realWantedOutputs = wantedOutputs;
|
||||
if (realWantedOutputs.empty())
|
||||
realWantedOutputs = resolvedDrv.outputNames();
|
||||
auto & resolvedResult = resolvedDrvGoal->buildResult;
|
||||
|
||||
DrvOutputs builtOutputs;
|
||||
|
||||
for (auto & wantedOutput : realWantedOutputs) {
|
||||
assert(initialOutputs.count(wantedOutput) != 0);
|
||||
assert(resolvedHashes.count(wantedOutput) != 0);
|
||||
auto realisation = worker.store.queryRealisation(
|
||||
DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput}
|
||||
);
|
||||
// We've just built it, but maybe the build failed, in which case the
|
||||
// realisation won't be there
|
||||
if (realisation) {
|
||||
auto newRealisation = *realisation;
|
||||
newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput};
|
||||
newRealisation.signatures.clear();
|
||||
newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath);
|
||||
signRealisation(newRealisation);
|
||||
worker.store.registerDrvOutput(newRealisation);
|
||||
outputPaths.insert(realisation->outPath);
|
||||
builtOutputs.emplace(realisation->id, *realisation);
|
||||
} else {
|
||||
// If we don't have a realisation, then it must mean that something
|
||||
// failed when building the resolved drv
|
||||
assert(!buildResult.success());
|
||||
if (resolvedResult.success()) {
|
||||
auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv);
|
||||
|
||||
StorePathSet outputPaths;
|
||||
|
||||
// `wantedOutputs` might be empty, which means “all the outputs”
|
||||
auto realWantedOutputs = wantedOutputs;
|
||||
if (realWantedOutputs.empty())
|
||||
realWantedOutputs = resolvedDrv.outputNames();
|
||||
|
||||
for (auto & wantedOutput : realWantedOutputs) {
|
||||
assert(initialOutputs.count(wantedOutput) != 0);
|
||||
assert(resolvedHashes.count(wantedOutput) != 0);
|
||||
auto realisation = resolvedResult.builtOutputs.at(
|
||||
DrvOutput { resolvedHashes.at(wantedOutput), wantedOutput });
|
||||
if (drv->type().isPure()) {
|
||||
auto newRealisation = realisation;
|
||||
newRealisation.id = DrvOutput { initialOutputs.at(wantedOutput).outputHash, wantedOutput };
|
||||
newRealisation.signatures.clear();
|
||||
if (!drv->type().isFixed())
|
||||
newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath);
|
||||
signRealisation(newRealisation);
|
||||
worker.store.registerDrvOutput(newRealisation);
|
||||
}
|
||||
outputPaths.insert(realisation.outPath);
|
||||
builtOutputs.emplace(realisation.id, realisation);
|
||||
}
|
||||
|
||||
runPostBuildHook(
|
||||
worker.store,
|
||||
*logger,
|
||||
drvPath,
|
||||
outputPaths
|
||||
);
|
||||
}
|
||||
|
||||
runPostBuildHook(
|
||||
worker.store,
|
||||
*logger,
|
||||
drvPath,
|
||||
outputPaths
|
||||
);
|
||||
|
||||
auto status = [&]() {
|
||||
auto & resolvedResult = resolvedDrvGoal->buildResult;
|
||||
switch (resolvedResult.status) {
|
||||
case BuildResult::AlreadyValid:
|
||||
return BuildResult::ResolvesToAlreadyValid;
|
||||
default:
|
||||
return resolvedResult.status;
|
||||
}
|
||||
}();
|
||||
auto status = resolvedResult.status;
|
||||
if (status == BuildResult::AlreadyValid)
|
||||
status = BuildResult::ResolvesToAlreadyValid;
|
||||
|
||||
done(status, std::move(builtOutputs));
|
||||
}
|
||||
|
|
@ -1236,6 +1261,7 @@ void DerivationGoal::flushLine()
|
|||
|
||||
std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap()
|
||||
{
|
||||
assert(drv->type().isPure());
|
||||
if (!useDerivation || drv->type().hasKnownOutputPaths()) {
|
||||
std::map<std::string, std::optional<StorePath>> res;
|
||||
for (auto & [name, output] : drv->outputs)
|
||||
|
|
@ -1248,6 +1274,7 @@ std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDeri
|
|||
|
||||
OutputPathMap DerivationGoal::queryDerivationOutputMap()
|
||||
{
|
||||
assert(drv->type().isPure());
|
||||
if (!useDerivation || drv->type().hasKnownOutputPaths()) {
|
||||
OutputPathMap res;
|
||||
for (auto & [name, output] : drv->outputsAndOptPaths(worker.store))
|
||||
|
|
@ -1261,6 +1288,8 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap()
|
|||
|
||||
std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
|
||||
{
|
||||
if (!drv->type().isPure()) return { false, {} };
|
||||
|
||||
bool checkHash = buildMode == bmRepair;
|
||||
auto wantedOutputsLeft = wantedOutputs;
|
||||
DrvOutputs validOutputs;
|
||||
|
|
@ -1304,6 +1333,7 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
|
|||
if (info.wanted && info.known && info.known->isValid())
|
||||
validOutputs.emplace(drvOutput, Realisation { drvOutput, info.known->path });
|
||||
}
|
||||
|
||||
// If we requested all the outputs via the empty set, we are always fine.
|
||||
// If we requested specific elements, the loop above removes all the valid
|
||||
// ones, so any that are left must be invalid.
|
||||
|
|
@ -1341,9 +1371,7 @@ void DerivationGoal::done(
|
|||
{
|
||||
buildResult.status = status;
|
||||
if (ex)
|
||||
// FIXME: strip: "error: "
|
||||
buildResult.errorMsg = ex->what();
|
||||
amDone(buildResult.success() ? ecSuccess : ecFailed, ex);
|
||||
buildResult.errorMsg = fmt("%s", normaltxt(ex->info().msg));
|
||||
if (buildResult.status == BuildResult::TimedOut)
|
||||
worker.timedOut = true;
|
||||
if (buildResult.status == BuildResult::PermanentFailure)
|
||||
|
|
@ -1370,7 +1398,21 @@ void DerivationGoal::done(
|
|||
fs.open(traceBuiltOutputsFile, std::fstream::out);
|
||||
fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl;
|
||||
}
|
||||
|
||||
amDone(buildResult.success() ? ecSuccess : ecFailed, ex);
|
||||
}
|
||||
|
||||
|
||||
void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result)
|
||||
{
|
||||
Goal::waiteeDone(waitee, result);
|
||||
|
||||
if (waitee->buildResult.success())
|
||||
if (auto bfd = std::get_if<DerivedPath::Built>(&waitee->buildResult.path))
|
||||
for (auto & [output, realisation] : waitee->buildResult.builtOutputs)
|
||||
inputDrvOutputs.insert_or_assign(
|
||||
{ bfd->drvPath, output.outputName },
|
||||
realisation.outPath);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -57,6 +57,11 @@ struct DerivationGoal : public Goal
|
|||
them. */
|
||||
StringSet wantedOutputs;
|
||||
|
||||
/* Mapping from input derivations + output names to actual store
|
||||
paths. This is filled in by waiteeDone() as each dependency
|
||||
finishes, before inputsRealised() is reached, */
|
||||
std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
|
||||
|
||||
/* Whether additional wanted outputs have been added. */
|
||||
bool needRestart = false;
|
||||
|
||||
|
|
@ -224,6 +229,8 @@ struct DerivationGoal : public Goal
|
|||
DrvOutputs builtOutputs = {},
|
||||
std::optional<Error> ex = {});
|
||||
|
||||
void waiteeDone(GoalPtr waitee, ExitCode result) override;
|
||||
|
||||
StorePathSet exportReferences(const StorePathSet & storePaths);
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ void DrvOutputSubstitutionGoal::tryNext()
|
|||
if (subs.size() == 0) {
|
||||
/* None left. Terminate this goal and let someone else deal
|
||||
with it. */
|
||||
debug("drv output '%s' is required, but there is no substituter that can provide it", id.to_string());
|
||||
debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string());
|
||||
|
||||
/* Hack: don't indicate failure if there were no substituters.
|
||||
In that case the calling derivation should just do a
|
||||
|
|
|
|||
|
|
@ -395,7 +395,7 @@ void LocalDerivationGoal::startBuilder()
|
|||
else if (settings.sandboxMode == smDisabled)
|
||||
useChroot = false;
|
||||
else if (settings.sandboxMode == smRelaxed)
|
||||
useChroot = !(derivationType.isImpure()) && !noChroot;
|
||||
useChroot = derivationType.isSandboxed() && !noChroot;
|
||||
}
|
||||
|
||||
auto & localStore = getLocalStore();
|
||||
|
|
@ -608,7 +608,7 @@ void LocalDerivationGoal::startBuilder()
|
|||
"nogroup:x:65534:\n", sandboxGid()));
|
||||
|
||||
/* Create /etc/hosts with localhost entry. */
|
||||
if (!(derivationType.isImpure()))
|
||||
if (derivationType.isSandboxed())
|
||||
writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
|
||||
|
||||
/* Make the closure of the inputs available in the chroot,
|
||||
|
|
@ -704,6 +704,9 @@ void LocalDerivationGoal::startBuilder()
|
|||
|
||||
/* Run the builder. */
|
||||
printMsg(lvlChatty, "executing builder '%1%'", drv->builder);
|
||||
printMsg(lvlChatty, "using builder args '%1%'", concatStringsSep(" ", drv->args));
|
||||
for (auto & i : drv->env)
|
||||
printMsg(lvlVomit, "setting builder env variable '%1%'='%2%'", i.first, i.second);
|
||||
|
||||
/* Create the log file. */
|
||||
Path logFile = openLogFile();
|
||||
|
|
@ -796,7 +799,7 @@ void LocalDerivationGoal::startBuilder()
|
|||
us.
|
||||
*/
|
||||
|
||||
if (!(derivationType.isImpure()))
|
||||
if (derivationType.isSandboxed())
|
||||
privateNetwork = true;
|
||||
|
||||
userNamespaceSync.create();
|
||||
|
|
@ -1060,7 +1063,7 @@ void LocalDerivationGoal::initEnv()
|
|||
to the builder is generally impure, but the output of
|
||||
fixed-output derivations is by definition pure (since we
|
||||
already know the cryptographic hash of the output). */
|
||||
if (derivationType.isImpure()) {
|
||||
if (!derivationType.isSandboxed()) {
|
||||
for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
|
||||
env[i] = getEnv(i).value_or("");
|
||||
}
|
||||
|
|
@ -1674,7 +1677,7 @@ void LocalDerivationGoal::runChild()
|
|||
/* Fixed-output derivations typically need to access the
|
||||
network, so give them access to /etc/resolv.conf and so
|
||||
on. */
|
||||
if (derivationType.isImpure()) {
|
||||
if (!derivationType.isSandboxed()) {
|
||||
// Only use nss functions to resolve hosts and
|
||||
// services. Don’t use it for anything else that may
|
||||
// be configured for this system. This limits the
|
||||
|
|
@ -1918,7 +1921,7 @@ void LocalDerivationGoal::runChild()
|
|||
|
||||
sandboxProfile += "(import \"sandbox-defaults.sb\")\n";
|
||||
|
||||
if (derivationType.isImpure())
|
||||
if (!derivationType.isSandboxed())
|
||||
sandboxProfile += "(import \"sandbox-network.sb\")\n";
|
||||
|
||||
/* Add the output paths we'll use at build-time to the chroot */
|
||||
|
|
@ -2402,6 +2405,13 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
|
|||
assert(false);
|
||||
},
|
||||
|
||||
[&](const DerivationOutput::Impure & doi) {
|
||||
return newInfoFromCA(DerivationOutput::CAFloating {
|
||||
.method = doi.method,
|
||||
.hashType = doi.hashType,
|
||||
});
|
||||
},
|
||||
|
||||
}, output.raw());
|
||||
|
||||
/* FIXME: set proper permissions in restorePath() so
|
||||
|
|
@ -2612,7 +2622,9 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
|
|||
},
|
||||
.outPath = newInfo.path
|
||||
};
|
||||
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
|
||||
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)
|
||||
&& drv->type().isPure())
|
||||
{
|
||||
signRealisation(thisRealisation);
|
||||
worker.store.registerDrvOutput(thisRealisation);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,9 +24,16 @@ PathSubstitutionGoal::~PathSubstitutionGoal()
|
|||
}
|
||||
|
||||
|
||||
void PathSubstitutionGoal::done(ExitCode result, BuildResult::Status status)
|
||||
void PathSubstitutionGoal::done(
|
||||
ExitCode result,
|
||||
BuildResult::Status status,
|
||||
std::optional<std::string> errorMsg)
|
||||
{
|
||||
buildResult.status = status;
|
||||
if (errorMsg) {
|
||||
debug(*errorMsg);
|
||||
buildResult.errorMsg = *errorMsg;
|
||||
}
|
||||
amDone(result);
|
||||
}
|
||||
|
||||
|
|
@ -67,12 +74,14 @@ void PathSubstitutionGoal::tryNext()
|
|||
if (subs.size() == 0) {
|
||||
/* None left. Terminate this goal and let someone else deal
|
||||
with it. */
|
||||
debug("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath));
|
||||
|
||||
/* Hack: don't indicate failure if there were no substituters.
|
||||
In that case the calling derivation should just do a
|
||||
build. */
|
||||
done(substituterFailed ? ecFailed : ecNoSubstituters, BuildResult::NoSubstituters);
|
||||
done(
|
||||
substituterFailed ? ecFailed : ecNoSubstituters,
|
||||
BuildResult::NoSubstituters,
|
||||
fmt("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath)));
|
||||
|
||||
if (substituterFailed) {
|
||||
worker.failedSubstitutions++;
|
||||
|
|
@ -171,10 +180,10 @@ void PathSubstitutionGoal::referencesValid()
|
|||
trace("all references realised");
|
||||
|
||||
if (nrFailed > 0) {
|
||||
debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath));
|
||||
done(
|
||||
nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed,
|
||||
BuildResult::DependencyFailed);
|
||||
BuildResult::DependencyFailed,
|
||||
fmt("some references of path '%s' could not be realised", worker.store.printStorePath(storePath)));
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -53,7 +53,10 @@ struct PathSubstitutionGoal : public Goal
|
|||
/* Content address for recomputing store path */
|
||||
std::optional<ContentAddress> ca;
|
||||
|
||||
void done(ExitCode result, BuildResult::Status status);
|
||||
void done(
|
||||
ExitCode result,
|
||||
BuildResult::Status status,
|
||||
std::optional<std::string> errorMsg = {});
|
||||
|
||||
public:
|
||||
PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
|
||||
|
|
|
|||
|
|
@ -26,11 +26,15 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
|
|||
[](const DerivationOutput::Deferred &) -> std::optional<StorePath> {
|
||||
return std::nullopt;
|
||||
},
|
||||
[](const DerivationOutput::Impure &) -> std::optional<StorePath> {
|
||||
return std::nullopt;
|
||||
},
|
||||
}, raw());
|
||||
}
|
||||
|
||||
|
||||
StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const {
|
||||
StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const
|
||||
{
|
||||
return store.makeFixedOutputPathFromCA(StorePathDescriptor {
|
||||
.name = outputPathName(drvName, outputName),
|
||||
.info = ca,
|
||||
|
|
@ -38,15 +42,27 @@ StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view
|
|||
}
|
||||
|
||||
|
||||
bool DerivationType::isCA() const {
|
||||
bool DerivationType::isCA() const
|
||||
{
|
||||
/* Normally we do the full `std::visit` to make sure we have
|
||||
exhaustively handled all variants, but so long as there is a
|
||||
variant called `ContentAddressed`, it must be the only one for
|
||||
which `isCA` is true for this to make sense!. */
|
||||
return std::holds_alternative<ContentAddressed>(raw());
|
||||
return std::visit(overloaded {
|
||||
[](const InputAddressed & ia) {
|
||||
return false;
|
||||
},
|
||||
[](const ContentAddressed & ca) {
|
||||
return true;
|
||||
},
|
||||
[](const Impure &) {
|
||||
return true;
|
||||
},
|
||||
}, raw());
|
||||
}
|
||||
|
||||
bool DerivationType::isFixed() const {
|
||||
bool DerivationType::isFixed() const
|
||||
{
|
||||
return std::visit(overloaded {
|
||||
[](const InputAddressed & ia) {
|
||||
return false;
|
||||
|
|
@ -54,10 +70,14 @@ bool DerivationType::isFixed() const {
|
|||
[](const ContentAddressed & ca) {
|
||||
return ca.fixed;
|
||||
},
|
||||
[](const Impure &) {
|
||||
return false;
|
||||
},
|
||||
}, raw());
|
||||
}
|
||||
|
||||
bool DerivationType::hasKnownOutputPaths() const {
|
||||
bool DerivationType::hasKnownOutputPaths() const
|
||||
{
|
||||
return std::visit(overloaded {
|
||||
[](const InputAddressed & ia) {
|
||||
return !ia.deferred;
|
||||
|
|
@ -65,17 +85,40 @@ bool DerivationType::hasKnownOutputPaths() const {
|
|||
[](const ContentAddressed & ca) {
|
||||
return ca.fixed;
|
||||
},
|
||||
[](const Impure &) {
|
||||
return false;
|
||||
},
|
||||
}, raw());
|
||||
}
|
||||
|
||||
|
||||
bool DerivationType::isImpure() const {
|
||||
bool DerivationType::isSandboxed() const
|
||||
{
|
||||
return std::visit(overloaded {
|
||||
[](const InputAddressed & ia) {
|
||||
return false;
|
||||
return true;
|
||||
},
|
||||
[](const ContentAddressed & ca) {
|
||||
return !ca.pure;
|
||||
return ca.sandboxed;
|
||||
},
|
||||
[](const Impure &) {
|
||||
return false;
|
||||
},
|
||||
}, raw());
|
||||
}
|
||||
|
||||
|
||||
bool DerivationType::isPure() const
|
||||
{
|
||||
return std::visit(overloaded {
|
||||
[](const InputAddressed & ia) {
|
||||
return true;
|
||||
},
|
||||
[](const ContentAddressed & ca) {
|
||||
return true;
|
||||
},
|
||||
[](const Impure &) {
|
||||
return false;
|
||||
},
|
||||
}, raw());
|
||||
}
|
||||
|
|
@ -174,7 +217,14 @@ static DerivationOutput parseDerivationOutput(const Store & store,
|
|||
if (hashAlgo != "") {
|
||||
ContentAddressMethod method = parseContentAddressingPrefix(hashAlgo);
|
||||
const auto hashType = parseHashType(hashAlgo);
|
||||
if (hashS != "") {
|
||||
if (hashS == "impure") {
|
||||
settings.requireExperimentalFeature(Xp::ImpureDerivations);
|
||||
assert(pathS == "");
|
||||
return DerivationOutput::Impure {
|
||||
.method = std::move(method),
|
||||
.hashType = std::move(hashType),
|
||||
};
|
||||
} else if (hashS != "") {
|
||||
validatePath(pathS);
|
||||
auto hash = Hash::parseNonSRIUnprefixed(hashS, hashType);
|
||||
return DerivationOutput::CAFixed {
|
||||
|
|
@ -343,6 +393,12 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs,
|
|||
s += ','; printUnquotedString(s, "");
|
||||
s += ','; printUnquotedString(s, "");
|
||||
s += ','; printUnquotedString(s, "");
|
||||
},
|
||||
[&](const DerivationOutputImpure & doi) {
|
||||
// FIXME
|
||||
s += ','; printUnquotedString(s, "");
|
||||
s += ','; printUnquotedString(s, makeContentAddressingPrefix(doi.method) + printHashType(doi.hashType));
|
||||
s += ','; printUnquotedString(s, "impure");
|
||||
}
|
||||
}, i.second.raw());
|
||||
s += ')';
|
||||
|
|
@ -408,8 +464,14 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName
|
|||
|
||||
DerivationType BasicDerivation::type() const
|
||||
{
|
||||
std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs, deferredIAOutputs;
|
||||
std::set<std::string_view>
|
||||
inputAddressedOutputs,
|
||||
fixedCAOutputs,
|
||||
floatingCAOutputs,
|
||||
deferredIAOutputs,
|
||||
impureOutputs;
|
||||
std::optional<HashType> floatingHashType;
|
||||
|
||||
for (auto & i : outputs) {
|
||||
std::visit(overloaded {
|
||||
[&](const DerivationOutput::InputAddressed &) {
|
||||
|
|
@ -424,43 +486,78 @@ DerivationType BasicDerivation::type() const
|
|||
floatingHashType = dof.hashType;
|
||||
} else {
|
||||
if (*floatingHashType != dof.hashType)
|
||||
throw Error("All floating outputs must use the same hash type");
|
||||
throw Error("all floating outputs must use the same hash type");
|
||||
}
|
||||
},
|
||||
[&](const DerivationOutput::Deferred &) {
|
||||
deferredIAOutputs.insert(i.first);
|
||||
deferredIAOutputs.insert(i.first);
|
||||
},
|
||||
[&](const DerivationOutput::Impure &) {
|
||||
impureOutputs.insert(i.first);
|
||||
},
|
||||
}, i.second.raw());
|
||||
}
|
||||
|
||||
if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
|
||||
throw Error("Must have at least one output");
|
||||
} else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
|
||||
if (inputAddressedOutputs.empty()
|
||||
&& fixedCAOutputs.empty()
|
||||
&& floatingCAOutputs.empty()
|
||||
&& deferredIAOutputs.empty()
|
||||
&& impureOutputs.empty())
|
||||
throw Error("must have at least one output");
|
||||
|
||||
if (!inputAddressedOutputs.empty()
|
||||
&& fixedCAOutputs.empty()
|
||||
&& floatingCAOutputs.empty()
|
||||
&& deferredIAOutputs.empty()
|
||||
&& impureOutputs.empty())
|
||||
return DerivationType::InputAddressed {
|
||||
.deferred = false,
|
||||
};
|
||||
} else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
|
||||
|
||||
if (inputAddressedOutputs.empty()
|
||||
&& !fixedCAOutputs.empty()
|
||||
&& floatingCAOutputs.empty()
|
||||
&& deferredIAOutputs.empty()
|
||||
&& impureOutputs.empty())
|
||||
{
|
||||
if (fixedCAOutputs.size() > 1)
|
||||
// FIXME: Experimental feature?
|
||||
throw Error("Only one fixed output is allowed for now");
|
||||
throw Error("only one fixed output is allowed for now");
|
||||
if (*fixedCAOutputs.begin() != "out")
|
||||
throw Error("Single fixed output must be named \"out\"");
|
||||
throw Error("single fixed output must be named \"out\"");
|
||||
return DerivationType::ContentAddressed {
|
||||
.pure = false,
|
||||
.sandboxed = false,
|
||||
.fixed = true,
|
||||
};
|
||||
} else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
|
||||
}
|
||||
|
||||
if (inputAddressedOutputs.empty()
|
||||
&& fixedCAOutputs.empty()
|
||||
&& !floatingCAOutputs.empty()
|
||||
&& deferredIAOutputs.empty()
|
||||
&& impureOutputs.empty())
|
||||
return DerivationType::ContentAddressed {
|
||||
.pure = true,
|
||||
.sandboxed = true,
|
||||
.fixed = false,
|
||||
};
|
||||
} else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && !deferredIAOutputs.empty()) {
|
||||
|
||||
if (inputAddressedOutputs.empty()
|
||||
&& fixedCAOutputs.empty()
|
||||
&& floatingCAOutputs.empty()
|
||||
&& !deferredIAOutputs.empty()
|
||||
&& impureOutputs.empty())
|
||||
return DerivationType::InputAddressed {
|
||||
.deferred = true,
|
||||
};
|
||||
} else {
|
||||
throw Error("Can't mix derivation output types");
|
||||
}
|
||||
|
||||
if (inputAddressedOutputs.empty()
|
||||
&& fixedCAOutputs.empty()
|
||||
&& floatingCAOutputs.empty()
|
||||
&& deferredIAOutputs.empty()
|
||||
&& !impureOutputs.empty())
|
||||
return DerivationType::Impure { };
|
||||
|
||||
throw Error("can't mix derivation output types");
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -472,7 +569,7 @@ Sync<DrvHashes> drvHashes;
|
|||
/* Look up the derivation by value and memoize the
|
||||
`hashDerivationModulo` call.
|
||||
*/
|
||||
static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath & drvPath)
|
||||
static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPath)
|
||||
{
|
||||
{
|
||||
auto hashes = drvHashes.lock();
|
||||
|
|
@ -507,7 +604,7 @@ static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath &
|
|||
don't leak the provenance of fixed outputs, reducing pointless cache
|
||||
misses as the build itself won't know this.
|
||||
*/
|
||||
DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
|
||||
DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
|
||||
{
|
||||
auto type = drv.type();
|
||||
|
||||
|
|
@ -522,7 +619,20 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
|
|||
+ store.printStorePath(dof.path(store, drv.name, i.first)));
|
||||
outputHashes.insert_or_assign(i.first, std::move(hash));
|
||||
}
|
||||
return outputHashes;
|
||||
return DrvHash {
|
||||
.hashes = outputHashes,
|
||||
.kind = DrvHash::Kind::Regular,
|
||||
};
|
||||
}
|
||||
|
||||
if (!type.isPure()) {
|
||||
std::map<std::string, Hash> outputHashes;
|
||||
for (const auto & [outputName, _] : drv.outputs)
|
||||
outputHashes.insert_or_assign(outputName, impureOutputHash);
|
||||
return DrvHash {
|
||||
.hashes = outputHashes,
|
||||
.kind = DrvHash::Kind::Deferred,
|
||||
};
|
||||
}
|
||||
|
||||
auto kind = std::visit(overloaded {
|
||||
|
|
@ -536,67 +646,41 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
|
|||
? DrvHash::Kind::Regular
|
||||
: DrvHash::Kind::Deferred;
|
||||
},
|
||||
[](const DerivationType::Impure &) -> DrvHash::Kind {
|
||||
assert(false);
|
||||
}
|
||||
}, drv.type().raw());
|
||||
|
||||
/* For other derivations, replace the inputs paths with recursive
|
||||
calls to this function. */
|
||||
std::map<std::string, StringSet> inputs2;
|
||||
for (auto & [drvPath, inputOutputs0] : drv.inputDrvs) {
|
||||
// Avoid lambda capture restriction with standard / Clang
|
||||
auto & inputOutputs = inputOutputs0;
|
||||
const auto & res = pathDerivationModulo(store, drvPath);
|
||||
std::visit(overloaded {
|
||||
// Regular non-CA derivation, replace derivation
|
||||
[&](const DrvHash & drvHash) {
|
||||
kind |= drvHash.kind;
|
||||
inputs2.insert_or_assign(drvHash.hash.to_string(Base16, false), inputOutputs);
|
||||
},
|
||||
// CA derivation's output hashes
|
||||
[&](const CaOutputHashes & outputHashes) {
|
||||
std::set<std::string> justOut = { "out" };
|
||||
for (auto & output : inputOutputs) {
|
||||
/* Put each one in with a single "out" output.. */
|
||||
const auto h = outputHashes.at(output);
|
||||
inputs2.insert_or_assign(
|
||||
h.to_string(Base16, false),
|
||||
justOut);
|
||||
}
|
||||
},
|
||||
}, res.raw());
|
||||
if (res.kind == DrvHash::Kind::Deferred)
|
||||
kind = DrvHash::Kind::Deferred;
|
||||
for (auto & outputName : inputOutputs) {
|
||||
const auto h = res.hashes.at(outputName);
|
||||
inputs2[h.to_string(Base16, false)].insert(outputName);
|
||||
}
|
||||
}
|
||||
|
||||
auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
|
||||
|
||||
return DrvHash { .hash = hash, .kind = kind };
|
||||
}
|
||||
|
||||
|
||||
void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept
|
||||
{
|
||||
switch (other) {
|
||||
case DrvHash::Kind::Regular:
|
||||
break;
|
||||
case DrvHash::Kind::Deferred:
|
||||
self = other;
|
||||
break;
|
||||
std::map<std::string, Hash> outputHashes;
|
||||
for (const auto & [outputName, _] : drv.outputs) {
|
||||
outputHashes.insert_or_assign(outputName, hash);
|
||||
}
|
||||
|
||||
return DrvHash {
|
||||
.hashes = outputHashes,
|
||||
.kind = kind,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv)
|
||||
{
|
||||
std::map<std::string, Hash> res;
|
||||
std::visit(overloaded {
|
||||
[&](const DrvHash & drvHash) {
|
||||
for (auto & outputName : drv.outputNames()) {
|
||||
res.insert({outputName, drvHash.hash});
|
||||
}
|
||||
},
|
||||
[&](const CaOutputHashes & outputHashes) {
|
||||
res = outputHashes;
|
||||
},
|
||||
}, hashDerivationModulo(store, drv, true).raw());
|
||||
return res;
|
||||
return hashDerivationModulo(store, drv, true).hashes;
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -623,7 +707,8 @@ StringSet BasicDerivation::outputNames() const
|
|||
return names;
|
||||
}
|
||||
|
||||
DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const {
|
||||
DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const
|
||||
{
|
||||
DerivationOutputsAndOptPaths outsAndOptPaths;
|
||||
for (auto output : outputs)
|
||||
outsAndOptPaths.insert(std::make_pair(
|
||||
|
|
@ -634,7 +719,8 @@ DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & s
|
|||
return outsAndOptPaths;
|
||||
}
|
||||
|
||||
std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) {
|
||||
std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath)
|
||||
{
|
||||
auto nameWithSuffix = drvPath.name();
|
||||
constexpr std::string_view extension = ".drv";
|
||||
assert(hasSuffix(nameWithSuffix, extension));
|
||||
|
|
@ -696,6 +782,11 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
|
|||
<< ""
|
||||
<< "";
|
||||
},
|
||||
[&](const DerivationOutput::Impure & doi) {
|
||||
out << ""
|
||||
<< (makeContentAddressingPrefix(doi.method) + printHashType(doi.hashType))
|
||||
<< "impure";
|
||||
},
|
||||
}, i.second.raw());
|
||||
}
|
||||
worker_proto::write(store, out, drv.inputSrcs);
|
||||
|
|
@ -721,21 +812,19 @@ std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath
|
|||
}
|
||||
|
||||
|
||||
static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) {
|
||||
|
||||
debug("Rewriting the derivation");
|
||||
|
||||
for (auto &rewrite: rewrites) {
|
||||
static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites)
|
||||
{
|
||||
for (auto & rewrite : rewrites) {
|
||||
debug("rewriting %s as %s", rewrite.first, rewrite.second);
|
||||
}
|
||||
|
||||
drv.builder = rewriteStrings(drv.builder, rewrites);
|
||||
for (auto & arg: drv.args) {
|
||||
for (auto & arg : drv.args) {
|
||||
arg = rewriteStrings(arg, rewrites);
|
||||
}
|
||||
|
||||
StringPairs newEnv;
|
||||
for (auto & envVar: drv.env) {
|
||||
for (auto & envVar : drv.env) {
|
||||
auto envName = rewriteStrings(envVar.first, rewrites);
|
||||
auto envValue = rewriteStrings(envVar.second, rewrites);
|
||||
newEnv.emplace(envName, envValue);
|
||||
|
|
@ -745,7 +834,7 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
|
|||
auto hashModulo = hashDerivationModulo(store, Derivation(drv), true);
|
||||
for (auto & [outputName, output] : drv.outputs) {
|
||||
if (std::holds_alternative<DerivationOutput::Deferred>(output.raw())) {
|
||||
auto & h = hashModulo.requireNoFixedNonDeferred();
|
||||
auto & h = hashModulo.hashes.at(outputName);
|
||||
auto outPath = store.makeOutputPath(outputName, h, drv.name);
|
||||
drv.env[outputName] = store.printStorePath(outPath);
|
||||
output = DerivationOutput::InputAddressed {
|
||||
|
|
@ -756,55 +845,48 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
|
|||
|
||||
}
|
||||
|
||||
const Hash & DrvHashModulo::requireNoFixedNonDeferred() const {
|
||||
auto * drvHashOpt = std::get_if<DrvHash>(&raw());
|
||||
assert(drvHashOpt);
|
||||
assert(drvHashOpt->kind == DrvHash::Kind::Regular);
|
||||
return drvHashOpt->hash;
|
||||
}
|
||||
|
||||
static bool tryResolveInput(
|
||||
Store & store, StorePathSet & inputSrcs, StringMap & inputRewrites,
|
||||
const StorePath & inputDrv, const StringSet & inputOutputs)
|
||||
std::optional<BasicDerivation> Derivation::tryResolve(Store & store) const
|
||||
{
|
||||
auto inputDrvOutputs = store.queryPartialDerivationOutputMap(inputDrv);
|
||||
std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
|
||||
|
||||
auto getOutput = [&](const std::string & outputName) {
|
||||
auto & actualPathOpt = inputDrvOutputs.at(outputName);
|
||||
if (!actualPathOpt)
|
||||
warn("output %s of input %s missing, aborting the resolving",
|
||||
outputName,
|
||||
store.printStorePath(inputDrv)
|
||||
);
|
||||
return actualPathOpt;
|
||||
};
|
||||
for (auto & input : inputDrvs)
|
||||
for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(input.first))
|
||||
if (outputPath)
|
||||
inputDrvOutputs.insert_or_assign({input.first, outputName}, *outputPath);
|
||||
|
||||
for (auto & outputName : inputOutputs) {
|
||||
auto actualPathOpt = getOutput(outputName);
|
||||
if (!actualPathOpt) return false;
|
||||
auto actualPath = *actualPathOpt;
|
||||
inputRewrites.emplace(
|
||||
downstreamPlaceholder(store, inputDrv, outputName),
|
||||
store.printStorePath(actualPath));
|
||||
inputSrcs.insert(std::move(actualPath));
|
||||
}
|
||||
|
||||
return true;
|
||||
return tryResolve(store, inputDrvOutputs);
|
||||
}
|
||||
|
||||
std::optional<BasicDerivation> Derivation::tryResolve(Store & store) {
|
||||
std::optional<BasicDerivation> Derivation::tryResolve(
|
||||
Store & store,
|
||||
const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const
|
||||
{
|
||||
BasicDerivation resolved { *this };
|
||||
|
||||
// Input paths that we'll want to rewrite in the derivation
|
||||
StringMap inputRewrites;
|
||||
|
||||
for (auto & [inputDrv, inputOutputs] : inputDrvs)
|
||||
if (!tryResolveInput(store, resolved.inputSrcs, inputRewrites, inputDrv, inputOutputs))
|
||||
return std::nullopt;
|
||||
for (auto & [inputDrv, inputOutputs] : inputDrvs) {
|
||||
for (auto & outputName : inputOutputs) {
|
||||
if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) {
|
||||
inputRewrites.emplace(
|
||||
downstreamPlaceholder(store, inputDrv, outputName),
|
||||
store.printStorePath(*actualPath));
|
||||
resolved.inputSrcs.insert(*actualPath);
|
||||
} else {
|
||||
warn("output '%s' of input '%s' missing, aborting the resolving",
|
||||
outputName,
|
||||
store.printStorePath(inputDrv));
|
||||
return {};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
rewriteDerivation(store, resolved, inputRewrites);
|
||||
|
||||
return resolved;
|
||||
}
|
||||
|
||||
const Hash impureOutputHash = hashString(htSHA256, "impure");
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -41,15 +41,26 @@ struct DerivationOutputCAFloating
|
|||
};
|
||||
|
||||
/* Input-addressed output which depends on a (CA) derivation whose hash isn't
|
||||
* known atm
|
||||
* known yet.
|
||||
*/
|
||||
struct DerivationOutputDeferred {};
|
||||
|
||||
/* Impure output which is moved to a content-addressed location (like
|
||||
CAFloating) but isn't registered as a realization.
|
||||
*/
|
||||
struct DerivationOutputImpure
|
||||
{
|
||||
/* information used for expected hash computation */
|
||||
ContentAddressMethod method;
|
||||
HashType hashType;
|
||||
};
|
||||
|
||||
typedef std::variant<
|
||||
DerivationOutputInputAddressed,
|
||||
DerivationOutputCAFixed,
|
||||
DerivationOutputCAFloating,
|
||||
DerivationOutputDeferred
|
||||
DerivationOutputDeferred,
|
||||
DerivationOutputImpure
|
||||
> _DerivationOutputRaw;
|
||||
|
||||
struct DerivationOutput : _DerivationOutputRaw
|
||||
|
|
@ -61,6 +72,7 @@ struct DerivationOutput : _DerivationOutputRaw
|
|||
using CAFixed = DerivationOutputCAFixed;
|
||||
using CAFloating = DerivationOutputCAFloating;
|
||||
using Deferred = DerivationOutputDeferred;
|
||||
using Impure = DerivationOutputImpure;
|
||||
|
||||
/* Note, when you use this function you should make sure that you're passing
|
||||
the right derivation name. When in doubt, you should use the safer
|
||||
|
|
@ -90,13 +102,17 @@ struct DerivationType_InputAddressed {
|
|||
};
|
||||
|
||||
struct DerivationType_ContentAddressed {
|
||||
bool pure;
|
||||
bool sandboxed;
|
||||
bool fixed;
|
||||
};
|
||||
|
||||
struct DerivationType_Impure {
|
||||
};
|
||||
|
||||
typedef std::variant<
|
||||
DerivationType_InputAddressed,
|
||||
DerivationType_ContentAddressed
|
||||
DerivationType_ContentAddressed,
|
||||
DerivationType_Impure
|
||||
> _DerivationTypeRaw;
|
||||
|
||||
struct DerivationType : _DerivationTypeRaw {
|
||||
|
|
@ -104,7 +120,7 @@ struct DerivationType : _DerivationTypeRaw {
|
|||
using Raw::Raw;
|
||||
using InputAddressed = DerivationType_InputAddressed;
|
||||
using ContentAddressed = DerivationType_ContentAddressed;
|
||||
|
||||
using Impure = DerivationType_Impure;
|
||||
|
||||
/* Do the outputs of the derivation have paths calculated from their content,
|
||||
or from the derivation itself? */
|
||||
|
|
@ -114,10 +130,18 @@ struct DerivationType : _DerivationTypeRaw {
|
|||
non-CA derivations. */
|
||||
bool isFixed() const;
|
||||
|
||||
/* Is the derivation impure and needs to access non-deterministic resources, or
|
||||
pure and can be sandboxed? Note that whether or not we actually sandbox the
|
||||
derivation is controlled separately. Never true for non-CA derivations. */
|
||||
bool isImpure() const;
|
||||
/* Whether the derivation is fully sandboxed. If false, the
|
||||
sandbox is opened up, e.g. the derivation has access to the
|
||||
network. Note that whether or not we actually sandbox the
|
||||
derivation is controlled separately. Always true for non-CA
|
||||
derivations. */
|
||||
bool isSandboxed() const;
|
||||
|
||||
/* Whether the derivation is expected to produce the same result
|
||||
every time, and therefore it only needs to be built once. This
|
||||
is only false for derivations that have the attribute '__impure
|
||||
= true'. */
|
||||
bool isPure() const;
|
||||
|
||||
/* Does the derivation knows its own output paths?
|
||||
Only true when there's no floating-ca derivation involved in the
|
||||
|
|
@ -173,7 +197,14 @@ struct Derivation : BasicDerivation
|
|||
added directly to input sources.
|
||||
|
||||
2. Input placeholders are replaced with realized input store paths. */
|
||||
std::optional<BasicDerivation> tryResolve(Store & store);
|
||||
std::optional<BasicDerivation> tryResolve(Store & store) const;
|
||||
|
||||
/* Like the above, but instead of querying the Nix database for
|
||||
realisations, uses a given mapping from input derivation paths
|
||||
+ output names to actual output store paths. */
|
||||
std::optional<BasicDerivation> tryResolve(
|
||||
Store & store,
|
||||
const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const;
|
||||
|
||||
Derivation() = default;
|
||||
Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { }
|
||||
|
|
@ -202,14 +233,16 @@ bool isDerivation(const std::string & fileName);
|
|||
the output name is "out". */
|
||||
std::string outputPathName(std::string_view drvName, std::string_view outputName);
|
||||
|
||||
// known CA drv's output hashes, current just for fixed-output derivations
|
||||
// whose output hashes are always known since they are fixed up-front.
|
||||
typedef std::map<std::string, Hash> CaOutputHashes;
|
||||
|
||||
// The hashes modulo of a derivation.
|
||||
//
|
||||
// Each output is given a hash, although in practice only the content-addressed
|
||||
// derivations (fixed-output or not) will have a different hash for each
|
||||
// output.
|
||||
struct DrvHash {
|
||||
Hash hash;
|
||||
std::map<std::string, Hash> hashes;
|
||||
|
||||
enum struct Kind: bool {
|
||||
enum struct Kind : bool {
|
||||
// Statically determined derivations.
|
||||
// This hash will be directly used to compute the output paths
|
||||
Regular,
|
||||
|
|
@ -222,28 +255,6 @@ struct DrvHash {
|
|||
|
||||
void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept;
|
||||
|
||||
typedef std::variant<
|
||||
// Regular normalized derivation hash, and whether it was deferred (because
|
||||
// an ancestor derivation is a floating content addressed derivation).
|
||||
DrvHash,
|
||||
// Fixed-output derivation hashes
|
||||
CaOutputHashes
|
||||
> _DrvHashModuloRaw;
|
||||
|
||||
struct DrvHashModulo : _DrvHashModuloRaw {
|
||||
using Raw = _DrvHashModuloRaw;
|
||||
using Raw::Raw;
|
||||
|
||||
/* Get hash, throwing if it is per-output CA hashes or a
|
||||
deferred Drv hash.
|
||||
*/
|
||||
const Hash & requireNoFixedNonDeferred() const;
|
||||
|
||||
inline const Raw & raw() const {
|
||||
return static_cast<const Raw &>(*this);
|
||||
}
|
||||
};
|
||||
|
||||
/* Returns hashes with the details of fixed-output subderivations
|
||||
expunged.
|
||||
|
||||
|
|
@ -267,16 +278,18 @@ struct DrvHashModulo : _DrvHashModuloRaw {
|
|||
ATerm, after subderivations have been likewise expunged from that
|
||||
derivation.
|
||||
*/
|
||||
DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs);
|
||||
DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs);
|
||||
|
||||
/*
|
||||
Return a map associating each output to a hash that uniquely identifies its
|
||||
derivation (modulo the self-references).
|
||||
|
||||
FIXME: what is the Hash in this map?
|
||||
*/
|
||||
std::map<std::string, Hash> staticOutputHashes(Store& store, const Derivation& drv);
|
||||
std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv);
|
||||
|
||||
/* Memoisation of hashDerivationModulo(). */
|
||||
typedef std::map<StorePath, DrvHashModulo> DrvHashes;
|
||||
typedef std::map<StorePath, DrvHash> DrvHashes;
|
||||
|
||||
// FIXME: global, though at least thread-safe.
|
||||
extern Sync<DrvHashes> drvHashes;
|
||||
|
|
@ -306,4 +319,6 @@ std::string hashPlaceholder(const std::string_view outputName);
|
|||
dependency which is a CA derivation. */
|
||||
std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName);
|
||||
|
||||
extern const Hash impureOutputHash;
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,6 +26,9 @@ struct DerivedPathOpaque {
|
|||
nlohmann::json toJSON(ref<Store> store) const;
|
||||
std::string to_string(const Store & store) const;
|
||||
static DerivedPathOpaque parse(const Store & store, std::string_view);
|
||||
|
||||
bool operator < (const DerivedPathOpaque & b) const
|
||||
{ return path < b.path; }
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
@ -47,6 +50,9 @@ struct DerivedPathBuilt {
|
|||
std::string to_string(const Store & store) const;
|
||||
static DerivedPathBuilt parse(const Store & store, std::string_view);
|
||||
nlohmann::json toJSON(ref<Store> store) const;
|
||||
|
||||
bool operator < (const DerivedPathBuilt & b) const
|
||||
{ return std::make_pair(drvPath, outputs) < std::make_pair(b.drvPath, b.outputs); }
|
||||
};
|
||||
|
||||
using _DerivedPathRaw = std::variant<
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ struct FileTransferSettings : Config
|
|||
R"(
|
||||
The timeout (in seconds) for establishing connections in the
|
||||
binary cache substituter. It corresponds to `curl`’s
|
||||
`--connect-timeout` option.
|
||||
`--connect-timeout` option. A value of 0 means no limit.
|
||||
)"};
|
||||
|
||||
Setting<unsigned long> stalledDownloadTimeout{
|
||||
|
|
@ -123,8 +123,6 @@ public:
|
|||
|
||||
template<typename... Args>
|
||||
FileTransferError(FileTransfer::Error error, std::optional<std::string> response, const Args & ... args);
|
||||
|
||||
virtual const char* sname() const override { return "FileTransferError"; }
|
||||
};
|
||||
|
||||
bool isUri(std::string_view s);
|
||||
|
|
|
|||
|
|
@ -695,16 +695,15 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
|
|||
// combinations that are currently prohibited.
|
||||
drv.type();
|
||||
|
||||
std::optional<Hash> h;
|
||||
std::optional<DrvHash> hashesModulo;
|
||||
for (auto & i : drv.outputs) {
|
||||
std::visit(overloaded {
|
||||
[&](const DerivationOutput::InputAddressed & doia) {
|
||||
if (!h) {
|
||||
if (!hashesModulo) {
|
||||
// somewhat expensive so we do lazily
|
||||
auto h0 = hashDerivationModulo(*this, drv, true);
|
||||
h = h0.requireNoFixedNonDeferred();
|
||||
hashesModulo = hashDerivationModulo(*this, drv, true);
|
||||
}
|
||||
StorePath recomputed = makeOutputPath(i.first, *h, drvName);
|
||||
StorePath recomputed = makeOutputPath(i.first, hashesModulo->hashes.at(i.first), drvName);
|
||||
if (doia.path != recomputed)
|
||||
throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
|
||||
printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed));
|
||||
|
|
@ -720,6 +719,9 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
|
|||
[&](const DerivationOutput::Deferred &) {
|
||||
/* Nothing to check */
|
||||
},
|
||||
[&](const DerivationOutput::Impure &) {
|
||||
/* Nothing to check */
|
||||
},
|
||||
}, i.second.raw());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -288,15 +288,15 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
|
|||
{
|
||||
std::set<Realisation> inputRealisations;
|
||||
|
||||
for (const auto& [inputDrv, outputNames] : drv.inputDrvs) {
|
||||
for (const auto & [inputDrv, outputNames] : drv.inputDrvs) {
|
||||
auto outputHashes =
|
||||
staticOutputHashes(store, store.readDerivation(inputDrv));
|
||||
for (const auto& outputName : outputNames) {
|
||||
for (const auto & outputName : outputNames) {
|
||||
auto thisRealisation = store.queryRealisation(
|
||||
DrvOutput{outputHashes.at(outputName), outputName});
|
||||
if (!thisRealisation)
|
||||
throw Error(
|
||||
"output '%s' of derivation '%s' isn’t built", outputName,
|
||||
"output '%s' of derivation '%s' isn't built", outputName,
|
||||
store.printStorePath(inputDrv));
|
||||
inputRealisations.insert(*thisRealisation);
|
||||
}
|
||||
|
|
@ -306,4 +306,5 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
|
|||
|
||||
return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#include "store-api.hh"
|
||||
|
||||
#include <sodium.h>
|
||||
|
||||
namespace nix {
|
||||
|
||||
static void checkName(std::string_view path, std::string_view name)
|
||||
|
|
@ -41,6 +43,13 @@ bool StorePath::isDerivation() const
|
|||
|
||||
StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x");
|
||||
|
||||
StorePath StorePath::random(std::string_view name)
|
||||
{
|
||||
Hash hash(htSHA1);
|
||||
randombytes_buf(hash.hash, hash.hashSize);
|
||||
return StorePath(hash, name);
|
||||
}
|
||||
|
||||
StorePath Store::parseStorePath(std::string_view path) const
|
||||
{
|
||||
auto p = canonPath(std::string(path));
|
||||
|
|
|
|||
|
|
@ -59,6 +59,8 @@ public:
|
|||
}
|
||||
|
||||
static StorePath dummy;
|
||||
|
||||
static StorePath random(std::string_view name);
|
||||
};
|
||||
|
||||
typedef std::set<StorePath> StorePathSet;
|
||||
|
|
|
|||
|
|
@ -215,7 +215,6 @@ void handleSQLiteBusy(const SQLiteBusy & e)
|
|||
if (now > lastWarned + 10) {
|
||||
lastWarned = now;
|
||||
logWarning({
|
||||
.name = "Sqlite busy",
|
||||
.msg = hintfmt(e.what())
|
||||
});
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue