mirror of
https://github.com/NixOS/nix
synced 2025-06-24 22:11:15 +02:00
Split DerivationGoal in two
This separation of concerns is generally good, but in particular sets up for removing `addWantedOutputs` next.
This commit is contained in:
John Ericson
parent
d1295448e0
commit
3b617e471b
10 changed files with 157 additions and 1589 deletions
|
|
@ -1,3 +1,4 @@
|
|||
#include "nix/store/build/derivation-building-goal.hh"
|
||||
#include "nix/store/build/derivation-goal.hh"
|
||||
#ifndef _WIN32 // TODO enable build hook on Windows
|
||||
# include "nix/store/build/hook-instance.hh"
|
||||
|
|
@ -23,47 +24,35 @@
|
|||
|
||||
namespace nix {
|
||||
|
||||
DerivationGoal::DerivationGoal(const StorePath & drvPath,
|
||||
const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode)
|
||||
: Goal(worker, loadDerivation())
|
||||
DerivationBuildingGoal::DerivationBuildingGoal(const StorePath & drvPath, const Derivation & drv_,
|
||||
Worker & worker, BuildMode buildMode)
|
||||
: Goal(worker, gaveUpOnSubstitution())
|
||||
, drvPath(drvPath)
|
||||
, wantedOutputs(wantedOutputs)
|
||||
, buildMode(buildMode)
|
||||
{
|
||||
name = fmt(
|
||||
"building of '%s' from .drv file",
|
||||
DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store));
|
||||
drv = std::make_unique<Derivation>(drv_);
|
||||
|
||||
if (auto parsedOpt = StructuredAttrs::tryParse(drv->env)) {
|
||||
parsedDrv = std::make_unique<StructuredAttrs>(*parsedOpt);
|
||||
}
|
||||
try {
|
||||
drvOptions = std::make_unique<DerivationOptions>(
|
||||
DerivationOptions::fromStructuredAttrs(drv->env, parsedDrv.get()));
|
||||
} catch (Error & e) {
|
||||
e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath));
|
||||
throw;
|
||||
}
|
||||
|
||||
name = fmt("building of '%s' from in-memory derivation", worker.store.printStorePath(drvPath));
|
||||
trace("created");
|
||||
|
||||
mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
|
||||
worker.updateProgress();
|
||||
}
|
||||
|
||||
|
||||
DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
|
||||
const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode)
|
||||
: Goal(worker, haveDerivation())
|
||||
, drvPath(drvPath)
|
||||
, wantedOutputs(wantedOutputs)
|
||||
, buildMode(buildMode)
|
||||
{
|
||||
this->drv = std::make_unique<Derivation>(drv);
|
||||
|
||||
name = fmt(
|
||||
"building of '%s' from in-memory derivation",
|
||||
DerivedPath::Built { makeConstantStorePathRef(drvPath), drv.outputNames() }.to_string(worker.store));
|
||||
trace("created");
|
||||
|
||||
mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
|
||||
worker.updateProgress();
|
||||
|
||||
/* Prevent the .chroot directory from being
|
||||
garbage-collected. (See isActiveTempFile() in gc.cc.) */
|
||||
worker.store.addTempRoot(this->drvPath);
|
||||
}
|
||||
|
||||
|
||||
DerivationGoal::~DerivationGoal()
|
||||
DerivationBuildingGoal::~DerivationBuildingGoal()
|
||||
{
|
||||
/* Careful: we should never ever throw an exception from a
|
||||
destructor. */
|
||||
|
|
@ -78,17 +67,17 @@ DerivationGoal::~DerivationGoal()
|
|||
}
|
||||
|
||||
|
||||
std::string DerivationGoal::key()
|
||||
std::string DerivationBuildingGoal::key()
|
||||
{
|
||||
/* Ensure that derivations get built in order of their name,
|
||||
i.e. a derivation named "aardvark" always comes before
|
||||
"baboon". And substitution goals always happen before
|
||||
derivation goals (due to "b$"). */
|
||||
return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath);
|
||||
return "bd$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath);
|
||||
}
|
||||
|
||||
|
||||
void DerivationGoal::killChild()
|
||||
void DerivationBuildingGoal::killChild()
|
||||
{
|
||||
#ifndef _WIN32 // TODO enable build hook on Windows
|
||||
hook.reset();
|
||||
|
|
@ -112,7 +101,7 @@ void DerivationGoal::killChild()
|
|||
}
|
||||
|
||||
|
||||
void DerivationGoal::timedOut(Error && ex)
|
||||
void DerivationBuildingGoal::timedOut(Error && ex)
|
||||
{
|
||||
killChild();
|
||||
// We're not inside a coroutine, hence we can't use co_return here.
|
||||
|
|
@ -120,198 +109,6 @@ void DerivationGoal::timedOut(Error && ex)
|
|||
[[maybe_unused]] Done _ = done(BuildResult::TimedOut, {}, std::move(ex));
|
||||
}
|
||||
|
||||
void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs)
|
||||
{
|
||||
auto newWanted = wantedOutputs.union_(outputs);
|
||||
switch (needRestart) {
|
||||
case NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed:
|
||||
if (!newWanted.isSubsetOf(wantedOutputs))
|
||||
needRestart = NeedRestartForMoreOutputs::OutputsAddedDoNeed;
|
||||
break;
|
||||
case NeedRestartForMoreOutputs::OutputsAddedDoNeed:
|
||||
/* No need to check whether we added more outputs, because a
|
||||
restart is already queued up. */
|
||||
break;
|
||||
case NeedRestartForMoreOutputs::BuildInProgressWillNotNeed:
|
||||
/* We are already building all outputs, so it doesn't matter if
|
||||
we now want more. */
|
||||
break;
|
||||
};
|
||||
wantedOutputs = newWanted;
|
||||
}
|
||||
|
||||
|
||||
Goal::Co DerivationGoal::loadDerivation() {
|
||||
trace("local derivation");
|
||||
|
||||
{
|
||||
/* The first thing to do is to make sure that the derivation
|
||||
exists. If it doesn't, it may be created through a
|
||||
substitute. */
|
||||
|
||||
if (buildMode != bmNormal || !worker.evalStore.isValidPath(drvPath)) {
|
||||
Goals waitees{upcast_goal(worker.makePathSubstitutionGoal(drvPath))};
|
||||
co_await await(std::move(waitees));
|
||||
}
|
||||
|
||||
trace("loading derivation");
|
||||
|
||||
if (nrFailed != 0) {
|
||||
co_return done(BuildResult::MiscFailure, {}, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)));
|
||||
}
|
||||
|
||||
/* `drvPath' should already be a root, but let's be on the safe
|
||||
side: if the user forgot to make it a root, we wouldn't want
|
||||
things being garbage collected while we're busy. */
|
||||
worker.evalStore.addTempRoot(drvPath);
|
||||
|
||||
/* Get the derivation. It is probably in the eval store, but it might be inthe main store:
|
||||
|
||||
- Resolved derivation are resolved against main store realisations, and so must be stored there.
|
||||
|
||||
- Dynamic derivations are built, and so are found in the main store.
|
||||
*/
|
||||
for (auto * drvStore : { &worker.evalStore, &worker.store }) {
|
||||
if (drvStore->isValidPath(drvPath)) {
|
||||
drv = std::make_unique<Derivation>(drvStore->readDerivation(drvPath));
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert(drv);
|
||||
}
|
||||
|
||||
co_return haveDerivation();
|
||||
}
|
||||
|
||||
|
||||
Goal::Co DerivationGoal::haveDerivation()
|
||||
{
|
||||
trace("have derivation");
|
||||
|
||||
if (auto parsedOpt = StructuredAttrs::tryParse(drv->env)) {
|
||||
parsedDrv = std::make_unique<StructuredAttrs>(*parsedOpt);
|
||||
}
|
||||
try {
|
||||
drvOptions = std::make_unique<DerivationOptions>(
|
||||
DerivationOptions::fromStructuredAttrs(drv->env, parsedDrv.get()));
|
||||
} catch (Error & e) {
|
||||
e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath));
|
||||
throw;
|
||||
}
|
||||
|
||||
if (!drv->type().hasKnownOutputPaths())
|
||||
experimentalFeatureSettings.require(Xp::CaDerivations);
|
||||
|
||||
for (auto & i : drv->outputsAndOptPaths(worker.store))
|
||||
if (i.second.second)
|
||||
worker.store.addTempRoot(*i.second.second);
|
||||
|
||||
{
|
||||
bool impure = drv->type().isImpure();
|
||||
|
||||
if (impure) experimentalFeatureSettings.require(Xp::ImpureDerivations);
|
||||
|
||||
auto outputHashes = staticOutputHashes(worker.evalStore, *drv);
|
||||
for (auto & [outputName, outputHash] : outputHashes) {
|
||||
InitialOutput v{
|
||||
.wanted = true, // Will be refined later
|
||||
.outputHash = outputHash
|
||||
};
|
||||
|
||||
/* TODO we might want to also allow randomizing the paths
|
||||
for regular CA derivations, e.g. for sake of checking
|
||||
determinism. */
|
||||
if (impure) {
|
||||
v.known = InitialOutputStatus {
|
||||
.path = StorePath::random(outputPathName(drv->name, outputName)),
|
||||
.status = PathStatus::Absent,
|
||||
};
|
||||
}
|
||||
|
||||
initialOutputs.insert({
|
||||
outputName,
|
||||
std::move(v),
|
||||
});
|
||||
}
|
||||
|
||||
if (impure) {
|
||||
/* We don't yet have any safe way to cache an impure derivation at
|
||||
this step. */
|
||||
co_return gaveUpOnSubstitution();
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
/* Check what outputs paths are not already valid. */
|
||||
auto [allValid, validOutputs] = checkPathValidity();
|
||||
|
||||
/* If they are all valid, then we're done. */
|
||||
if (allValid && buildMode == bmNormal) {
|
||||
co_return done(BuildResult::AlreadyValid, std::move(validOutputs));
|
||||
}
|
||||
}
|
||||
|
||||
Goals waitees;
|
||||
|
||||
/* We are first going to try to create the invalid output paths
|
||||
through substitutes. If that doesn't work, we'll build
|
||||
them. */
|
||||
if (settings.useSubstitutes && drvOptions->substitutesAllowed())
|
||||
for (auto & [outputName, status] : initialOutputs) {
|
||||
if (!status.wanted) continue;
|
||||
if (!status.known)
|
||||
waitees.insert(
|
||||
upcast_goal(
|
||||
worker.makeDrvOutputSubstitutionGoal(
|
||||
DrvOutput{status.outputHash, outputName},
|
||||
buildMode == bmRepair ? Repair : NoRepair
|
||||
)
|
||||
)
|
||||
);
|
||||
else {
|
||||
auto * cap = getDerivationCA(*drv);
|
||||
waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(
|
||||
status.known->path,
|
||||
buildMode == bmRepair ? Repair : NoRepair,
|
||||
cap ? std::optional { *cap } : std::nullopt)));
|
||||
}
|
||||
}
|
||||
|
||||
co_await await(std::move(waitees));
|
||||
|
||||
trace("all outputs substituted (maybe)");
|
||||
|
||||
assert(!drv->type().isImpure());
|
||||
|
||||
if (nrFailed > 0 && nrFailed > nrNoSubstituters && !settings.tryFallback) {
|
||||
co_return done(BuildResult::TransientFailure, {},
|
||||
Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
|
||||
worker.store.printStorePath(drvPath)));
|
||||
}
|
||||
|
||||
nrFailed = nrNoSubstituters = 0;
|
||||
|
||||
if (needRestart == NeedRestartForMoreOutputs::OutputsAddedDoNeed) {
|
||||
needRestart = NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed;
|
||||
co_return haveDerivation();
|
||||
}
|
||||
|
||||
auto [allValid, validOutputs] = checkPathValidity();
|
||||
|
||||
if (buildMode == bmNormal && allValid) {
|
||||
co_return done(BuildResult::Substituted, std::move(validOutputs));
|
||||
}
|
||||
if (buildMode == bmRepair && allValid) {
|
||||
co_return repairClosure();
|
||||
}
|
||||
if (buildMode == bmCheck && !allValid)
|
||||
throw Error("some outputs of '%s' are not valid, so checking is not possible",
|
||||
worker.store.printStorePath(drvPath));
|
||||
|
||||
/* Nothing to wait for; tail call */
|
||||
co_return gaveUpOnSubstitution();
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Used for `inputGoals` local variable below
|
||||
|
|
@ -343,12 +140,8 @@ std::string showKnownOutputs(Store & store, const Derivation & drv)
|
|||
|
||||
/* At least one of the output paths could not be
|
||||
produced using a substitute. So we have to build instead. */
|
||||
Goal::Co DerivationGoal::gaveUpOnSubstitution()
|
||||
Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution()
|
||||
{
|
||||
/* At this point we are building all outputs, so if more are wanted there
|
||||
is no need to restart. */
|
||||
needRestart = NeedRestartForMoreOutputs::BuildInProgressWillNotNeed;
|
||||
|
||||
Goals waitees;
|
||||
|
||||
std::map<ref<const SingleDerivedPath>, GoalPtr, value_comparison> inputGoals;
|
||||
|
|
@ -501,8 +294,9 @@ Goal::Co DerivationGoal::gaveUpOnSubstitution()
|
|||
worker.store.printStorePath(pathResolved),
|
||||
});
|
||||
|
||||
// FIXME wanted outputs
|
||||
auto resolvedDrvGoal = worker.makeDerivationGoal(
|
||||
pathResolved, wantedOutputs, buildMode);
|
||||
pathResolved, OutputsSpec::All{}, buildMode);
|
||||
{
|
||||
Goals waitees{resolvedDrvGoal};
|
||||
co_await await(std::move(waitees));
|
||||
|
|
@ -511,7 +305,10 @@ Goal::Co DerivationGoal::gaveUpOnSubstitution()
|
|||
trace("resolved derivation finished");
|
||||
|
||||
auto resolvedDrv = *resolvedDrvGoal->drv;
|
||||
auto & resolvedResult = resolvedDrvGoal->buildResult;
|
||||
auto resolvedResult = resolvedDrvGoal->getBuildResult(DerivedPath::Built{
|
||||
.drvPath = makeConstantStorePathRef(pathResolved),
|
||||
.outputs = OutputsSpec::All{},
|
||||
});
|
||||
|
||||
SingleDrvOutputs builtOutputs;
|
||||
|
||||
|
|
@ -615,7 +412,7 @@ Goal::Co DerivationGoal::gaveUpOnSubstitution()
|
|||
co_return tryToBuild();
|
||||
}
|
||||
|
||||
void DerivationGoal::started()
|
||||
void DerivationBuildingGoal::started()
|
||||
{
|
||||
auto msg = fmt(
|
||||
buildMode == bmRepair ? "repairing outputs of '%s'" :
|
||||
|
|
@ -637,7 +434,7 @@ void DerivationGoal::started()
|
|||
worker.updateProgress();
|
||||
}
|
||||
|
||||
Goal::Co DerivationGoal::tryToBuild()
|
||||
Goal::Co DerivationBuildingGoal::tryToBuild()
|
||||
{
|
||||
trace("trying to build");
|
||||
|
||||
|
|
@ -773,15 +570,15 @@ Goal::Co DerivationGoal::tryToBuild()
|
|||
* Local implementation of these virtual methods, consider
|
||||
* this just a record of lambdas.
|
||||
*/
|
||||
struct DerivationGoalCallbacks : DerivationBuilderCallbacks
|
||||
struct DerivationBuildingGoalCallbacks : DerivationBuilderCallbacks
|
||||
{
|
||||
DerivationGoal & goal;
|
||||
DerivationBuildingGoal & goal;
|
||||
|
||||
DerivationGoalCallbacks(DerivationGoal & goal, std::unique_ptr<DerivationBuilder> & builder)
|
||||
DerivationBuildingGoalCallbacks(DerivationBuildingGoal & goal, std::unique_ptr<DerivationBuilder> & builder)
|
||||
: goal{goal}
|
||||
{}
|
||||
|
||||
~DerivationGoalCallbacks() override = default;
|
||||
~DerivationBuildingGoalCallbacks() override = default;
|
||||
|
||||
void childStarted(Descriptor builderOut) override
|
||||
{
|
||||
|
|
@ -826,7 +623,7 @@ Goal::Co DerivationGoal::tryToBuild()
|
|||
already be created, so we don't need to create it again. */
|
||||
builder = makeDerivationBuilder(
|
||||
worker.store,
|
||||
std::make_unique<DerivationGoalCallbacks>(*this, builder),
|
||||
std::make_unique<DerivationBuildingGoalCallbacks>(*this, builder),
|
||||
DerivationBuilderParams {
|
||||
drvPath,
|
||||
buildMode,
|
||||
|
|
@ -889,78 +686,6 @@ Goal::Co DerivationGoal::tryToBuild()
|
|||
}
|
||||
|
||||
|
||||
Goal::Co DerivationGoal::repairClosure()
|
||||
{
|
||||
assert(!drv->type().isImpure());
|
||||
|
||||
/* If we're repairing, we now know that our own outputs are valid.
|
||||
Now check whether the other paths in the outputs closure are
|
||||
good. If not, then start derivation goals for the derivations
|
||||
that produced those outputs. */
|
||||
|
||||
/* Get the output closure. */
|
||||
auto outputs = queryDerivationOutputMap();
|
||||
StorePathSet outputClosure;
|
||||
for (auto & i : outputs) {
|
||||
if (!wantedOutputs.contains(i.first)) continue;
|
||||
worker.store.computeFSClosure(i.second, outputClosure);
|
||||
}
|
||||
|
||||
/* Filter out our own outputs (which we have already checked). */
|
||||
for (auto & i : outputs)
|
||||
outputClosure.erase(i.second);
|
||||
|
||||
/* Get all dependencies of this derivation so that we know which
|
||||
derivation is responsible for which path in the output
|
||||
closure. */
|
||||
StorePathSet inputClosure;
|
||||
|
||||
/* If we're working from an in-memory derivation with no in-store
|
||||
`*.drv` file, we cannot do this part. */
|
||||
if (worker.store.isValidPath(drvPath))
|
||||
worker.store.computeFSClosure(drvPath, inputClosure);
|
||||
|
||||
std::map<StorePath, StorePath> outputsToDrv;
|
||||
for (auto & i : inputClosure)
|
||||
if (i.isDerivation()) {
|
||||
auto depOutputs = worker.store.queryPartialDerivationOutputMap(i, &worker.evalStore);
|
||||
for (auto & j : depOutputs)
|
||||
if (j.second)
|
||||
outputsToDrv.insert_or_assign(*j.second, i);
|
||||
}
|
||||
|
||||
Goals waitees;
|
||||
|
||||
/* Check each path (slow!). */
|
||||
for (auto & i : outputClosure) {
|
||||
if (worker.pathContentsGood(i)) continue;
|
||||
printError(
|
||||
"found corrupted or missing path '%s' in the output closure of '%s'",
|
||||
worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
|
||||
auto drvPath2 = outputsToDrv.find(i);
|
||||
if (drvPath2 == outputsToDrv.end())
|
||||
waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(i, Repair)));
|
||||
else
|
||||
waitees.insert(worker.makeGoal(
|
||||
DerivedPath::Built {
|
||||
.drvPath = makeConstantStorePathRef(drvPath2->second),
|
||||
.outputs = OutputsSpec::All { },
|
||||
},
|
||||
bmRepair));
|
||||
}
|
||||
|
||||
co_await await(std::move(waitees));
|
||||
|
||||
if (!waitees.empty()) {
|
||||
trace("closure repaired");
|
||||
if (nrFailed > 0)
|
||||
throw Error("some paths in the output closure of derivation '%s' could not be repaired",
|
||||
worker.store.printStorePath(drvPath));
|
||||
}
|
||||
co_return done(BuildResult::AlreadyValid, assertPathValidity());
|
||||
}
|
||||
|
||||
|
||||
void runPostBuildHook(
|
||||
Store & store,
|
||||
Logger & logger,
|
||||
|
|
@ -1020,7 +745,7 @@ void runPostBuildHook(
|
|||
}
|
||||
|
||||
|
||||
void DerivationGoal::appendLogTailErrorMsg(std::string & msg)
|
||||
void DerivationBuildingGoal::appendLogTailErrorMsg(std::string & msg)
|
||||
{
|
||||
if (!logger->isVerbose() && !logTail.empty()) {
|
||||
msg += fmt("\nLast %d log lines:\n", logTail.size());
|
||||
|
|
@ -1042,7 +767,7 @@ void DerivationGoal::appendLogTailErrorMsg(std::string & msg)
|
|||
}
|
||||
|
||||
|
||||
Goal::Co DerivationGoal::hookDone()
|
||||
Goal::Co DerivationBuildingGoal::hookDone()
|
||||
{
|
||||
#ifndef _WIN32
|
||||
assert(hook);
|
||||
|
|
@ -1129,7 +854,7 @@ Goal::Co DerivationGoal::hookDone()
|
|||
co_return done(BuildResult::Built, std::move(builtOutputs));
|
||||
}
|
||||
|
||||
HookReply DerivationGoal::tryBuildHook()
|
||||
HookReply DerivationBuildingGoal::tryBuildHook()
|
||||
{
|
||||
#ifdef _WIN32 // TODO enable build hook on Windows
|
||||
return rpDecline;
|
||||
|
|
@ -1244,7 +969,7 @@ HookReply DerivationGoal::tryBuildHook()
|
|||
}
|
||||
|
||||
|
||||
Path DerivationGoal::openLogFile()
|
||||
Path DerivationBuildingGoal::openLogFile()
|
||||
{
|
||||
logSize = 0;
|
||||
|
||||
|
|
@ -1282,7 +1007,7 @@ Path DerivationGoal::openLogFile()
|
|||
}
|
||||
|
||||
|
||||
void DerivationGoal::closeLogFile()
|
||||
void DerivationBuildingGoal::closeLogFile()
|
||||
{
|
||||
auto logSink2 = std::dynamic_pointer_cast<CompressionSink>(logSink);
|
||||
if (logSink2) logSink2->finish();
|
||||
|
|
@ -1292,7 +1017,7 @@ void DerivationGoal::closeLogFile()
|
|||
}
|
||||
|
||||
|
||||
bool DerivationGoal::isReadDesc(Descriptor fd)
|
||||
bool DerivationBuildingGoal::isReadDesc(Descriptor fd)
|
||||
{
|
||||
#ifdef _WIN32 // TODO enable build hook on Windows
|
||||
return false;
|
||||
|
|
@ -1304,7 +1029,7 @@ bool DerivationGoal::isReadDesc(Descriptor fd)
|
|||
#endif
|
||||
}
|
||||
|
||||
void DerivationGoal::handleChildOutput(Descriptor fd, std::string_view data)
|
||||
void DerivationBuildingGoal::handleChildOutput(Descriptor fd, std::string_view data)
|
||||
{
|
||||
// local & `ssh://`-builds are dealt with here.
|
||||
auto isWrittenToLog = isReadDesc(fd);
|
||||
|
|
@ -1375,14 +1100,14 @@ void DerivationGoal::handleChildOutput(Descriptor fd, std::string_view data)
|
|||
}
|
||||
|
||||
|
||||
void DerivationGoal::handleEOF(Descriptor fd)
|
||||
void DerivationBuildingGoal::handleEOF(Descriptor fd)
|
||||
{
|
||||
if (!currentLogLine.empty()) flushLine();
|
||||
worker.wakeUp(shared_from_this());
|
||||
}
|
||||
|
||||
|
||||
void DerivationGoal::flushLine()
|
||||
void DerivationBuildingGoal::flushLine()
|
||||
{
|
||||
if (handleJSONLogMessage(currentLogLine, *act, builderActivities, "the derivation builder", false))
|
||||
;
|
||||
|
|
@ -1399,7 +1124,7 @@ void DerivationGoal::flushLine()
|
|||
}
|
||||
|
||||
|
||||
std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap()
|
||||
std::map<std::string, std::optional<StorePath>> DerivationBuildingGoal::queryPartialDerivationOutputMap()
|
||||
{
|
||||
assert(!drv->type().isImpure());
|
||||
|
||||
|
|
@ -1415,35 +1140,11 @@ std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDeri
|
|||
return res;
|
||||
}
|
||||
|
||||
OutputPathMap DerivationGoal::queryDerivationOutputMap()
|
||||
{
|
||||
assert(!drv->type().isImpure());
|
||||
|
||||
for (auto * drvStore : { &worker.evalStore, &worker.store })
|
||||
if (drvStore->isValidPath(drvPath))
|
||||
return worker.store.queryDerivationOutputMap(drvPath, drvStore);
|
||||
|
||||
// See comment in `DerivationGoal::queryPartialDerivationOutputMap`.
|
||||
OutputPathMap res;
|
||||
for (auto & [name, output] : drv->outputsAndOptPaths(worker.store))
|
||||
res.insert_or_assign(name, *output.second);
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
std::pair<bool, SingleDrvOutputs> DerivationGoal::checkPathValidity()
|
||||
std::pair<bool, SingleDrvOutputs> DerivationBuildingGoal::checkPathValidity()
|
||||
{
|
||||
if (drv->type().isImpure()) return { false, {} };
|
||||
|
||||
bool checkHash = buildMode == bmRepair;
|
||||
auto wantedOutputsLeft = std::visit(overloaded {
|
||||
[&](const OutputsSpec::All &) {
|
||||
return StringSet {};
|
||||
},
|
||||
[&](const OutputsSpec::Names & names) {
|
||||
return static_cast<StringSet>(names);
|
||||
},
|
||||
}, wantedOutputs.raw);
|
||||
SingleDrvOutputs validOutputs;
|
||||
|
||||
for (auto & i : queryPartialDerivationOutputMap()) {
|
||||
|
|
@ -1452,9 +1153,7 @@ std::pair<bool, SingleDrvOutputs> DerivationGoal::checkPathValidity()
|
|||
// this is an invalid output, gets catched with (!wantedOutputsLeft.empty())
|
||||
continue;
|
||||
auto & info = *initialOutput;
|
||||
info.wanted = wantedOutputs.contains(i.first);
|
||||
if (info.wanted)
|
||||
wantedOutputsLeft.erase(i.first);
|
||||
info.wanted = true;
|
||||
if (i.second) {
|
||||
auto outputPath = *i.second;
|
||||
info.known = {
|
||||
|
|
@ -1490,14 +1189,6 @@ std::pair<bool, SingleDrvOutputs> DerivationGoal::checkPathValidity()
|
|||
validOutputs.emplace(i.first, Realisation { drvOutput, info.known->path });
|
||||
}
|
||||
|
||||
// If we requested all the outputs, we are always fine.
|
||||
// If we requested specific elements, the loop above removes all the valid
|
||||
// ones, so any that are left must be invalid.
|
||||
if (!wantedOutputsLeft.empty())
|
||||
throw Error("derivation '%s' does not have wanted outputs %s",
|
||||
worker.store.printStorePath(drvPath),
|
||||
concatStringsSep(", ", quoteStrings(wantedOutputsLeft)));
|
||||
|
||||
bool allValid = true;
|
||||
for (auto & [_, status] : initialOutputs) {
|
||||
if (!status.wanted) continue;
|
||||
|
|
@ -1511,7 +1202,7 @@ std::pair<bool, SingleDrvOutputs> DerivationGoal::checkPathValidity()
|
|||
}
|
||||
|
||||
|
||||
SingleDrvOutputs DerivationGoal::assertPathValidity()
|
||||
SingleDrvOutputs DerivationBuildingGoal::assertPathValidity()
|
||||
{
|
||||
auto [allValid, validOutputs] = checkPathValidity();
|
||||
if (!allValid)
|
||||
|
|
@ -1520,7 +1211,7 @@ SingleDrvOutputs DerivationGoal::assertPathValidity()
|
|||
}
|
||||
|
||||
|
||||
Goal::Done DerivationGoal::done(
|
||||
Goal::Done DerivationBuildingGoal::done(
|
||||
BuildResult::Status status,
|
||||
SingleDrvOutputs builtOutputs,
|
||||
std::optional<Error> ex)
|
||||
|
|
@ -1534,13 +1225,10 @@ Goal::Done DerivationGoal::done(
|
|||
if (buildResult.status == BuildResult::PermanentFailure)
|
||||
worker.permanentFailure = true;
|
||||
|
||||
mcExpectedBuilds.reset();
|
||||
mcRunningBuilds.reset();
|
||||
|
||||
if (buildResult.success()) {
|
||||
auto wantedBuiltOutputs = filterDrvOutputs(wantedOutputs, std::move(builtOutputs));
|
||||
assert(!wantedBuiltOutputs.empty());
|
||||
buildResult.builtOutputs = std::move(wantedBuiltOutputs);
|
||||
buildResult.builtOutputs = std::move(builtOutputs);
|
||||
if (status == BuildResult::Built)
|
||||
worker.doneBuilds++;
|
||||
} else {
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -155,7 +155,7 @@ Goal::Done Goal::amDone(ExitCode result, std::optional<Error> ex)
|
|||
exitCode = result;
|
||||
|
||||
if (ex) {
|
||||
if (!waiters.empty())
|
||||
if (!preserveException && !waiters.empty())
|
||||
logError(ex->info());
|
||||
else
|
||||
this->ex = std::move(*ex);
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
#include "nix/store/build/substitution-goal.hh"
|
||||
#include "nix/store/build/drv-output-substitution-goal.hh"
|
||||
#include "nix/store/build/derivation-goal.hh"
|
||||
#include "nix/store/build/derivation-building-goal.hh"
|
||||
#ifndef _WIN32 // TODO Enable building on Windows
|
||||
# include "nix/store/build/hook-instance.hh"
|
||||
#endif
|
||||
|
|
@ -87,6 +88,20 @@ std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath
|
|||
}
|
||||
|
||||
|
||||
std::shared_ptr<DerivationBuildingGoal> Worker::makeDerivationBuildingGoal(const StorePath & drvPath,
|
||||
const Derivation & drv, BuildMode buildMode)
|
||||
{
|
||||
std::weak_ptr<DerivationBuildingGoal> & goal_weak = derivationBuildingGoals[drvPath];
|
||||
auto goal = goal_weak.lock(); // FIXME
|
||||
if (!goal) {
|
||||
goal = std::make_shared<DerivationBuildingGoal>(drvPath, drv, *this, buildMode);
|
||||
goal_weak = goal;
|
||||
wakeUp(goal);
|
||||
}
|
||||
return goal;
|
||||
}
|
||||
|
||||
|
||||
std::shared_ptr<PathSubstitutionGoal> Worker::makePathSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
|
||||
{
|
||||
return initGoalIfNeeded(substitutionGoals[path], path, *this, repair, ca);
|
||||
|
|
@ -134,8 +149,9 @@ void Worker::removeGoal(GoalPtr goal)
|
|||
{
|
||||
if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
|
||||
nix::removeGoal(drvGoal, derivationGoals);
|
||||
else
|
||||
if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal))
|
||||
else if (auto drvBuildingGoal = std::dynamic_pointer_cast<DerivationBuildingGoal>(goal))
|
||||
nix::removeGoal(drvBuildingGoal, derivationBuildingGoals);
|
||||
else if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal))
|
||||
nix::removeGoal(subGoal, substitutionGoals);
|
||||
else if (auto subGoal = std::dynamic_pointer_cast<DrvOutputSubstitutionGoal>(goal))
|
||||
nix::removeGoal(subGoal, drvOutputSubstitutionGoals);
|
||||
|
|
@ -198,6 +214,9 @@ void Worker::childStarted(GoalPtr goal, const std::set<MuxablePipePollState::Com
|
|||
case JobCategory::Build:
|
||||
nrLocalBuilds++;
|
||||
break;
|
||||
case JobCategory::Administration:
|
||||
/* Intentionally not limited, see docs */
|
||||
break;
|
||||
default:
|
||||
unreachable();
|
||||
}
|
||||
|
|
@ -221,6 +240,9 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers)
|
|||
assert(nrLocalBuilds > 0);
|
||||
nrLocalBuilds--;
|
||||
break;
|
||||
case JobCategory::Administration:
|
||||
/* Intentionally not limited, see docs */
|
||||
break;
|
||||
default:
|
||||
unreachable();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,43 +31,11 @@ void runPostBuildHook(
|
|||
/**
|
||||
* A goal for building some or all of the outputs of a derivation.
|
||||
*/
|
||||
struct DerivationGoal : public Goal
|
||||
struct DerivationBuildingGoal : public Goal
|
||||
{
|
||||
/** The path of the derivation. */
|
||||
StorePath drvPath;
|
||||
|
||||
/**
|
||||
* The specific outputs that we need to build.
|
||||
*/
|
||||
OutputsSpec wantedOutputs;
|
||||
|
||||
/**
|
||||
* See `needRestart`; just for that field.
|
||||
*/
|
||||
enum struct NeedRestartForMoreOutputs {
|
||||
/**
|
||||
* The goal state machine is progressing based on the current value of
|
||||
* `wantedOutputs. No actions are needed.
|
||||
*/
|
||||
OutputsUnmodifedDontNeed,
|
||||
/**
|
||||
* `wantedOutputs` has been extended, but the state machine is
|
||||
* proceeding according to its old value, so we need to restart.
|
||||
*/
|
||||
OutputsAddedDoNeed,
|
||||
/**
|
||||
* The goal state machine has progressed to the point of doing a build,
|
||||
* in which case all outputs will be produced, so extensions to
|
||||
* `wantedOutputs` no longer require a restart.
|
||||
*/
|
||||
BuildInProgressWillNotNeed,
|
||||
};
|
||||
|
||||
/**
|
||||
* Whether additional wanted outputs have been added.
|
||||
*/
|
||||
NeedRestartForMoreOutputs needRestart = NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed;
|
||||
|
||||
/**
|
||||
* The derivation stored at drvPath.
|
||||
*/
|
||||
|
|
@ -125,7 +93,7 @@ struct DerivationGoal : public Goal
|
|||
|
||||
BuildMode buildMode;
|
||||
|
||||
std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
|
||||
std::unique_ptr<MaintainCount<uint64_t>> mcRunningBuilds;
|
||||
|
||||
std::unique_ptr<Activity> act;
|
||||
|
||||
|
|
@ -141,28 +109,18 @@ struct DerivationGoal : public Goal
|
|||
*/
|
||||
std::string machineName;
|
||||
|
||||
DerivationGoal(const StorePath & drvPath,
|
||||
const OutputsSpec & wantedOutputs, Worker & worker,
|
||||
DerivationBuildingGoal(const StorePath & drvPath, const Derivation & drv,
|
||||
Worker & worker,
|
||||
BuildMode buildMode = bmNormal);
|
||||
DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
|
||||
const OutputsSpec & wantedOutputs, Worker & worker,
|
||||
BuildMode buildMode = bmNormal);
|
||||
~DerivationGoal();
|
||||
~DerivationBuildingGoal();
|
||||
|
||||
void timedOut(Error && ex) override;
|
||||
|
||||
std::string key() override;
|
||||
|
||||
/**
|
||||
* Add wanted outputs to an already existing derivation goal.
|
||||
*/
|
||||
void addWantedOutputs(const OutputsSpec & outputs);
|
||||
|
||||
/**
|
||||
* The states.
|
||||
*/
|
||||
Co loadDerivation();
|
||||
Co haveDerivation();
|
||||
Co gaveUpOnSubstitution();
|
||||
Co tryToBuild();
|
||||
Co hookDone();
|
||||
|
|
@ -197,7 +155,6 @@ struct DerivationGoal : public Goal
|
|||
* there also is no DB entry.
|
||||
*/
|
||||
std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
|
||||
OutputPathMap queryDerivationOutputMap();
|
||||
|
||||
/**
|
||||
* Update 'initialOutputs' to determine the current status of the
|
||||
|
|
@ -218,8 +175,6 @@ struct DerivationGoal : public Goal
|
|||
*/
|
||||
void killChild();
|
||||
|
||||
Co repairClosure();
|
||||
|
||||
void started();
|
||||
|
||||
Done done(
|
||||
|
|
|
|||
|
|
@ -14,13 +14,6 @@ namespace nix {
|
|||
|
||||
using std::map;
|
||||
|
||||
#ifndef _WIN32 // TODO enable build hook on Windows
|
||||
struct HookInstance;
|
||||
struct DerivationBuilder;
|
||||
#endif
|
||||
|
||||
typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
|
||||
|
||||
/** Used internally */
|
||||
void runPostBuildHook(
|
||||
Store & store,
|
||||
|
|
@ -73,73 +66,15 @@ struct DerivationGoal : public Goal
|
|||
*/
|
||||
std::unique_ptr<Derivation> drv;
|
||||
|
||||
std::unique_ptr<StructuredAttrs> parsedDrv;
|
||||
std::unique_ptr<DerivationOptions> drvOptions;
|
||||
|
||||
/**
|
||||
* The remainder is state held during the build.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Locks on (fixed) output paths.
|
||||
*/
|
||||
PathLocks outputLocks;
|
||||
|
||||
/**
|
||||
* All input paths (that is, the union of FS closures of the
|
||||
* immediate input paths).
|
||||
*/
|
||||
StorePathSet inputPaths;
|
||||
|
||||
std::map<std::string, InitialOutput> initialOutputs;
|
||||
|
||||
/**
|
||||
* File descriptor for the log file.
|
||||
*/
|
||||
AutoCloseFD fdLogFile;
|
||||
std::shared_ptr<BufferedSink> logFileSink, logSink;
|
||||
|
||||
/**
|
||||
* Number of bytes received from the builder's stdout/stderr.
|
||||
*/
|
||||
unsigned long logSize;
|
||||
|
||||
/**
|
||||
* The most recent log lines.
|
||||
*/
|
||||
std::list<std::string> logTail;
|
||||
|
||||
std::string currentLogLine;
|
||||
size_t currentLogLinePos = 0; // to handle carriage return
|
||||
|
||||
std::string currentHookLine;
|
||||
|
||||
#ifndef _WIN32 // TODO enable build hook on Windows
|
||||
/**
|
||||
* The build hook.
|
||||
*/
|
||||
std::unique_ptr<HookInstance> hook;
|
||||
|
||||
std::unique_ptr<DerivationBuilder> builder;
|
||||
#endif
|
||||
|
||||
BuildMode buildMode;
|
||||
|
||||
std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
|
||||
|
||||
std::unique_ptr<Activity> act;
|
||||
|
||||
/**
|
||||
* Activity that denotes waiting for a lock.
|
||||
*/
|
||||
std::unique_ptr<Activity> actLock;
|
||||
|
||||
std::map<ActivityId, Activity> builderActivities;
|
||||
|
||||
/**
|
||||
* The remote machine on which we're building.
|
||||
*/
|
||||
std::string machineName;
|
||||
std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds;
|
||||
|
||||
DerivationGoal(const StorePath & drvPath,
|
||||
const OutputsSpec & wantedOutputs, Worker & worker,
|
||||
|
|
@ -147,9 +82,9 @@ struct DerivationGoal : public Goal
|
|||
DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
|
||||
const OutputsSpec & wantedOutputs, Worker & worker,
|
||||
BuildMode buildMode = bmNormal);
|
||||
~DerivationGoal();
|
||||
~DerivationGoal() = default;
|
||||
|
||||
void timedOut(Error && ex) override;
|
||||
void timedOut(Error && ex) override { unreachable(); };
|
||||
|
||||
std::string key() override;
|
||||
|
||||
|
|
@ -163,33 +98,6 @@ struct DerivationGoal : public Goal
|
|||
*/
|
||||
Co loadDerivation();
|
||||
Co haveDerivation();
|
||||
Co gaveUpOnSubstitution();
|
||||
Co tryToBuild();
|
||||
Co hookDone();
|
||||
|
||||
/**
|
||||
* Is the build hook willing to perform the build?
|
||||
*/
|
||||
HookReply tryBuildHook();
|
||||
|
||||
/**
|
||||
* Open a log file and a pipe to it.
|
||||
*/
|
||||
Path openLogFile();
|
||||
|
||||
/**
|
||||
* Close the log file.
|
||||
*/
|
||||
void closeLogFile();
|
||||
|
||||
bool isReadDesc(Descriptor fd);
|
||||
|
||||
/**
|
||||
* Callback used by the worker to write to the log.
|
||||
*/
|
||||
void handleChildOutput(Descriptor fd, std::string_view data) override;
|
||||
void handleEOF(Descriptor fd) override;
|
||||
void flushLine();
|
||||
|
||||
/**
|
||||
* Wrappers around the corresponding Store methods that first consult the
|
||||
|
|
@ -213,26 +121,15 @@ struct DerivationGoal : public Goal
|
|||
*/
|
||||
SingleDrvOutputs assertPathValidity();
|
||||
|
||||
/**
|
||||
* Forcibly kill the child process, if any.
|
||||
*/
|
||||
void killChild();
|
||||
|
||||
Co repairClosure();
|
||||
|
||||
void started();
|
||||
|
||||
Done done(
|
||||
BuildResult::Status status,
|
||||
SingleDrvOutputs builtOutputs = {},
|
||||
std::optional<Error> ex = {});
|
||||
|
||||
void appendLogTailErrorMsg(std::string & msg);
|
||||
|
||||
StorePathSet exportReferences(const StorePathSet & storePaths);
|
||||
|
||||
JobCategory jobCategory() const override {
|
||||
return JobCategory::Build;
|
||||
return JobCategory::Administration;
|
||||
};
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -50,6 +50,16 @@ enum struct JobCategory {
|
|||
* A substitution an arbitrary store object; it will use network resources.
|
||||
*/
|
||||
Substitution,
|
||||
/**
|
||||
* A goal that does no "real" work by itself, and just exists to depend on
|
||||
* other goals which *do* do real work. These goals therefore are not
|
||||
* limited.
|
||||
*
|
||||
* These goals cannot infinitely create themselves, so there is no risk of
|
||||
* a "fork bomb" type situation (which would be a problem even though the
|
||||
* goal do no real work) either.
|
||||
*/
|
||||
Administration,
|
||||
};
|
||||
|
||||
struct Goal : public std::enable_shared_from_this<Goal>
|
||||
|
|
@ -360,6 +370,17 @@ public:
|
|||
*/
|
||||
BuildResult getBuildResult(const DerivedPath &) const;
|
||||
|
||||
/**
|
||||
* Hack to say that this goal should not log `ex`, but instead keep
|
||||
* it around. Set by a waitee which sees itself as the designated
|
||||
* continuation of this goal, responsible for reporting its
|
||||
* successes or failures.
|
||||
*
|
||||
* @todo this is yet another not-nice hack in the goal system that
|
||||
* we ought to get rid of. See #11927
|
||||
*/
|
||||
bool preserveException = false;
|
||||
|
||||
/**
|
||||
* Exception containing an error message, if any.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ namespace nix {
|
|||
|
||||
/* Forward definition. */
|
||||
struct DerivationGoal;
|
||||
struct DerivationBuildingGoal;
|
||||
struct PathSubstitutionGoal;
|
||||
class DrvOutputSubstitutionGoal;
|
||||
|
||||
|
|
@ -104,6 +105,7 @@ private:
|
|||
* same derivation / path.
|
||||
*/
|
||||
std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals;
|
||||
std::map<StorePath, std::weak_ptr<DerivationBuildingGoal>> derivationBuildingGoals;
|
||||
std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals;
|
||||
std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;
|
||||
|
||||
|
|
@ -210,6 +212,13 @@ public:
|
|||
const StorePath & drvPath, const BasicDerivation & drv,
|
||||
const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal);
|
||||
|
||||
/**
|
||||
* @ref DerivationBuildingGoal "derivation goal"
|
||||
*/
|
||||
std::shared_ptr<DerivationBuildingGoal> makeDerivationBuildingGoal(
|
||||
const StorePath & drvPath, const Derivation & drv,
|
||||
BuildMode buildMode = bmNormal);
|
||||
|
||||
/**
|
||||
* @ref PathSubstitutionGoal "substitution goal"
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ headers = [config_pub_h] + files(
|
|||
'binary-cache-store.hh',
|
||||
'build-result.hh',
|
||||
'build/derivation-goal.hh',
|
||||
'build/derivation-building-goal.hh',
|
||||
'build/derivation-building-misc.hh',
|
||||
'build/drv-output-substitution-goal.hh',
|
||||
'build/goal.hh',
|
||||
|
|
|
|||
|
|
@ -254,6 +254,7 @@ sources = files(
|
|||
'binary-cache-store.cc',
|
||||
'build-result.cc',
|
||||
'build/derivation-goal.cc',
|
||||
'build/derivation-building-goal.cc',
|
||||
'build/drv-output-substitution-goal.cc',
|
||||
'build/entry-points.cc',
|
||||
'build/goal.cc',
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue