wroclaw-git-backups/nix
1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-06-25 10:41:16 +02:00
Code Activity

Split DerivationGoal in two

Browse source 
This separation of concerns is generally good, but in particular sets up
for removing `addWantedOutputs` next.
This commit is contained in:
John Ericson 2025-04-21 18:05:03 -04:00
parent d1295448e0
commit 3b617e471b
10 changed files with 157 additions and 1589 deletions
Download patch file Download diff file Expand all files Collapse all files

416
src/libstore/build/derivation-building-goal.cc
View file

@ -1,3 +1,4 @@
#include "nix/store/build/derivation-building-goal.hh"
#include "nix/store/build/derivation-goal.hh" #include "nix/store/build/derivation-goal.hh"
#ifndef _WIN32 // TODO enable build hook on Windows #ifndef _WIN32 // TODO enable build hook on Windows
# include "nix/store/build/hook-instance.hh" # include "nix/store/build/hook-instance.hh"
@ -23,47 +24,35 @@
namespace nix { namespace nix {
DerivationGoal::DerivationGoal(const StorePath & drvPath, DerivationBuildingGoal::DerivationBuildingGoal(const StorePath & drvPath, const Derivation & drv_,
const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) Worker & worker, BuildMode buildMode)
: Goal(worker, loadDerivation()) : Goal(worker, gaveUpOnSubstitution())
, drvPath(drvPath) , drvPath(drvPath)
, wantedOutputs(wantedOutputs)
, buildMode(buildMode) , buildMode(buildMode)
{ {
name = fmt( drv = std::make_unique<Derivation>(drv_);
"building of '%s' from .drv file",
DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store)); if (auto parsedOpt = StructuredAttrs::tryParse(drv->env)) {
parsedDrv = std::make_unique<StructuredAttrs>(*parsedOpt);
}
try {
drvOptions = std::make_unique<DerivationOptions>(
DerivationOptions::fromStructuredAttrs(drv->env, parsedDrv.get()));
} catch (Error & e) {
e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath));
throw;
}
name = fmt("building of '%s' from in-memory derivation", worker.store.printStorePath(drvPath));
trace("created"); trace("created");
mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
worker.updateProgress();
}
DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode)
: Goal(worker, haveDerivation())
, drvPath(drvPath)
, wantedOutputs(wantedOutputs)
, buildMode(buildMode)
{
this->drv = std::make_unique<Derivation>(drv);
name = fmt(
"building of '%s' from in-memory derivation",
DerivedPath::Built { makeConstantStorePathRef(drvPath), drv.outputNames() }.to_string(worker.store));
trace("created");
mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
worker.updateProgress();
/* Prevent the .chroot directory from being /* Prevent the .chroot directory from being
garbage-collected. (See isActiveTempFile() in gc.cc.) */ garbage-collected. (See isActiveTempFile() in gc.cc.) */
worker.store.addTempRoot(this->drvPath); worker.store.addTempRoot(this->drvPath);
} }
DerivationGoal::~DerivationGoal() DerivationBuildingGoal::~DerivationBuildingGoal()
{ {
/* Careful: we should never ever throw an exception from a /* Careful: we should never ever throw an exception from a
destructor. */ destructor. */
@ -78,17 +67,17 @@ DerivationGoal::~DerivationGoal()
} }
std::string DerivationGoal::key() std::string DerivationBuildingGoal::key()
{ {
/* Ensure that derivations get built in order of their name, /* Ensure that derivations get built in order of their name,
i.e. a derivation named "aardvark" always comes before i.e. a derivation named "aardvark" always comes before
"baboon". And substitution goals always happen before "baboon". And substitution goals always happen before
derivation goals (due to "b$"). */ derivation goals (due to "b$"). */
return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); return "bd$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath);
} }
void DerivationGoal::killChild() void DerivationBuildingGoal::killChild()
{ {
#ifndef _WIN32 // TODO enable build hook on Windows #ifndef _WIN32 // TODO enable build hook on Windows
hook.reset(); hook.reset();
@ -112,7 +101,7 @@ void DerivationGoal::killChild()
} }
void DerivationGoal::timedOut(Error && ex) void DerivationBuildingGoal::timedOut(Error && ex)
{ {
killChild(); killChild();
// We're not inside a coroutine, hence we can't use co_return here. // We're not inside a coroutine, hence we can't use co_return here.
@ -120,198 +109,6 @@ void DerivationGoal::timedOut(Error && ex)
[[maybe_unused]] Done _ = done(BuildResult::TimedOut, {}, std::move(ex)); [[maybe_unused]] Done _ = done(BuildResult::TimedOut, {}, std::move(ex));
} }
void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs)
{
auto newWanted = wantedOutputs.union_(outputs);
switch (needRestart) {
case NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed:
if (!newWanted.isSubsetOf(wantedOutputs))
needRestart = NeedRestartForMoreOutputs::OutputsAddedDoNeed;
break;
case NeedRestartForMoreOutputs::OutputsAddedDoNeed:
/* No need to check whether we added more outputs, because a
restart is already queued up. */
break;
case NeedRestartForMoreOutputs::BuildInProgressWillNotNeed:
/* We are already building all outputs, so it doesn't matter if
we now want more. */
break;
};
wantedOutputs = newWanted;
}
Goal::Co DerivationGoal::loadDerivation() {
trace("local derivation");
{
/* The first thing to do is to make sure that the derivation
exists. If it doesn't, it may be created through a
substitute. */
if (buildMode != bmNormal || !worker.evalStore.isValidPath(drvPath)) {
Goals waitees{upcast_goal(worker.makePathSubstitutionGoal(drvPath))};
co_await await(std::move(waitees));
}
trace("loading derivation");
if (nrFailed != 0) {
co_return done(BuildResult::MiscFailure, {}, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)));
}
/* `drvPath' should already be a root, but let's be on the safe
side: if the user forgot to make it a root, we wouldn't want
things being garbage collected while we're busy. */
worker.evalStore.addTempRoot(drvPath);
/* Get the derivation. It is probably in the eval store, but it might be inthe main store:
- Resolved derivation are resolved against main store realisations, and so must be stored there.
- Dynamic derivations are built, and so are found in the main store.
*/
for (auto * drvStore : { &worker.evalStore, &worker.store }) {
if (drvStore->isValidPath(drvPath)) {
drv = std::make_unique<Derivation>(drvStore->readDerivation(drvPath));
break;
}
}
assert(drv);
}
co_return haveDerivation();
}
Goal::Co DerivationGoal::haveDerivation()
{
trace("have derivation");
if (auto parsedOpt = StructuredAttrs::tryParse(drv->env)) {
parsedDrv = std::make_unique<StructuredAttrs>(*parsedOpt);
}
try {
drvOptions = std::make_unique<DerivationOptions>(
DerivationOptions::fromStructuredAttrs(drv->env, parsedDrv.get()));
} catch (Error & e) {
e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath));
throw;
}
if (!drv->type().hasKnownOutputPaths())
experimentalFeatureSettings.require(Xp::CaDerivations);
for (auto & i : drv->outputsAndOptPaths(worker.store))
if (i.second.second)
worker.store.addTempRoot(*i.second.second);
{
bool impure = drv->type().isImpure();
if (impure) experimentalFeatureSettings.require(Xp::ImpureDerivations);
auto outputHashes = staticOutputHashes(worker.evalStore, *drv);
for (auto & [outputName, outputHash] : outputHashes) {
InitialOutput v{
.wanted = true, // Will be refined later
.outputHash = outputHash
};
/* TODO we might want to also allow randomizing the paths
for regular CA derivations, e.g. for sake of checking
determinism. */
if (impure) {
v.known = InitialOutputStatus {
.path = StorePath::random(outputPathName(drv->name, outputName)),
.status = PathStatus::Absent,
};
}
initialOutputs.insert({
outputName,
std::move(v),
});
}
if (impure) {
/* We don't yet have any safe way to cache an impure derivation at
this step. */
co_return gaveUpOnSubstitution();
}
}
{
/* Check what outputs paths are not already valid. */
auto [allValid, validOutputs] = checkPathValidity();
/* If they are all valid, then we're done. */
if (allValid && buildMode == bmNormal) {
co_return done(BuildResult::AlreadyValid, std::move(validOutputs));
}
}
Goals waitees;
/* We are first going to try to create the invalid output paths
through substitutes. If that doesn't work, we'll build
them. */
if (settings.useSubstitutes && drvOptions->substitutesAllowed())
for (auto & [outputName, status] : initialOutputs) {
if (!status.wanted) continue;
if (!status.known)
waitees.insert(
upcast_goal(
worker.makeDrvOutputSubstitutionGoal(
DrvOutput{status.outputHash, outputName},
buildMode == bmRepair ? Repair : NoRepair
)
)
);
else {
auto * cap = getDerivationCA(*drv);
waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(
status.known->path,
buildMode == bmRepair ? Repair : NoRepair,
cap ? std::optional { *cap } : std::nullopt)));
}
}
co_await await(std::move(waitees));
trace("all outputs substituted (maybe)");
assert(!drv->type().isImpure());
if (nrFailed > 0 && nrFailed > nrNoSubstituters && !settings.tryFallback) {
co_return done(BuildResult::TransientFailure, {},
Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
worker.store.printStorePath(drvPath)));
}
nrFailed = nrNoSubstituters = 0;
if (needRestart == NeedRestartForMoreOutputs::OutputsAddedDoNeed) {
needRestart = NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed;
co_return haveDerivation();
}
auto [allValid, validOutputs] = checkPathValidity();
if (buildMode == bmNormal && allValid) {
co_return done(BuildResult::Substituted, std::move(validOutputs));
}
if (buildMode == bmRepair && allValid) {
co_return repairClosure();
}
if (buildMode == bmCheck && !allValid)
throw Error("some outputs of '%s' are not valid, so checking is not possible",
worker.store.printStorePath(drvPath));
/* Nothing to wait for; tail call */
co_return gaveUpOnSubstitution();
}
/** /**
* Used for `inputGoals` local variable below * Used for `inputGoals` local variable below
@ -343,12 +140,8 @@ std::string showKnownOutputs(Store & store, const Derivation & drv)
/* At least one of the output paths could not be /* At least one of the output paths could not be
produced using a substitute. So we have to build instead. */ produced using a substitute. So we have to build instead. */
Goal::Co DerivationGoal::gaveUpOnSubstitution() Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution()
{ {
/* At this point we are building all outputs, so if more are wanted there
is no need to restart. */
needRestart = NeedRestartForMoreOutputs::BuildInProgressWillNotNeed;
Goals waitees; Goals waitees;
std::map<ref<const SingleDerivedPath>, GoalPtr, value_comparison> inputGoals; std::map<ref<const SingleDerivedPath>, GoalPtr, value_comparison> inputGoals;
@ -501,8 +294,9 @@ Goal::Co DerivationGoal::gaveUpOnSubstitution()
worker.store.printStorePath(pathResolved), worker.store.printStorePath(pathResolved),
}); });
// FIXME wanted outputs
auto resolvedDrvGoal = worker.makeDerivationGoal( auto resolvedDrvGoal = worker.makeDerivationGoal(
pathResolved, wantedOutputs, buildMode); pathResolved, OutputsSpec::All{}, buildMode);
{ {
Goals waitees{resolvedDrvGoal}; Goals waitees{resolvedDrvGoal};
co_await await(std::move(waitees)); co_await await(std::move(waitees));
@ -511,7 +305,10 @@ Goal::Co DerivationGoal::gaveUpOnSubstitution()
trace("resolved derivation finished"); trace("resolved derivation finished");
auto resolvedDrv = *resolvedDrvGoal->drv; auto resolvedDrv = *resolvedDrvGoal->drv;
auto & resolvedResult = resolvedDrvGoal->buildResult; auto resolvedResult = resolvedDrvGoal->getBuildResult(DerivedPath::Built{
.drvPath = makeConstantStorePathRef(pathResolved),
.outputs = OutputsSpec::All{},
});
SingleDrvOutputs builtOutputs; SingleDrvOutputs builtOutputs;
@ -615,7 +412,7 @@ Goal::Co DerivationGoal::gaveUpOnSubstitution()
co_return tryToBuild(); co_return tryToBuild();
} }
void DerivationGoal::started() void DerivationBuildingGoal::started()
{ {
auto msg = fmt( auto msg = fmt(
buildMode == bmRepair ? "repairing outputs of '%s'" : buildMode == bmRepair ? "repairing outputs of '%s'" :
@ -637,7 +434,7 @@ void DerivationGoal::started()
worker.updateProgress(); worker.updateProgress();
} }
Goal::Co DerivationGoal::tryToBuild() Goal::Co DerivationBuildingGoal::tryToBuild()
{ {
trace("trying to build"); trace("trying to build");
@ -773,15 +570,15 @@ Goal::Co DerivationGoal::tryToBuild()
* Local implementation of these virtual methods, consider * Local implementation of these virtual methods, consider
* this just a record of lambdas. * this just a record of lambdas.
*/ */
struct DerivationGoalCallbacks : DerivationBuilderCallbacks struct DerivationBuildingGoalCallbacks : DerivationBuilderCallbacks
{ {
DerivationGoal & goal; DerivationBuildingGoal & goal;
DerivationGoalCallbacks(DerivationGoal & goal, std::unique_ptr<DerivationBuilder> & builder) DerivationBuildingGoalCallbacks(DerivationBuildingGoal & goal, std::unique_ptr<DerivationBuilder> & builder)
: goal{goal} : goal{goal}
{} {}
~DerivationGoalCallbacks() override = default; ~DerivationBuildingGoalCallbacks() override = default;
void childStarted(Descriptor builderOut) override void childStarted(Descriptor builderOut) override
{ {
@ -826,7 +623,7 @@ Goal::Co DerivationGoal::tryToBuild()
already be created, so we don't need to create it again. */ already be created, so we don't need to create it again. */
builder = makeDerivationBuilder( builder = makeDerivationBuilder(
worker.store, worker.store,
std::make_unique<DerivationGoalCallbacks>(*this, builder), std::make_unique<DerivationBuildingGoalCallbacks>(*this, builder),
DerivationBuilderParams { DerivationBuilderParams {
drvPath, drvPath,
buildMode, buildMode,
@ -889,78 +686,6 @@ Goal::Co DerivationGoal::tryToBuild()
} }
Goal::Co DerivationGoal::repairClosure()
{
assert(!drv->type().isImpure());
/* If we're repairing, we now know that our own outputs are valid.
Now check whether the other paths in the outputs closure are
good. If not, then start derivation goals for the derivations
that produced those outputs. */
/* Get the output closure. */
auto outputs = queryDerivationOutputMap();
StorePathSet outputClosure;
for (auto & i : outputs) {
if (!wantedOutputs.contains(i.first)) continue;
worker.store.computeFSClosure(i.second, outputClosure);
}
/* Filter out our own outputs (which we have already checked). */
for (auto & i : outputs)
outputClosure.erase(i.second);
/* Get all dependencies of this derivation so that we know which
derivation is responsible for which path in the output
closure. */
StorePathSet inputClosure;
/* If we're working from an in-memory derivation with no in-store
`*.drv` file, we cannot do this part. */
if (worker.store.isValidPath(drvPath))
worker.store.computeFSClosure(drvPath, inputClosure);
std::map<StorePath, StorePath> outputsToDrv;
for (auto & i : inputClosure)
if (i.isDerivation()) {
auto depOutputs = worker.store.queryPartialDerivationOutputMap(i, &worker.evalStore);
for (auto & j : depOutputs)
if (j.second)
outputsToDrv.insert_or_assign(*j.second, i);
}
Goals waitees;
/* Check each path (slow!). */
for (auto & i : outputClosure) {
if (worker.pathContentsGood(i)) continue;
printError(
"found corrupted or missing path '%s' in the output closure of '%s'",
worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
auto drvPath2 = outputsToDrv.find(i);
if (drvPath2 == outputsToDrv.end())
waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(i, Repair)));
else
waitees.insert(worker.makeGoal(
DerivedPath::Built {
.drvPath = makeConstantStorePathRef(drvPath2->second),
.outputs = OutputsSpec::All { },
},
bmRepair));
}
co_await await(std::move(waitees));
if (!waitees.empty()) {
trace("closure repaired");
if (nrFailed > 0)
throw Error("some paths in the output closure of derivation '%s' could not be repaired",
worker.store.printStorePath(drvPath));
}
co_return done(BuildResult::AlreadyValid, assertPathValidity());
}
void runPostBuildHook( void runPostBuildHook(
Store & store, Store & store,
Logger & logger, Logger & logger,
@ -1020,7 +745,7 @@ void runPostBuildHook(
} }
void DerivationGoal::appendLogTailErrorMsg(std::string & msg) void DerivationBuildingGoal::appendLogTailErrorMsg(std::string & msg)
{ {
if (!logger->isVerbose() && !logTail.empty()) { if (!logger->isVerbose() && !logTail.empty()) {
msg += fmt("\nLast %d log lines:\n", logTail.size()); msg += fmt("\nLast %d log lines:\n", logTail.size());
@ -1042,7 +767,7 @@ void DerivationGoal::appendLogTailErrorMsg(std::string & msg)
} }
Goal::Co DerivationGoal::hookDone() Goal::Co DerivationBuildingGoal::hookDone()
{ {
#ifndef _WIN32 #ifndef _WIN32
assert(hook); assert(hook);
@ -1129,7 +854,7 @@ Goal::Co DerivationGoal::hookDone()
co_return done(BuildResult::Built, std::move(builtOutputs)); co_return done(BuildResult::Built, std::move(builtOutputs));
} }
HookReply DerivationGoal::tryBuildHook() HookReply DerivationBuildingGoal::tryBuildHook()
{ {
#ifdef _WIN32 // TODO enable build hook on Windows #ifdef _WIN32 // TODO enable build hook on Windows
return rpDecline; return rpDecline;
@ -1244,7 +969,7 @@ HookReply DerivationGoal::tryBuildHook()
} }
Path DerivationGoal::openLogFile() Path DerivationBuildingGoal::openLogFile()
{ {
logSize = 0; logSize = 0;
@ -1282,7 +1007,7 @@ Path DerivationGoal::openLogFile()
} }
void DerivationGoal::closeLogFile() void DerivationBuildingGoal::closeLogFile()
{ {
auto logSink2 = std::dynamic_pointer_cast<CompressionSink>(logSink); auto logSink2 = std::dynamic_pointer_cast<CompressionSink>(logSink);
if (logSink2) logSink2->finish(); if (logSink2) logSink2->finish();
@ -1292,7 +1017,7 @@ void DerivationGoal::closeLogFile()
} }
bool DerivationGoal::isReadDesc(Descriptor fd) bool DerivationBuildingGoal::isReadDesc(Descriptor fd)
{ {
#ifdef _WIN32 // TODO enable build hook on Windows #ifdef _WIN32 // TODO enable build hook on Windows
return false; return false;
@ -1304,7 +1029,7 @@ bool DerivationGoal::isReadDesc(Descriptor fd)
#endif #endif
} }
void DerivationGoal::handleChildOutput(Descriptor fd, std::string_view data) void DerivationBuildingGoal::handleChildOutput(Descriptor fd, std::string_view data)
{ {
// local & `ssh://`-builds are dealt with here. // local & `ssh://`-builds are dealt with here.
auto isWrittenToLog = isReadDesc(fd); auto isWrittenToLog = isReadDesc(fd);
@ -1375,14 +1100,14 @@ void DerivationGoal::handleChildOutput(Descriptor fd, std::string_view data)
} }
void DerivationGoal::handleEOF(Descriptor fd) void DerivationBuildingGoal::handleEOF(Descriptor fd)
{ {
if (!currentLogLine.empty()) flushLine(); if (!currentLogLine.empty()) flushLine();
worker.wakeUp(shared_from_this()); worker.wakeUp(shared_from_this());
} }
void DerivationGoal::flushLine() void DerivationBuildingGoal::flushLine()
{ {
if (handleJSONLogMessage(currentLogLine, *act, builderActivities, "the derivation builder", false)) if (handleJSONLogMessage(currentLogLine, *act, builderActivities, "the derivation builder", false))
; ;
@ -1399,7 +1124,7 @@ void DerivationGoal::flushLine()
} }
std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap() std::map<std::string, std::optional<StorePath>> DerivationBuildingGoal::queryPartialDerivationOutputMap()
{ {
assert(!drv->type().isImpure()); assert(!drv->type().isImpure());
@ -1415,35 +1140,11 @@ std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDeri
return res; return res;
} }
OutputPathMap DerivationGoal::queryDerivationOutputMap() std::pair<bool, SingleDrvOutputs> DerivationBuildingGoal::checkPathValidity()
{
assert(!drv->type().isImpure());
for (auto * drvStore : { &worker.evalStore, &worker.store })
if (drvStore->isValidPath(drvPath))
return worker.store.queryDerivationOutputMap(drvPath, drvStore);
// See comment in `DerivationGoal::queryPartialDerivationOutputMap`.
OutputPathMap res;
for (auto & [name, output] : drv->outputsAndOptPaths(worker.store))
res.insert_or_assign(name, *output.second);
return res;
}
std::pair<bool, SingleDrvOutputs> DerivationGoal::checkPathValidity()
{ {
if (drv->type().isImpure()) return { false, {} }; if (drv->type().isImpure()) return { false, {} };
bool checkHash = buildMode == bmRepair; bool checkHash = buildMode == bmRepair;
auto wantedOutputsLeft = std::visit(overloaded {
[&](const OutputsSpec::All &) {
return StringSet {};
},
[&](const OutputsSpec::Names & names) {
return static_cast<StringSet>(names);
},
}, wantedOutputs.raw);
SingleDrvOutputs validOutputs; SingleDrvOutputs validOutputs;
for (auto & i : queryPartialDerivationOutputMap()) { for (auto & i : queryPartialDerivationOutputMap()) {
@ -1452,9 +1153,7 @@ std::pair<bool, SingleDrvOutputs> DerivationGoal::checkPathValidity()
// this is an invalid output, gets catched with (!wantedOutputsLeft.empty()) // this is an invalid output, gets catched with (!wantedOutputsLeft.empty())
continue; continue;
auto & info = *initialOutput; auto & info = *initialOutput;
info.wanted = wantedOutputs.contains(i.first); info.wanted = true;
if (info.wanted)
wantedOutputsLeft.erase(i.first);
if (i.second) { if (i.second) {
auto outputPath = *i.second; auto outputPath = *i.second;
info.known = { info.known = {
@ -1490,14 +1189,6 @@ std::pair<bool, SingleDrvOutputs> DerivationGoal::checkPathValidity()
validOutputs.emplace(i.first, Realisation { drvOutput, info.known->path }); validOutputs.emplace(i.first, Realisation { drvOutput, info.known->path });
} }
// If we requested all the outputs, we are always fine.
// If we requested specific elements, the loop above removes all the valid
// ones, so any that are left must be invalid.
if (!wantedOutputsLeft.empty())
throw Error("derivation '%s' does not have wanted outputs %s",
worker.store.printStorePath(drvPath),
concatStringsSep(", ", quoteStrings(wantedOutputsLeft)));
bool allValid = true; bool allValid = true;
for (auto & [_, status] : initialOutputs) { for (auto & [_, status] : initialOutputs) {
if (!status.wanted) continue; if (!status.wanted) continue;
@ -1511,7 +1202,7 @@ std::pair<bool, SingleDrvOutputs> DerivationGoal::checkPathValidity()
} }
SingleDrvOutputs DerivationGoal::assertPathValidity() SingleDrvOutputs DerivationBuildingGoal::assertPathValidity()
{ {
auto [allValid, validOutputs] = checkPathValidity(); auto [allValid, validOutputs] = checkPathValidity();
if (!allValid) if (!allValid)
@ -1520,7 +1211,7 @@ SingleDrvOutputs DerivationGoal::assertPathValidity()
} }
Goal::Done DerivationGoal::done( Goal::Done DerivationBuildingGoal::done(
BuildResult::Status status, BuildResult::Status status,
SingleDrvOutputs builtOutputs, SingleDrvOutputs builtOutputs,
std::optional<Error> ex) std::optional<Error> ex)
@ -1534,13 +1225,10 @@ Goal::Done DerivationGoal::done(
if (buildResult.status == BuildResult::PermanentFailure) if (buildResult.status == BuildResult::PermanentFailure)
worker.permanentFailure = true; worker.permanentFailure = true;
mcExpectedBuilds.reset();
mcRunningBuilds.reset(); mcRunningBuilds.reset();
if (buildResult.success()) { if (buildResult.success()) {
auto wantedBuiltOutputs = filterDrvOutputs(wantedOutputs, std::move(builtOutputs)); buildResult.builtOutputs = std::move(builtOutputs);
assert(!wantedBuiltOutputs.empty());
buildResult.builtOutputs = std::move(wantedBuiltOutputs);
if (status == BuildResult::Built) if (status == BuildResult::Built)
worker.doneBuilds++; worker.doneBuilds++;
} else { } else {

1094
src/libstore/build/derivation-goal.cc
View file

File diff suppressed because it is too large Load diff

2
src/libstore/build/goal.cc
View file

@ -155,7 +155,7 @@ Goal::Done Goal::amDone(ExitCode result, std::optional<Error> ex)
exitCode = result; exitCode = result;
if (ex) { if (ex) {
if (!waiters.empty()) if (!preserveException && !waiters.empty())
logError(ex->info()); logError(ex->info());
else else
this->ex = std::move(*ex); this->ex = std::move(*ex);

26
src/libstore/build/worker.cc
View file

@ -4,6 +4,7 @@
#include "nix/store/build/substitution-goal.hh" #include "nix/store/build/substitution-goal.hh"
#include "nix/store/build/drv-output-substitution-goal.hh" #include "nix/store/build/drv-output-substitution-goal.hh"
#include "nix/store/build/derivation-goal.hh" #include "nix/store/build/derivation-goal.hh"
#include "nix/store/build/derivation-building-goal.hh"
#ifndef _WIN32 // TODO Enable building on Windows #ifndef _WIN32 // TODO Enable building on Windows
# include "nix/store/build/hook-instance.hh" # include "nix/store/build/hook-instance.hh"
#endif #endif
@ -87,6 +88,20 @@ std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath
} }
std::shared_ptr<DerivationBuildingGoal> Worker::makeDerivationBuildingGoal(const StorePath & drvPath,
const Derivation & drv, BuildMode buildMode)
{
std::weak_ptr<DerivationBuildingGoal> & goal_weak = derivationBuildingGoals[drvPath];
auto goal = goal_weak.lock(); // FIXME
if (!goal) {
goal = std::make_shared<DerivationBuildingGoal>(drvPath, drv, *this, buildMode);
goal_weak = goal;
wakeUp(goal);
}
return goal;
}
std::shared_ptr<PathSubstitutionGoal> Worker::makePathSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca) std::shared_ptr<PathSubstitutionGoal> Worker::makePathSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
{ {
return initGoalIfNeeded(substitutionGoals[path], path, *this, repair, ca); return initGoalIfNeeded(substitutionGoals[path], path, *this, repair, ca);
@ -134,8 +149,9 @@ void Worker::removeGoal(GoalPtr goal)
{ {
if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal)) if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
nix::removeGoal(drvGoal, derivationGoals); nix::removeGoal(drvGoal, derivationGoals);
else else if (auto drvBuildingGoal = std::dynamic_pointer_cast<DerivationBuildingGoal>(goal))
if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal)) nix::removeGoal(drvBuildingGoal, derivationBuildingGoals);
else if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal))
nix::removeGoal(subGoal, substitutionGoals); nix::removeGoal(subGoal, substitutionGoals);
else if (auto subGoal = std::dynamic_pointer_cast<DrvOutputSubstitutionGoal>(goal)) else if (auto subGoal = std::dynamic_pointer_cast<DrvOutputSubstitutionGoal>(goal))
nix::removeGoal(subGoal, drvOutputSubstitutionGoals); nix::removeGoal(subGoal, drvOutputSubstitutionGoals);
@ -198,6 +214,9 @@ void Worker::childStarted(GoalPtr goal, const std::set<MuxablePipePollState::Com
case JobCategory::Build: case JobCategory::Build:
nrLocalBuilds++; nrLocalBuilds++;
break; break;
case JobCategory::Administration:
/* Intentionally not limited, see docs */
break;
default: default:
unreachable(); unreachable();
} }
@ -221,6 +240,9 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers)
assert(nrLocalBuilds > 0); assert(nrLocalBuilds > 0);
nrLocalBuilds--; nrLocalBuilds--;
break; break;
case JobCategory::Administration:
/* Intentionally not limited, see docs */
break;
default: default:
unreachable(); unreachable();
} }

55
src/libstore/include/nix/store/build/derivation-building-goal.hh
View file

@ -31,43 +31,11 @@ void runPostBuildHook(
/** /**
* A goal for building some or all of the outputs of a derivation. * A goal for building some or all of the outputs of a derivation.
*/ */
struct DerivationGoal : public Goal struct DerivationBuildingGoal : public Goal
{ {
/** The path of the derivation. */ /** The path of the derivation. */
StorePath drvPath; StorePath drvPath;
/**
* The specific outputs that we need to build.
*/
OutputsSpec wantedOutputs;
/**
* See `needRestart`; just for that field.
*/
enum struct NeedRestartForMoreOutputs {
/**
* The goal state machine is progressing based on the current value of
* `wantedOutputs. No actions are needed.
*/
OutputsUnmodifedDontNeed,
/**
* `wantedOutputs` has been extended, but the state machine is
* proceeding according to its old value, so we need to restart.
*/
OutputsAddedDoNeed,
/**
* The goal state machine has progressed to the point of doing a build,
* in which case all outputs will be produced, so extensions to
* `wantedOutputs` no longer require a restart.
*/
BuildInProgressWillNotNeed,
};
/**
* Whether additional wanted outputs have been added.
*/
NeedRestartForMoreOutputs needRestart = NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed;
/** /**
* The derivation stored at drvPath. * The derivation stored at drvPath.
*/ */
@ -125,7 +93,7 @@ struct DerivationGoal : public Goal
BuildMode buildMode; BuildMode buildMode;
std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds; std::unique_ptr<MaintainCount<uint64_t>> mcRunningBuilds;
std::unique_ptr<Activity> act; std::unique_ptr<Activity> act;
@ -141,28 +109,18 @@ struct DerivationGoal : public Goal
*/ */
std::string machineName; std::string machineName;
DerivationGoal(const StorePath & drvPath, DerivationBuildingGoal(const StorePath & drvPath, const Derivation & drv,
const OutputsSpec & wantedOutputs, Worker & worker, Worker & worker,
BuildMode buildMode = bmNormal); BuildMode buildMode = bmNormal);
DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, ~DerivationBuildingGoal();
const OutputsSpec & wantedOutputs, Worker & worker,
BuildMode buildMode = bmNormal);
~DerivationGoal();
void timedOut(Error && ex) override; void timedOut(Error && ex) override;
std::string key() override; std::string key() override;
/**
* Add wanted outputs to an already existing derivation goal.
*/
void addWantedOutputs(const OutputsSpec & outputs);
/** /**
* The states. * The states.
*/ */
Co loadDerivation();
Co haveDerivation();
Co gaveUpOnSubstitution(); Co gaveUpOnSubstitution();
Co tryToBuild(); Co tryToBuild();
Co hookDone(); Co hookDone();
@ -197,7 +155,6 @@ struct DerivationGoal : public Goal
* there also is no DB entry. * there also is no DB entry.
*/ */
std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(); std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
OutputPathMap queryDerivationOutputMap();
/** /**
* Update 'initialOutputs' to determine the current status of the * Update 'initialOutputs' to determine the current status of the
@ -218,8 +175,6 @@ struct DerivationGoal : public Goal
*/ */
void killChild(); void killChild();
Co repairClosure();
void started(); void started();
Done done( Done done(

111
src/libstore/include/nix/store/build/derivation-goal.hh
View file

@ -14,13 +14,6 @@ namespace nix {
using std::map; using std::map;
#ifndef _WIN32 // TODO enable build hook on Windows
struct HookInstance;
struct DerivationBuilder;
#endif
typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
/** Used internally */ /** Used internally */
void runPostBuildHook( void runPostBuildHook(
Store & store, Store & store,
@ -73,73 +66,15 @@ struct DerivationGoal : public Goal
*/ */
std::unique_ptr<Derivation> drv; std::unique_ptr<Derivation> drv;
std::unique_ptr<StructuredAttrs> parsedDrv;
std::unique_ptr<DerivationOptions> drvOptions;
/** /**
* The remainder is state held during the build. * The remainder is state held during the build.
*/ */
/**
* Locks on (fixed) output paths.
*/
PathLocks outputLocks;
/**
* All input paths (that is, the union of FS closures of the
* immediate input paths).
*/
StorePathSet inputPaths;
std::map<std::string, InitialOutput> initialOutputs; std::map<std::string, InitialOutput> initialOutputs;
/**
* File descriptor for the log file.
*/
AutoCloseFD fdLogFile;
std::shared_ptr<BufferedSink> logFileSink, logSink;
/**
* Number of bytes received from the builder's stdout/stderr.
*/
unsigned long logSize;
/**
* The most recent log lines.
*/
std::list<std::string> logTail;
std::string currentLogLine;
size_t currentLogLinePos = 0; // to handle carriage return
std::string currentHookLine;
#ifndef _WIN32 // TODO enable build hook on Windows
/**
* The build hook.
*/
std::unique_ptr<HookInstance> hook;
std::unique_ptr<DerivationBuilder> builder;
#endif
BuildMode buildMode; BuildMode buildMode;
std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds; std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds;
std::unique_ptr<Activity> act;
/**
* Activity that denotes waiting for a lock.
*/
std::unique_ptr<Activity> actLock;
std::map<ActivityId, Activity> builderActivities;
/**
* The remote machine on which we're building.
*/
std::string machineName;
DerivationGoal(const StorePath & drvPath, DerivationGoal(const StorePath & drvPath,
const OutputsSpec & wantedOutputs, Worker & worker, const OutputsSpec & wantedOutputs, Worker & worker,
@ -147,9 +82,9 @@ struct DerivationGoal : public Goal
DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
const OutputsSpec & wantedOutputs, Worker & worker, const OutputsSpec & wantedOutputs, Worker & worker,
BuildMode buildMode = bmNormal); BuildMode buildMode = bmNormal);
~DerivationGoal(); ~DerivationGoal() = default;
void timedOut(Error && ex) override; void timedOut(Error && ex) override { unreachable(); };
std::string key() override; std::string key() override;
@ -163,33 +98,6 @@ struct DerivationGoal : public Goal
*/ */
Co loadDerivation(); Co loadDerivation();
Co haveDerivation(); Co haveDerivation();
Co gaveUpOnSubstitution();
Co tryToBuild();
Co hookDone();
/**
* Is the build hook willing to perform the build?
*/
HookReply tryBuildHook();
/**
* Open a log file and a pipe to it.
*/
Path openLogFile();
/**
* Close the log file.
*/
void closeLogFile();
bool isReadDesc(Descriptor fd);
/**
* Callback used by the worker to write to the log.
*/
void handleChildOutput(Descriptor fd, std::string_view data) override;
void handleEOF(Descriptor fd) override;
void flushLine();
/** /**
* Wrappers around the corresponding Store methods that first consult the * Wrappers around the corresponding Store methods that first consult the
@ -213,26 +121,15 @@ struct DerivationGoal : public Goal
*/ */
SingleDrvOutputs assertPathValidity(); SingleDrvOutputs assertPathValidity();
/**
* Forcibly kill the child process, if any.
*/
void killChild();
Co repairClosure(); Co repairClosure();
void started();
Done done( Done done(
BuildResult::Status status, BuildResult::Status status,
SingleDrvOutputs builtOutputs = {}, SingleDrvOutputs builtOutputs = {},
std::optional<Error> ex = {}); std::optional<Error> ex = {});
void appendLogTailErrorMsg(std::string & msg);
StorePathSet exportReferences(const StorePathSet & storePaths);
JobCategory jobCategory() const override { JobCategory jobCategory() const override {
return JobCategory::Build; return JobCategory::Administration;
}; };
}; };

21
src/libstore/include/nix/store/build/goal.hh
View file

@ -50,6 +50,16 @@ enum struct JobCategory {
* A substitution an arbitrary store object; it will use network resources. * A substitution an arbitrary store object; it will use network resources.
*/ */
Substitution, Substitution,
/**
* A goal that does no "real" work by itself, and just exists to depend on
* other goals which *do* do real work. These goals therefore are not
* limited.
*
* These goals cannot infinitely create themselves, so there is no risk of
* a "fork bomb" type situation (which would be a problem even though the
* goal do no real work) either.
*/
Administration,
}; };
struct Goal : public std::enable_shared_from_this<Goal> struct Goal : public std::enable_shared_from_this<Goal>
@ -360,6 +370,17 @@ public:
*/ */
BuildResult getBuildResult(const DerivedPath &) const; BuildResult getBuildResult(const DerivedPath &) const;
/**
* Hack to say that this goal should not log `ex`, but instead keep
* it around. Set by a waitee which sees itself as the designated
* continuation of this goal, responsible for reporting its
* successes or failures.
*
* @todo this is yet another not-nice hack in the goal system that
* we ought to get rid of. See #11927
*/
bool preserveException = false;
/** /**
* Exception containing an error message, if any. * Exception containing an error message, if any.
*/ */

9
src/libstore/include/nix/store/build/worker.hh
View file

@ -14,6 +14,7 @@ namespace nix {
/* Forward definition. */ /* Forward definition. */
struct DerivationGoal; struct DerivationGoal;
struct DerivationBuildingGoal;
struct PathSubstitutionGoal; struct PathSubstitutionGoal;
class DrvOutputSubstitutionGoal; class DrvOutputSubstitutionGoal;
@ -104,6 +105,7 @@ private:
* same derivation / path. * same derivation / path.
*/ */
std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals; std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals;
std::map<StorePath, std::weak_ptr<DerivationBuildingGoal>> derivationBuildingGoals;
std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals; std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals;
std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals; std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;
@ -210,6 +212,13 @@ public:
const StorePath & drvPath, const BasicDerivation & drv, const StorePath & drvPath, const BasicDerivation & drv,
const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal); const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal);
/**
* @ref DerivationBuildingGoal "derivation goal"
*/
std::shared_ptr<DerivationBuildingGoal> makeDerivationBuildingGoal(
const StorePath & drvPath, const Derivation & drv,
BuildMode buildMode = bmNormal);
/** /**
* @ref PathSubstitutionGoal "substitution goal" * @ref PathSubstitutionGoal "substitution goal"
*/ */

1
src/libstore/include/nix/store/meson.build
View file

@ -13,6 +13,7 @@ headers = [config_pub_h] + files(
'binary-cache-store.hh', 'binary-cache-store.hh',
'build-result.hh', 'build-result.hh',
'build/derivation-goal.hh', 'build/derivation-goal.hh',
'build/derivation-building-goal.hh',
'build/derivation-building-misc.hh', 'build/derivation-building-misc.hh',
'build/drv-output-substitution-goal.hh', 'build/drv-output-substitution-goal.hh',
'build/goal.hh', 'build/goal.hh',

1
src/libstore/meson.build
View file

@ -254,6 +254,7 @@ sources = files(
'binary-cache-store.cc', 'binary-cache-store.cc',
'build-result.cc', 'build-result.cc',
'build/derivation-goal.cc', 'build/derivation-goal.cc',
'build/derivation-building-goal.cc',
'build/drv-output-substitution-goal.cc', 'build/drv-output-substitution-goal.cc',
'build/entry-points.cc', 'build/entry-points.cc',
'build/goal.cc', 'build/goal.cc',