wroclaw-git-backups/nix
1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-06-25 10:41:16 +02:00
Code Activity

Merge pull request #12567 from obsidiansystems/slightly-rework-drv-resolution

Browse source 
Rework derivation input resolution
This commit is contained in:
Jörg Thalheim 2025-03-12 23:15:45 +01:00 • committed by GitHub
commit cba1a2155a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 103 additions and 142 deletions
Download patch file Download diff file Expand all files Collapse all files

125
src/libstore/build/derivation-goal.cc
View file

@ -322,6 +322,18 @@ Goal::Co DerivationGoal::haveDerivation()
}
/**
* Used for `inputGoals` local variable below
*/
struct value_comparison
{
template <typename T>
bool operator()(const ref<T> & lhs, const ref<T> & rhs) const {
return *lhs < *rhs;
}
};
/* At least one of the output paths could not be
produced using a substitute. So we have to build instead. */
Goal::Co DerivationGoal::gaveUpOnSubstitution()
@ -330,19 +342,22 @@ Goal::Co DerivationGoal::gaveUpOnSubstitution()
is no need to restart. */
needRestart = NeedRestartForMoreOutputs::BuildInProgressWillNotNeed;
/* The inputs must be built before we can build this goal. */
inputDrvOutputs.clear();
if (useDerivation) {
std::function<void(ref<SingleDerivedPath>, const DerivedPathMap<StringSet>::ChildNode &)> addWaiteeDerivedPath;
std::map<ref<const SingleDerivedPath>, GoalPtr, value_comparison> inputGoals;
addWaiteeDerivedPath = [&](ref<SingleDerivedPath> inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
if (!inputNode.value.empty())
addWaitee(worker.makeGoal(
if (useDerivation) {
std::function<void(ref<const SingleDerivedPath>, const DerivedPathMap<StringSet>::ChildNode &)> addWaiteeDerivedPath;
addWaiteeDerivedPath = [&](ref<const SingleDerivedPath> inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
if (!inputNode.value.empty()) {
auto g = worker.makeGoal(
DerivedPath::Built {
.drvPath = inputDrv,
.outputs = inputNode.value,
},
buildMode == bmRepair ? bmRepair : bmNormal));
buildMode == bmRepair ? bmRepair : bmNormal);
inputGoals.insert_or_assign(inputDrv, g);
addWaitee(std::move(g));
}
for (const auto & [outputName, childNode] : inputNode.childMap)
addWaiteeDerivedPath(
make_ref<SingleDerivedPath>(SingleDerivedPath::Built { inputDrv, outputName }),
@ -430,7 +445,12 @@ Goal::Co DerivationGoal::gaveUpOnSubstitution()
[&](const DerivationType::Impure &) {
return true;
}
}, drvType.raw);
}, drvType.raw)
/* no inputs are outputs of dynamic derivations */
|| std::ranges::any_of(
fullDrv.inputDrvs.map.begin(),
fullDrv.inputDrvs.map.end(),
[](auto & pair) { return !pair.second.childMap.empty(); });
if (resolveDrv && !fullDrv.inputDrvs.map.empty()) {
experimentalFeatureSettings.require(Xp::CaDerivations);
@ -438,7 +458,19 @@ Goal::Co DerivationGoal::gaveUpOnSubstitution()
/* We are be able to resolve this derivation based on the
now-known results of dependencies. If so, we become a
stub goal aliasing that resolved derivation goal. */
std::optional attempt = fullDrv.tryResolve(worker.store, inputDrvOutputs);
std::optional attempt = fullDrv.tryResolve(worker.store,
[&](ref<const SingleDerivedPath> drvPath, const std::string & outputName) -> std::optional<StorePath> {
auto mEntry = get(inputGoals, drvPath);
if (!mEntry) return std::nullopt;
auto buildResult = (*mEntry)->getBuildResult(DerivedPath::Built{drvPath, OutputsSpec::Names{outputName}});
if (!buildResult.success()) return std::nullopt;
auto i = get(buildResult.builtOutputs, outputName);
if (!i) return std::nullopt;
return i->outPath;
});
if (!attempt) {
/* TODO (impure derivations-induced tech debt) (see below):
The above attempt should have found it, but because we manage
@ -469,28 +501,14 @@ Goal::Co DerivationGoal::gaveUpOnSubstitution()
co_return resolvedFinished();
}
std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumInputPaths;
/* If we get this far, we know no dynamic drvs inputs */
accumInputPaths = [&](const StorePath & depDrvPath, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
/* Add the relevant output closures of the input derivation
`i' as input paths. Only add the closures of output paths
that are specified as inputs. */
auto getOutput = [&](const std::string & outputName) {
/* TODO (impure derivations-induced tech debt):
Tracking input derivation outputs statefully through the
goals is error prone and has led to bugs.
For a robust nix, we need to move towards the `else` branch,
which does not rely on goal state to match up with the
reality of the store, which is our real source of truth.
However, the impure derivations feature still relies on this
fragile way of doing things, because its builds do not have
a representation in the store, which is a usability problem
in itself. When implementing this logic entirely with lookups
make sure that they're cached. */
if (auto outPath = get(inputDrvOutputs, { depDrvPath, outputName })) {
return *outPath;
}
else {
for (auto & [depDrvPath, depNode] : fullDrv.inputDrvs.map) {
for (auto & outputName : depNode.value) {
/* Don't need to worry about `inputGoals`, because
impure derivations are always resolved above. Can
just use DB. This case only happens in the (older)
input addressed and fixed output derivation cases. */
auto outMap = [&]{
for (auto * drvStore : { &worker.evalStore, &worker.store })
if (drvStore->isValidPath(depDrvPath))
@ -504,19 +522,10 @@ Goal::Co DerivationGoal::gaveUpOnSubstitution()
"derivation '%s' requires non-existent output '%s' from input derivation '%s'",
worker.store.printStorePath(drvPath), outputName, worker.store.printStorePath(depDrvPath));
}
return outMapPath->second;
worker.store.computeFSClosure(outMapPath->second, inputPaths);
}
}
};
for (auto & outputName : inputNode.value)
worker.store.computeFSClosure(getOutput(outputName), inputPaths);
for (auto & [outputName, childNode] : inputNode.childMap)
accumInputPaths(getOutput(outputName), childNode);
};
for (auto & [depDrvPath, depNode] : fullDrv.inputDrvs.map)
accumInputPaths(depDrvPath, depNode);
}
/* Second, the input sources. */
@ -1545,34 +1554,4 @@ Goal::Done DerivationGoal::done(
return amDone(buildResult.success() ? ecSuccess : ecFailed, std::move(ex));
}
void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result)
{
Goal::waiteeDone(waitee, result);
if (!useDerivation || !drv) return;
auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
auto * dg = dynamic_cast<DerivationGoal *>(&*waitee);
if (!dg) return;
auto * nodeP = fullDrv.inputDrvs.findSlot(DerivedPath::Opaque { .path = dg->drvPath });
if (!nodeP) return;
auto & outputs = nodeP->value;
for (auto & outputName : outputs) {
auto buildResult = dg->getBuildResult(DerivedPath::Built {
.drvPath = makeConstantStorePathRef(dg->drvPath),
.outputs = OutputsSpec::Names { outputName },
});
if (buildResult.success()) {
auto i = buildResult.builtOutputs.find(outputName);
if (i != buildResult.builtOutputs.end())
inputDrvOutputs.insert_or_assign(
{ dg->drvPath, outputName },
i->second.outPath);
}
}
}
}

9
src/libstore/build/derivation-goal.hh
View file

@ -78,13 +78,6 @@ struct DerivationGoal : public Goal
*/
OutputsSpec wantedOutputs;
/**
* Mapping from input derivations + output names to actual store
* paths. This is filled in by waiteeDone() as each dependency
* finishes, before `trace("all inputs realised")` is reached.
*/
std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
/**
* See `needRestart`; just for that field.
*/
@ -331,8 +324,6 @@ struct DerivationGoal : public Goal
SingleDrvOutputs builtOutputs = {},
std::optional<Error> ex = {});
void waiteeDone(GoalPtr waitee, ExitCode result) override;
StorePathSet exportReferences(const StorePathSet & storePaths);
JobCategory jobCategory() const override {

2
src/libstore/build/goal.hh
View file

@ -396,7 +396,7 @@ public:
void addWaitee(GoalPtr waitee);
virtual void waiteeDone(GoalPtr waitee, ExitCode result);
void waiteeDone(GoalPtr waitee, ExitCode result);
virtual void handleChildOutput(Descriptor fd, std::string_view data)
{

5
src/libstore/build/worker.cc
View file

@ -552,4 +552,9 @@ GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal)
return subGoal;
}
GoalPtr upcast_goal(std::shared_ptr<DerivationGoal> subGoal)
{
return subGoal;
}
}

60
src/libstore/derivations.cc
View file

@ -1052,49 +1052,36 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
std::optional<BasicDerivation> Derivation::tryResolve(Store & store, Store * evalStore) const
{
std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accum;
accum = [&](auto & inputDrv, auto & node) {
for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(inputDrv, evalStore)) {
if (outputPath) {
inputDrvOutputs.insert_or_assign({inputDrv, outputName}, *outputPath);
if (auto p = get(node.childMap, outputName))
accum(*outputPath, *p);
return tryResolve(
store,
[&](ref<const SingleDerivedPath> drvPath, const std::string & outputName) -> std::optional<StorePath> {
try {
return resolveDerivedPath(store, SingleDerivedPath::Built{drvPath, outputName}, evalStore);
} catch (Error &) {
return std::nullopt;
}
}
};
for (auto & [inputDrv, node] : inputDrvs.map)
accum(inputDrv, node);
return tryResolve(store, inputDrvOutputs);
});
}
static bool tryResolveInput(
Store & store, StorePathSet & inputSrcs, StringMap & inputRewrites,
const DownstreamPlaceholder * placeholderOpt,
const StorePath & inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode,
const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs)
ref<const SingleDerivedPath> drvPath, const DerivedPathMap<StringSet>::ChildNode & inputNode,
std::function<std::optional<StorePath>(ref<const SingleDerivedPath> drvPath, const std::string & outputName)> queryResolutionChain)
{
auto getOutput = [&](const std::string & outputName) {
auto * actualPathOpt = get(inputDrvOutputs, { inputDrv, outputName });
if (!actualPathOpt)
warn("output %s of input %s missing, aborting the resolving",
outputName,
store.printStorePath(inputDrv)
);
return actualPathOpt;
};
auto getPlaceholder = [&](const std::string & outputName) {
return placeholderOpt
? DownstreamPlaceholder::unknownDerivation(*placeholderOpt, outputName)
: DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName);
: [&]{
auto * p = std::get_if<SingleDerivedPath::Opaque>(&drvPath->raw());
// otherwise we should have had a placeholder to build-upon already
assert(p);
return DownstreamPlaceholder::unknownCaOutput(p->path, outputName);
}();
};
for (auto & outputName : inputNode.value) {
auto actualPathOpt = getOutput(outputName);
auto actualPathOpt = queryResolutionChain(drvPath, outputName);
if (!actualPathOpt) return false;
auto actualPath = *actualPathOpt;
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
@ -1106,13 +1093,12 @@ static bool tryResolveInput(
}
for (auto & [outputName, childNode] : inputNode.childMap) {
auto actualPathOpt = getOutput(outputName);
if (!actualPathOpt) return false;
auto actualPath = *actualPathOpt;
auto nextPlaceholder = getPlaceholder(outputName);
if (!tryResolveInput(store, inputSrcs, inputRewrites,
&nextPlaceholder, actualPath, childNode,
inputDrvOutputs))
&nextPlaceholder,
make_ref<const SingleDerivedPath>(SingleDerivedPath::Built{drvPath, outputName}),
childNode,
queryResolutionChain))
return false;
}
return true;
@ -1120,7 +1106,7 @@ static bool tryResolveInput(
std::optional<BasicDerivation> Derivation::tryResolve(
Store & store,
const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const
std::function<std::optional<StorePath>(ref<const SingleDerivedPath> drvPath, const std::string & outputName)> queryResolutionChain) const
{
BasicDerivation resolved { *this };
@ -1129,7 +1115,7 @@ std::optional<BasicDerivation> Derivation::tryResolve(
for (auto & [inputDrv, inputNode] : inputDrvs.map)
if (!tryResolveInput(store, resolved.inputSrcs, inputRewrites,
nullptr, inputDrv, inputNode, inputDrvOutputs))
nullptr, make_ref<const SingleDerivedPath>(SingleDerivedPath::Opaque{inputDrv}), inputNode, queryResolutionChain))
return std::nullopt;
rewriteDerivation(store, resolved, inputRewrites);

2
src/libstore/derivations.hh
View file

@ -369,7 +369,7 @@ struct Derivation : BasicDerivation
*/
std::optional<BasicDerivation> tryResolve(
Store & store,
const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const;
std::function<std::optional<StorePath>(ref<const SingleDerivedPath> drvPath, const std::string & outputName)> queryResolutionChain) const;
/**
* Check that the derivation is valid and does not present any

8
src/libstore/derived-path.cc
View file

@ -170,7 +170,7 @@ void drvRequireExperiment(
}
SingleDerivedPath::Built SingleDerivedPath::Built::parse(
const StoreDirConfig & store, ref<SingleDerivedPath> drv,
const StoreDirConfig & store, ref<const SingleDerivedPath> drv,
OutputNameView output,
const ExperimentalFeatureSettings & xpSettings)
{
@ -182,7 +182,7 @@ SingleDerivedPath::Built SingleDerivedPath::Built::parse(
}
DerivedPath::Built DerivedPath::Built::parse(
const StoreDirConfig & store, ref<SingleDerivedPath> drv,
const StoreDirConfig & store, ref<const SingleDerivedPath> drv,
OutputNameView outputsS,
const ExperimentalFeatureSettings & xpSettings)
{
@ -201,7 +201,7 @@ static SingleDerivedPath parseWithSingle(
return n == s.npos
? (SingleDerivedPath) SingleDerivedPath::Opaque::parse(store, s)
: (SingleDerivedPath) SingleDerivedPath::Built::parse(store,
make_ref<SingleDerivedPath>(parseWithSingle(
make_ref<const SingleDerivedPath>(parseWithSingle(
store,
s.substr(0, n),
separator,
@ -234,7 +234,7 @@ static DerivedPath parseWith(
return n == s.npos
? (DerivedPath) DerivedPath::Opaque::parse(store, s)
: (DerivedPath) DerivedPath::Built::parse(store,
make_ref<SingleDerivedPath>(parseWithSingle(
make_ref<const SingleDerivedPath>(parseWithSingle(
store,
s.substr(0, n),
separator,

8
src/libstore/derived-path.hh
View file

@ -45,7 +45,7 @@ struct SingleDerivedPath;
* path of the given output name.
*/
struct SingleDerivedPathBuilt {
ref<SingleDerivedPath> drvPath;
ref<const SingleDerivedPath> drvPath;
OutputName output;
/**
@ -74,7 +74,7 @@ struct SingleDerivedPathBuilt {
* @param xpSettings Stop-gap to avoid globals during unit tests.
*/
static SingleDerivedPathBuilt parse(
const StoreDirConfig & store, ref<SingleDerivedPath> drvPath,
const StoreDirConfig & store, ref<const SingleDerivedPath> drvPath,
OutputNameView outputs,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
nlohmann::json toJSON(Store & store) const;
@ -172,7 +172,7 @@ static inline ref<SingleDerivedPath> makeConstantStorePathRef(StorePath drvPath)
* output name.
*/
struct DerivedPathBuilt {
ref<SingleDerivedPath> drvPath;
ref<const SingleDerivedPath> drvPath;
OutputsSpec outputs;
/**
@ -201,7 +201,7 @@ struct DerivedPathBuilt {
* @param xpSettings Stop-gap to avoid globals during unit tests.
*/
static DerivedPathBuilt parse(
const StoreDirConfig & store, ref<SingleDerivedPath>,
const StoreDirConfig & store, ref<const SingleDerivedPath>,
std::string_view,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
nlohmann::json toJSON(Store & store) const;

2
src/nix/log.cc
View file

@ -35,7 +35,7 @@ struct CmdLog : InstallableCommand
// For compat with CLI today, TODO revisit
auto oneUp = std::visit(overloaded {
[&](const DerivedPath::Opaque & bo) {
return make_ref<SingleDerivedPath>(bo);
return make_ref<const SingleDerivedPath>(bo);
},
[&](const DerivedPath::Built & bfd) {
return bfd.drvPath;