wroclaw-git-backups/nix
1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-06-25 02:21:16 +02:00
Code Activity

Revert "Use the hash modulo in the derivation outputs"

Browse source 
This reverts commit bab1cda0e6.
This commit is contained in:
John Ericson 2025-02-12 23:54:24 -05:00
parent fb028ae701
commit c6fefcb2f4
48 changed files with 920 additions and 961 deletions
Download patch file Download diff file Expand all files Collapse all files

2
maintainers/flake-module.nix
View file

@ -255,8 +255,6 @@
''^src/libstore/include/nix/store/pathlocks\.hh$'' ''^src/libstore/include/nix/store/pathlocks\.hh$''
''^src/libstore/profiles\.cc$'' ''^src/libstore/profiles\.cc$''
''^src/libstore/include/nix/store/profiles\.hh$'' ''^src/libstore/include/nix/store/profiles\.hh$''
''^src/libstore/realisation\.cc$''
''^src/libstore/include/nix/store/realisation\.hh$''
''^src/libstore/remote-fs-accessor\.cc$'' ''^src/libstore/remote-fs-accessor\.cc$''
''^src/libstore/include/nix/store/remote-fs-accessor\.hh$'' ''^src/libstore/include/nix/store/remote-fs-accessor\.hh$''
''^src/libstore/include/nix/store/remote-store-connection\.hh$'' ''^src/libstore/include/nix/store/remote-store-connection\.hh$''

6
src/build-remote/build-remote.cc
View file

@ -345,13 +345,11 @@ connected:
} }
auto outputHashes = staticOutputHashes(*store, drv);
std::set<Realisation> missingRealisations; std::set<Realisation> missingRealisations;
StorePathSet missingPaths; StorePathSet missingPaths;
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv.type().hasKnownOutputPaths()) { if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv.type().hasKnownOutputPaths()) {
for (auto & outputName : wantedOutputs) { for (auto & outputName : wantedOutputs) {
auto thisOutputHash = outputHashes.at(outputName); auto thisOutputId = DrvOutput{ *drvPath, outputName };
auto thisOutputId = DrvOutput{ thisOutputHash, outputName };
if (!store->queryRealisation(thisOutputId)) { if (!store->queryRealisation(thisOutputId)) {
debug("missing output %s", outputName); debug("missing output %s", outputName);
assert(optResult); assert(optResult);
@ -359,7 +357,7 @@ connected:
auto i = result.builtOutputs.find(outputName); auto i = result.builtOutputs.find(outputName);
assert(i != result.builtOutputs.end()); assert(i != result.builtOutputs.end());
auto & newRealisation = i->second; auto & newRealisation = i->second;
missingRealisations.insert(newRealisation); missingRealisations.insert({newRealisation, thisOutputId});
missingPaths.insert(newRealisation.outPath); missingPaths.insert(newRealisation.outPath);
} }
} }

23
src/libcmd/built-path.cc
View file

@ -116,21 +116,16 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const
overloaded{ overloaded{
[&](const BuiltPath::Opaque & p) { res.insert(p.path); }, [&](const BuiltPath::Opaque & p) { res.insert(p.path); },
[&](const BuiltPath::Built & p) { [&](const BuiltPath::Built & p) {
auto drvHashes =
staticOutputHashes(store, store.readDerivation(p.drvPath->outPath()));
for (auto& [outputName, outputPath] : p.outputs) { for (auto& [outputName, outputPath] : p.outputs) {
if (experimentalFeatureSettings.isEnabled( if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
Xp::CaDerivations)) { DrvOutput key{
auto drvOutput = get(drvHashes, outputName); .drvPath = p.drvPath->outPath(),
if (!drvOutput) .outputName = outputName,
throw Error( };
"the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)", auto thisRealisation = store.queryRealisation(key);
store.printStorePath(p.drvPath->outPath()), outputName); // We’ve built it, so we must have the realisation.
auto thisRealisation = store.queryRealisation( assert(thisRealisation);
DrvOutput{*drvOutput, outputName}); res.insert(Realisation{*thisRealisation, key});
assert(thisRealisation); // We’ve built it, so we must
// have the realisation
res.insert(*thisRealisation);
} else { } else {
res.insert(outputPath); res.insert(outputPath);
} }

26
src/libexpr/primops.cc
View file

@ -1565,31 +1565,7 @@ static void derivationStrictInternal(
DerivationOutput::Deferred { }); DerivationOutput::Deferred { });
} }
auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true); resolveInputAddressed(*state.store, drv);
switch (hashModulo.kind) {
case DrvHash::Kind::Regular:
for (auto & i : outputs) {
auto h = get(hashModulo.hashes, i);
if (!h)
state.error<AssertionError>(
"derivation produced no hash for output '%s'",
i
).atPos(v).debugThrow();
auto outPath = state.store->makeOutputPath(i, *h, drvName);
drv.env[i] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign(
i,
DerivationOutput::InputAddressed {
.path = std::move(outPath),
});
}
break;
;
case DrvHash::Kind::Deferred:
for (auto & i : outputs) {
drv.outputs.insert_or_assign(i, DerivationOutput::Deferred {});
}
}
} }
/* Write the resulting term into the Nix store directory. */ /* Write the resulting term into the Nix store directory. */

45
src/libstore-tests/common-protocol.cc
View file

@ -96,51 +96,6 @@ CHARACTERIZATION_TEST(
}, },
})) }))
CHARACTERIZATION_TEST(
drvOutput,
"drv-output",
(std::tuple<DrvOutput, DrvOutput> {
{
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
.outputName = "baz",
},
DrvOutput {
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "quux",
},
}))
CHARACTERIZATION_TEST(
realisation,
"realisation",
(std::tuple<Realisation, Realisation> {
Realisation {
.id = DrvOutput {
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
.outputName = "baz",
},
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
.signatures = { "asdf", "qwer" },
},
Realisation {
.id = {
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
.outputName = "baz",
},
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
.signatures = { "asdf", "qwer" },
.dependentRealisations = {
{
DrvOutput {
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "quux",
},
StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
},
},
},
}))
CHARACTERIZATION_TEST( CHARACTERIZATION_TEST(
vector, vector,
"vector", "vector",

61
src/libstore-tests/serve-protocol.cc
View file

@ -73,51 +73,44 @@ VERSIONED_CHARACTERIZATION_TEST(
VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST(
ServeProtoTest, ServeProtoTest,
drvOutput, drvOutput,
"drv-output", "drv-output-2.8",
defaultVersion, 2 << 8 | 8,
(std::tuple<DrvOutput, DrvOutput> { (std::tuple<DrvOutput, DrvOutput> {
{ {
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), .drvPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" },
.outputName = "baz", .outputName = "baz",
}, },
DrvOutput { DrvOutput {
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), .drvPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" },
.outputName = "quux", .outputName = "quux",
}, },
})) }))
#endif #endif
VERSIONED_CHARACTERIZATION_TEST(
ServeProtoTest,
unkeyedRealisation,
"unkeyed-realisation-2.8",
2 << 8 | 8,
(UnkeyedRealisation {
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
.signatures = { "asdf", "qwer" },
}))
VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST(
ServeProtoTest, ServeProtoTest,
realisation, realisation,
"realisation", "realisation-2.8",
defaultVersion, 2 << 8 | 8,
(std::tuple<Realisation, Realisation> { (Realisation {
Realisation { UnkeyedRealisation {
.id = DrvOutput {
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
.outputName = "baz",
},
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
.signatures = { "asdf", "qwer" }, .signatures = { "asdf", "qwer" },
}, },
Realisation { DrvOutput {
.id = { .drvPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" },
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), .outputName = "baz",
.outputName = "baz",
},
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
.signatures = { "asdf", "qwer" },
.dependentRealisations = {
{
DrvOutput {
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "quux",
},
StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
},
},
}, },
})) }))
@ -176,8 +169,8 @@ VERSIONED_CHARACTERIZATION_TEST(
VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST(
ServeProtoTest, ServeProtoTest,
buildResult_2_6, buildResult_2_6,
"build-result-2.6", "build-result-2.8",
2 << 8 | 6, 2 << 8 | 8,
({ ({
using namespace std::literals::chrono_literals; using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t { std::tuple<BuildResult, BuildResult, BuildResult> t {
@ -200,20 +193,12 @@ VERSIONED_CHARACTERIZATION_TEST(
{ {
"foo", "foo",
{ {
.id = DrvOutput {
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "foo",
},
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
}, },
}, },
{ {
"bar", "bar",
{ {
.id = DrvOutput {
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "bar",
},
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" }, .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" },
}, },
}, },

73
src/libstore-tests/worker-protocol.cc
View file

@ -125,49 +125,42 @@ VERSIONED_CHARACTERIZATION_TEST(
VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest, WorkerProtoTest,
drvOutput, drvOutput,
"drv-output", "drv-output-1.39",
defaultVersion, 1 << 8 | 39,
(std::tuple<DrvOutput, DrvOutput> { (std::tuple<DrvOutput, DrvOutput> {
{ {
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), .drvPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" },
.outputName = "baz", .outputName = "baz",
}, },
DrvOutput { DrvOutput {
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), .drvPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" },
.outputName = "quux", .outputName = "quux",
}, },
})) }))
VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
unkeyedRealisation,
"unkeyed-realisation-1.39",
1 << 8 | 39,
(UnkeyedRealisation {
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
.signatures = { "asdf", "qwer" },
}))
VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest, WorkerProtoTest,
realisation, realisation,
"realisation", "realisation-1.39",
defaultVersion, 1 << 8 | 39,
(std::tuple<Realisation, Realisation> { (Realisation {
Realisation { UnkeyedRealisation {
.id = DrvOutput {
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
.outputName = "baz",
},
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
.signatures = { "asdf", "qwer" }, .signatures = { "asdf", "qwer" },
}, },
Realisation { DrvOutput {
.id = { .drvPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" },
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), .outputName = "baz",
.outputName = "baz",
},
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
.signatures = { "asdf", "qwer" },
.dependentRealisations = {
{
DrvOutput {
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "quux",
},
StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
},
},
}, },
})) }))
@ -216,20 +209,12 @@ VERSIONED_CHARACTERIZATION_TEST(
{ {
"foo", "foo",
{ {
.id = DrvOutput {
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "foo",
},
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
}, },
}, },
{ {
"bar", "bar",
{ {
.id = DrvOutput {
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "bar",
},
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" }, .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" },
}, },
}, },
@ -266,20 +251,12 @@ VERSIONED_CHARACTERIZATION_TEST(
{ {
"foo", "foo",
{ {
.id = DrvOutput {
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "foo",
},
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
}, },
}, },
{ {
"bar", "bar",
{ {
.id = DrvOutput {
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "bar",
},
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" }, .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" },
}, },
}, },
@ -318,20 +295,12 @@ VERSIONED_CHARACTERIZATION_TEST(
{ {
"foo", "foo",
{ {
.id = DrvOutput {
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "foo",
},
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
}, },
}, },
{ {
"bar", "bar",
{ {
.id = DrvOutput {
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "bar",
},
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" }, .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" },
}, },
}, },

37
src/libstore/binary-cache-store.cc
View file

@ -493,22 +493,36 @@ StorePath BinaryCacheStore::addToStore(
})->path; })->path;
} }
void BinaryCacheStore::queryRealisationUncached(const DrvOutput & id, std::string BinaryCacheStore::makeRealisationPath(const DrvOutput & id)
Callback<std::shared_ptr<const Realisation>> callback) noexcept
{ {
auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi"; return realisationsPrefix
+ "/" + id.drvPath.to_string()
+ "/" + id.outputName
+ ".doi";
}
void BinaryCacheStore::queryRealisationUncached(const DrvOutput & id,
Callback<std::shared_ptr<const UnkeyedRealisation>> callback) noexcept
{
auto outputInfoFilePath = makeRealisationPath(id);
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
Callback<std::optional<std::string>> newCallback = { Callback<std::optional<std::string>> newCallback = {
[=](std::future<std::optional<std::string>> fut) { [=,this](std::future<std::optional<std::string>> fut) {
try { try {
auto data = fut.get(); auto data = fut.get();
if (!data) return (*callbackPtr)({}); if (!data) return (*callbackPtr)({});
auto realisation = Realisation::fromJSON( UnkeyedRealisation realisation { .outPath = StorePath::dummy, };
nlohmann::json::parse(*data), outputInfoFilePath); auto json = nlohmann::json::parse(*data);
return (*callbackPtr)(std::make_shared<const Realisation>(realisation)); try {
realisation = UnkeyedRealisation::fromJSON(*this, json);
} catch (Error & e) {
e.addTrace({}, "reading build trace key-value file '%s'", outputInfoFilePath);
throw;
}
return (*callbackPtr)(std::make_shared<const UnkeyedRealisation>(realisation));
} catch (...) { } catch (...) {
callbackPtr->rethrow(); callbackPtr->rethrow();
} }
@ -518,11 +532,14 @@ void BinaryCacheStore::queryRealisationUncached(const DrvOutput & id,
getFile(outputInfoFilePath, std::move(newCallback)); getFile(outputInfoFilePath, std::move(newCallback));
} }
void BinaryCacheStore::registerDrvOutput(const Realisation& info) { void BinaryCacheStore::registerDrvOutput(const Realisation & info)
{
if (diskCache) if (diskCache)
diskCache->upsertRealisation(getUri(), info); diskCache->upsertRealisation(getUri(), info);
auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi"; upsertFile(
upsertFile(filePath, info.toJSON().dump(), "application/json"); makeRealisationPath(info.id),
info.toJSON(*this).dump(),
"application/json");
} }
ref<SourceAccessor> BinaryCacheStore::getFSAccessor(bool requireValidPath) ref<SourceAccessor> BinaryCacheStore::getFSAccessor(bool requireValidPath)

46
src/libstore/build/derivation-building-goal.cc
View file

@ -309,14 +309,11 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution()
SingleDrvOutputs builtOutputs; SingleDrvOutputs builtOutputs;
if (resolvedResult.success()) { if (resolvedResult.success()) {
auto resolvedHashes = staticOutputHashes(worker.store, drvResolved);
StorePathSet outputPaths; StorePathSet outputPaths;
for (auto & outputName : drvResolved.outputNames()) { for (auto & outputName : drvResolved.outputNames()) {
auto initialOutput = get(initialOutputs, outputName); auto initialOutput = get(initialOutputs, outputName);
auto resolvedHash = get(resolvedHashes, outputName); if ((!initialOutput))
if ((!initialOutput) || (!resolvedHash))
throw Error( throw Error(
"derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolve)", "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolve)",
worker.store.printStorePath(drvPath), outputName); worker.store.printStorePath(drvPath), outputName);
@ -325,11 +322,14 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution()
auto take1 = get(resolvedResult.builtOutputs, outputName); auto take1 = get(resolvedResult.builtOutputs, outputName);
if (take1) return *take1; if (take1) return *take1;
/* The above `get` should work. But sateful tracking of /* The above `get` should work. But stateful tracking of
outputs in resolvedResult, this can get out of sync with the outputs in resolvedResult, this can get out of sync with the
store, which is our actual source of truth. For now we just store, which is our actual source of truth. For now we just
check the store directly if it fails. */ check the store directly if it fails. */
auto take2 = worker.evalStore.queryRealisation(DrvOutput { *resolvedHash, outputName }); auto take2 = worker.evalStore.queryRealisation(DrvOutput {
.drvPath = pathResolved,
.outputName = outputName,
});
if (take2) return *take2; if (take2) return *take2;
throw Error( throw Error(
@ -337,19 +337,6 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution()
worker.store.printStorePath(pathResolved), outputName); worker.store.printStorePath(pathResolved), outputName);
}(); }();
if (!drv->type().isImpure()) {
auto newRealisation = realisation;
newRealisation.id = DrvOutput { initialOutput->outputHash, outputName };
newRealisation.signatures.clear();
if (!drv->type().isFixed()) {
auto & drvStore = worker.evalStore.isValidPath(drvPath)
? worker.evalStore
: worker.store;
newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath, &drvStore);
}
worker.store.signRealisation(newRealisation);
worker.store.registerDrvOutput(newRealisation);
}
outputPaths.insert(realisation.outPath); outputPaths.insert(realisation.outPath);
builtOutputs.emplace(outputName, realisation); builtOutputs.emplace(outputName, realisation);
} }
@ -1161,7 +1148,7 @@ std::pair<bool, SingleDrvOutputs> DerivationBuildingGoal::checkPathValidity()
: PathStatus::Corrupt, : PathStatus::Corrupt,
}; };
} }
auto drvOutput = DrvOutput{info.outputHash, i.first}; auto drvOutput = DrvOutput{drvPath, i.first};
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
if (auto real = worker.store.queryRealisation(drvOutput)) { if (auto real = worker.store.queryRealisation(drvOutput)) {
info.known = { info.known = {
@ -1173,16 +1160,21 @@ std::pair<bool, SingleDrvOutputs> DerivationBuildingGoal::checkPathValidity()
// derivation, and the output path is valid, but we don't have // derivation, and the output path is valid, but we don't have
// its realisation stored (probably because it has been built // its realisation stored (probably because it has been built
// without the `ca-derivations` experimental flag). // without the `ca-derivations` experimental flag).
worker.store.registerDrvOutput( worker.store.registerDrvOutput(Realisation {
Realisation { {
drvOutput, .outPath = info.known->path,
info.known->path, },
} drvOutput,
); });
} }
} }
if (info.known && info.known->isValid()) if (info.known && info.known->isValid())
validOutputs.emplace(i.first, Realisation { drvOutput, info.known->path }); validOutputs.emplace(i.first, Realisation {
{
.outPath = info.known->path,
},
drvOutput,
});
} }
bool allValid = true; bool allValid = true;

27
src/libstore/build/derivation-goal.cc
View file

@ -118,11 +118,9 @@ Goal::Co DerivationGoal::haveDerivation()
if (impure) experimentalFeatureSettings.require(Xp::ImpureDerivations); if (impure) experimentalFeatureSettings.require(Xp::ImpureDerivations);
auto outputHashes = staticOutputHashes(worker.evalStore, *drv); for (auto & [outputName, _] : drv->outputs) {
for (auto & [outputName, outputHash] : outputHashes) {
InitialOutput v{ InitialOutput v{
.wanted = true, // Will be refined later .wanted = true, // Will be refined later
.outputHash = outputHash
}; };
/* TODO we might want to also allow randomizing the paths /* TODO we might want to also allow randomizing the paths
@ -170,7 +168,7 @@ Goal::Co DerivationGoal::haveDerivation()
waitees.insert( waitees.insert(
upcast_goal( upcast_goal(
worker.makeDrvOutputSubstitutionGoal( worker.makeDrvOutputSubstitutionGoal(
DrvOutput{status.outputHash, outputName}, DrvOutput{drvPath, outputName},
buildMode == bmRepair ? Repair : NoRepair buildMode == bmRepair ? Repair : NoRepair
) )
) )
@ -358,7 +356,7 @@ std::pair<bool, SingleDrvOutputs> DerivationGoal::checkPathValidity()
: PathStatus::Corrupt, : PathStatus::Corrupt,
}; };
} }
auto drvOutput = DrvOutput{info.outputHash, i.first}; auto drvOutput = DrvOutput{drvPath, i.first};
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
if (auto real = worker.store.queryRealisation(drvOutput)) { if (auto real = worker.store.queryRealisation(drvOutput)) {
info.known = { info.known = {
@ -370,16 +368,21 @@ std::pair<bool, SingleDrvOutputs> DerivationGoal::checkPathValidity()
// derivation, and the output path is valid, but we don't have // derivation, and the output path is valid, but we don't have
// its realisation stored (probably because it has been built // its realisation stored (probably because it has been built
// without the `ca-derivations` experimental flag). // without the `ca-derivations` experimental flag).
worker.store.registerDrvOutput( worker.store.registerDrvOutput(Realisation {
Realisation { {
drvOutput, .outPath = info.known->path,
info.known->path, },
} drvOutput,
); });
} }
} }
if (info.known && info.known->isValid()) if (info.known && info.known->isValid())
validOutputs.emplace(i.first, Realisation { drvOutput, info.known->path }); validOutputs.emplace(i.first, Realisation {
{
.outPath = info.known->path,
},
drvOutput,
});
} }
// If we requested all the outputs, we are always fine. // If we requested all the outputs, we are always fine.

66
src/libstore/build/drv-output-substitution-goal.cc
View file

@ -12,7 +12,7 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(
: Goal(worker, init()) : Goal(worker, init())
, id(id) , id(id)
{ {
name = fmt("substitution of '%s'", id.to_string()); name = fmt("substitution of '%s'", id.render(worker.store));
trace("created"); trace("created");
} }
@ -42,10 +42,10 @@ Goal::Co DrvOutputSubstitutionGoal::init()
outPipe->createAsyncPipe(worker.ioport.get()); outPipe->createAsyncPipe(worker.ioport.get());
#endif #endif
auto promise = std::make_shared<std::promise<std::shared_ptr<const Realisation>>>(); auto promise = std::make_shared<std::promise<std::shared_ptr<const UnkeyedRealisation>>>();
sub->queryRealisation( sub->queryRealisation(
id, {[outPipe(outPipe), promise(promise)](std::future<std::shared_ptr<const Realisation>> res) { id, {[outPipe(outPipe), promise(promise)](std::future<std::shared_ptr<const UnkeyedRealisation>> res) {
try { try {
Finally updateStats([&]() { outPipe->writeSide.close(); }); Finally updateStats([&]() { outPipe->writeSide.close(); });
promise->set_value(res.get()); promise->set_value(res.get());
@ -70,12 +70,6 @@ Goal::Co DrvOutputSubstitutionGoal::init()
worker.childTerminated(this); worker.childTerminated(this);
/*
* The realisation corresponding to the given output id.
* Will be filled once we can get it.
*/
std::shared_ptr<const Realisation> outputInfo;
try { try {
outputInfo = promise->get_future().get(); outputInfo = promise->get_future().get();
} catch (std::exception & e) { } catch (std::exception & e) {
@ -86,38 +80,24 @@ Goal::Co DrvOutputSubstitutionGoal::init()
if (!outputInfo) if (!outputInfo)
continue; continue;
bool failed = false; co_await await(Goals{worker.makePathSubstitutionGoal(outputInfo->outPath)});
Goals waitees; trace("output path substituted");
for (const auto & [depId, depPath] : outputInfo->dependentRealisations) { if (nrFailed > 0) {
if (depId != id) { debug("The output path of the derivation output '%s' could not be substituted", id.render(worker.store));
if (auto localOutputInfo = worker.store.queryRealisation(depId); co_return amDone(nrNoSubstituters > 0 ? ecNoSubstituters : ecFailed);
localOutputInfo && localOutputInfo->outPath != depPath) {
warn(
"substituter '%s' has an incompatible realisation for '%s', ignoring.\n"
"Local: %s\n"
"Remote: %s",
sub->getUri(),
depId.to_string(),
worker.store.printStorePath(localOutputInfo->outPath),
worker.store.printStorePath(depPath));
failed = true;
break;
}
waitees.insert(worker.makeDrvOutputSubstitutionGoal(depId));
}
} }
if (failed) worker.store.registerDrvOutput({*outputInfo, id});
continue;
co_return realisationFetched(std::move(waitees), outputInfo, sub); trace("finished");
co_return amDone(ecSuccess);
} }
/* None left. Terminate this goal and let someone else deal /* None left. Terminate this goal and let someone else deal
with it. */ with it. */
debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string()); debug("derivation output '%s' is required, but there is no substituter that can provide it", id.render(worker.store));
if (substituterFailed) { if (substituterFailed) {
worker.failedSubstitutions++; worker.failedSubstitutions++;
@ -130,31 +110,11 @@ Goal::Co DrvOutputSubstitutionGoal::init()
co_return amDone(substituterFailed ? ecFailed : ecNoSubstituters); co_return amDone(substituterFailed ? ecFailed : ecNoSubstituters);
} }
Goal::Co DrvOutputSubstitutionGoal::realisationFetched(
Goals waitees, std::shared_ptr<const Realisation> outputInfo, nix::ref<nix::Store> sub)
{
waitees.insert(worker.makePathSubstitutionGoal(outputInfo->outPath));
co_await await(std::move(waitees));
trace("output path substituted");
if (nrFailed > 0) {
debug("The output path of the derivation output '%s' could not be substituted", id.to_string());
co_return amDone(nrNoSubstituters > 0 ? ecNoSubstituters : ecFailed);
}
worker.store.registerDrvOutput(*outputInfo);
trace("finished");
co_return amDone(ecSuccess);
}
std::string DrvOutputSubstitutionGoal::key() std::string DrvOutputSubstitutionGoal::key()
{ {
/* "a$" ensures substitution goals happen before derivation /* "a$" ensures substitution goals happen before derivation
goals. */ goals. */
return "a$" + std::string(id.to_string()); return "a$" + std::string(id.render(worker.store));
} }
void DrvOutputSubstitutionGoal::handleEOF(Descriptor fd) void DrvOutputSubstitutionGoal::handleEOF(Descriptor fd)

27
src/libstore/ca-specific-schema.sql
View file

@ -12,30 +12,3 @@ create table if not exists Realisations (
); );
create index if not exists IndexRealisations on Realisations(drvPath, outputName); create index if not exists IndexRealisations on Realisations(drvPath, outputName);
-- We can end-up in a weird edge-case where a path depends on itself because
-- it’s an output of a CA derivation, that happens to be the same as one of its
-- dependencies.
-- In that case we have a dependency loop (path -> realisation1 -> realisation2
-- -> path) that we need to break by removing the dependencies between the
-- realisations
create trigger if not exists DeleteSelfRefsViaRealisations before delete on ValidPaths
begin
delete from RealisationsRefs where realisationReference in (
select id from Realisations where outputPath = old.id
);
end;
create table if not exists RealisationsRefs (
referrer integer not null,
realisationReference integer,
foreign key (referrer) references Realisations(id) on delete cascade,
foreign key (realisationReference) references Realisations(id) on delete restrict
);
-- used by deletion trigger
create index if not exists IndexRealisationsRefsRealisationReference on RealisationsRefs(realisationReference);
-- used by QueryRealisationReferences
create index if not exists IndexRealisationsRefs on RealisationsRefs(referrer);
-- used by cascade deletion when ValidPaths is deleted
create index if not exists IndexRealisationsRefsOnOutputPath on Realisations(outputPath);

26
src/libstore/common-protocol.cc
View file

@ -46,32 +46,6 @@ void CommonProto::Serialise<ContentAddress>::write(const StoreDirConfig & store,
} }
Realisation CommonProto::Serialise<Realisation>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
{
std::string rawInput = readString(conn.from);
return Realisation::fromJSON(
nlohmann::json::parse(rawInput),
"remote-protocol"
);
}
void CommonProto::Serialise<Realisation>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const Realisation & realisation)
{
conn.to << realisation.toJSON().dump();
}
DrvOutput CommonProto::Serialise<DrvOutput>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
{
return DrvOutput::parse(readString(conn.from));
}
void CommonProto::Serialise<DrvOutput>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput)
{
conn.to << drvOutput.to_string();
}
std::optional<StorePath> CommonProto::Serialise<std::optional<StorePath>>::read(const StoreDirConfig & store, CommonProto::ReadConn conn) std::optional<StorePath> CommonProto::Serialise<std::optional<StorePath>>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
{ {
auto s = readString(conn.from); auto s = readString(conn.from);

26
src/libstore/daemon.cc
View file

@ -960,32 +960,30 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case WorkerProto::Op::RegisterDrvOutput: { case WorkerProto::Op::RegisterDrvOutput: {
logger->startWork(); logger->startWork();
if (GET_PROTOCOL_MINOR(conn.protoVersion) < 31) { // TODO move to WorkerProto::Serialise<DrvOutput> and friends
auto outputId = DrvOutput::parse(readString(conn.from)); //if (GET_PROTOCOL_MINOR(conn.protoVersion) < 39) {
auto outputPath = StorePath(readString(conn.from)); // throw Error("old-style build traces no longer supported");
store->registerDrvOutput(Realisation{ //}
.id = outputId, .outPath = outputPath}); auto realisation = WorkerProto::Serialise<Realisation>::read(*store, rconn);
} else { store->registerDrvOutput(realisation);
auto realisation = WorkerProto::Serialise<Realisation>::read(*store, rconn);
store->registerDrvOutput(realisation);
}
logger->stopWork(); logger->stopWork();
break; break;
} }
case WorkerProto::Op::QueryRealisation: { case WorkerProto::Op::QueryRealisation: {
logger->startWork(); logger->startWork();
auto outputId = DrvOutput::parse(readString(conn.from)); auto outputId = WorkerProto::Serialise<DrvOutput>::read(*store, rconn);
auto info = store->queryRealisation(outputId); std::optional<UnkeyedRealisation> info = *store->queryRealisation(outputId);
logger->stopWork(); logger->stopWork();
if (GET_PROTOCOL_MINOR(conn.protoVersion) < 31) { if (GET_PROTOCOL_MINOR(conn.protoVersion) < 31) {
std::set<StorePath> outPaths; std::set<StorePath> outPaths;
if (info) outPaths.insert(info->outPath); if (info) outPaths.insert(info->outPath);
WorkerProto::write(*store, wconn, outPaths); WorkerProto::write(*store, wconn, outPaths);
} else if (GET_PROTOCOL_MINOR(conn.protoVersion) < 39) {
// No longer support this format
WorkerProto::write(*store, wconn, StringSet{});
} else { } else {
std::set<Realisation> realisations; WorkerProto::write(*store, wconn, info);
if (info) realisations.insert(*info);
WorkerProto::write(*store, wconn, realisations);
} }
break; break;
} }

225
src/libstore/derivations.cc
View file

@ -787,7 +787,7 @@ Sync<DrvHashes> drvHashes;
/* Look up the derivation by value and memoize the /* Look up the derivation by value and memoize the
`hashDerivationModulo` call. `hashDerivationModulo` call.
*/ */
static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPath) static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath & drvPath)
{ {
{ {
auto hashes = drvHashes.lock(); auto hashes = drvHashes.lock();
@ -796,13 +796,16 @@ static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPa
return h->second; return h->second;
} }
} }
auto h = hashDerivationModulo(
store,
store.readInvalidDerivation(drvPath),
false);
// Cache it // Cache it
drvHashes.lock()->insert_or_assign(drvPath, h); auto [iter, _] = drvHashes.lock()->insert_or_assign(
return h; drvPath,
hashDerivationModulo(
store,
store.readInvalidDerivation(drvPath),
false));
return iter->second;
} }
/* See the header for interface details. These are the implementation details. /* See the header for interface details. These are the implementation details.
@ -822,12 +825,10 @@ static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPa
don't leak the provenance of fixed outputs, reducing pointless cache don't leak the provenance of fixed outputs, reducing pointless cache
misses as the build itself won't know this. misses as the build itself won't know this.
*/ */
DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
{ {
auto type = drv.type();
/* Return a fixed hash for fixed-output derivations. */ /* Return a fixed hash for fixed-output derivations. */
if (type.isFixed()) { if (drv.type().isFixed()) {
std::map<std::string, Hash> outputHashes; std::map<std::string, Hash> outputHashes;
for (const auto & i : drv.outputs) { for (const auto & i : drv.outputs) {
auto & dof = std::get<DerivationOutput::CAFixed>(i.second.raw); auto & dof = std::get<DerivationOutput::CAFixed>(i.second.raw);
@ -837,58 +838,69 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
+ store.printStorePath(dof.path(store, drv.name, i.first))); + store.printStorePath(dof.path(store, drv.name, i.first)));
outputHashes.insert_or_assign(i.first, std::move(hash)); outputHashes.insert_or_assign(i.first, std::move(hash));
} }
return DrvHash { return outputHashes;
.hashes = outputHashes,
.kind = DrvHash::Kind::Regular,
};
} }
auto kind = std::visit(overloaded { if (std::visit(overloaded {
[](const DerivationType::InputAddressed & ia) { [](const DerivationType::InputAddressed & ia) {
/* This might be a "pesimistically" deferred output, so we don't /* This might be a "pesimistically" deferred output, so we don't
"taint" the kind yet. */ "taint" the kind yet. */
return DrvHash::Kind::Regular; return false;
}, },
[](const DerivationType::ContentAddressed & ca) { [](const DerivationType::ContentAddressed & ca) {
return ca.fixed // Already covered
? DrvHash::Kind::Regular assert(!ca.fixed);
: DrvHash::Kind::Deferred; return true;
}, },
[](const DerivationType::Impure &) -> DrvHash::Kind { [](const DerivationType::Impure &) {
return DrvHash::Kind::Deferred; return true;
} }
}, drv.type().raw); }, drv.type().raw)) {
return DrvHashModulo::DeferredDrv{};
}
/* For other derivations, replace the inputs paths with recursive
calls to this function. */
DerivedPathMap<StringSet>::ChildNode::Map inputs2; DerivedPathMap<StringSet>::ChildNode::Map inputs2;
for (auto & [drvPath, node] : drv.inputDrvs.map) { for (auto & [drvPath, node] : drv.inputDrvs.map) {
/* Need to build and resolve dynamic derivations first */
if (!node.childMap.empty()) {
return DrvHashModulo::DeferredDrv{};
}
const auto & res = pathDerivationModulo(store, drvPath); const auto & res = pathDerivationModulo(store, drvPath);
if (res.kind == DrvHash::Kind::Deferred) if (std::visit(overloaded {
kind = DrvHash::Kind::Deferred; [&](const DrvHashModulo::DeferredDrv &) {
for (auto & outputName : node.value) { return true;
const auto h = get(res.hashes, outputName); },
if (!h) // Regular non-CA derivation, replace derivation
throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); [&](const DrvHashModulo::DrvHash & drvHash) {
inputs2[h->to_string(HashFormat::Base16, false)].value.insert(outputName); inputs2.insert_or_assign(
drvHash.to_string(HashFormat::Base16, false),
node);
return false;
},
// CA derivation's output hashes
[&](const DrvHashModulo::CaOutputHashes & outputHashes) {
for (auto & outputName : node.value) {
/* Put each one in with a single "out" output.. */
const auto h = get(outputHashes, outputName);
if (!h)
throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name);
inputs2.insert_or_assign(
h->to_string(HashFormat::Base16, false),
DerivedPathMap<StringSet>::ChildNode{
.value = {"out"},
});
}
return false;
},
}, res.raw)) {
return DrvHashModulo::DeferredDrv{};
} }
} }
auto hash = hashString(HashAlgorithm::SHA256, drv.unparse(store, maskOutputs, &inputs2)); return hashString(HashAlgorithm::SHA256, drv.unparse(store, maskOutputs, &inputs2));
std::map<std::string, Hash> outputHashes;
for (const auto & [outputName, _] : drv.outputs) {
outputHashes.insert_or_assign(outputName, hash);
}
return DrvHash {
.hashes = outputHashes,
.kind = kind,
};
}
std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv)
{
return hashDerivationModulo(store, drv, true).hashes;
} }
@ -1029,25 +1041,39 @@ void BasicDerivation::applyRewrites(const StringMap & rewrites)
env = std::move(newEnv); env = std::move(newEnv);
} }
static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) void resolveInputAddressed(Store & store, Derivation & drv)
{ {
drv.applyRewrites(rewrites); std::optional<DrvHashModulo> hashModulo_;
auto hashModulo = [&]() -> const auto & {
if (!hashModulo_) {
// somewhat expensive so we do lazily
hashModulo_ = hashDerivationModulo(store, drv, true);
}
return *hashModulo_;
};
auto hashModulo = hashDerivationModulo(store, Derivation(drv), true);
for (auto & [outputName, output] : drv.outputs) { for (auto & [outputName, output] : drv.outputs) {
if (std::holds_alternative<DerivationOutput::Deferred>(output.raw)) { if (std::holds_alternative<DerivationOutput::Deferred>(output.raw)) {
auto h = get(hashModulo.hashes, outputName); std::visit(overloaded {
if (!h) [&](const DrvHashModulo::DrvHash & drvHash) {
throw Error("derivation '%s' output '%s' has no hash (derivations.cc/rewriteDerivation)", auto outPath = store.makeOutputPath(outputName, drvHash, drv.name);
drv.name, outputName); drv.env.insert_or_assign(outputName, store.printStorePath(outPath));
auto outPath = store.makeOutputPath(outputName, *h, drv.name); output = DerivationOutput::InputAddressed {
drv.env[outputName] = store.printStorePath(outPath); .path = std::move(outPath),
output = DerivationOutput::InputAddressed { };
.path = std::move(outPath), },
}; [&](const DrvHashModulo::CaOutputHashes &) {
/* Shouldn't happen as the original output is
deferred (waiting to be input-addressed). */
assert(false);
},
[&](const DrvHashModulo::DeferredDrv &) {
// Nothing to do, already deferred
},
}, hashModulo().raw);
} }
} }
} }
std::optional<BasicDerivation> Derivation::tryResolve(Store & store, Store * evalStore) const std::optional<BasicDerivation> Derivation::tryResolve(Store & store, Store * evalStore) const
@ -1118,9 +1144,13 @@ std::optional<BasicDerivation> Derivation::tryResolve(
nullptr, make_ref<const SingleDerivedPath>(SingleDerivedPath::Opaque{inputDrv}), inputNode, queryResolutionChain)) nullptr, make_ref<const SingleDerivedPath>(SingleDerivedPath::Opaque{inputDrv}), inputNode, queryResolutionChain))
return std::nullopt; return std::nullopt;
rewriteDerivation(store, resolved, inputRewrites); resolved.applyRewrites(inputRewrites);
return resolved; Derivation resolved2{std::move(resolved)};
resolveInputAddressed(store, resolved2);
return resolved2;
} }
@ -1148,38 +1178,73 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const
// combinations that are currently prohibited. // combinations that are currently prohibited.
type(); type();
std::optional<DrvHash> hashesModulo; std::optional<DrvHashModulo> hashModulo_;
for (auto & i : outputs) {
auto hashModulo = [&]() -> const auto & {
if (!hashModulo_) {
// somewhat expensive so we do lazily
hashModulo_ = hashDerivationModulo(store, *this, true);
}
return *hashModulo_;
};
for (auto & [outputName, output] : outputs) {
std::visit(overloaded { std::visit(overloaded {
[&](const DerivationOutput::InputAddressed & doia) { [&](const DerivationOutput::InputAddressed & doia) {
if (!hashesModulo) { std::visit(overloaded {
// somewhat expensive so we do lazily [&](const DrvHashModulo::DrvHash & drvHash) {
hashesModulo = hashDerivationModulo(store, *this, true); StorePath recomputed = store.makeOutputPath(outputName, drvHash, drvName);
} if (doia.path != recomputed)
auto currentOutputHash = get(hashesModulo->hashes, i.first); throw Error(
if (!currentOutputHash) "derivation '%s' has incorrect output '%s', should be '%s'",
throw Error("derivation '%s' has unexpected output '%s' (local-store / hashesModulo) named '%s'", store.printStorePath(drvPath),
store.printStorePath(drvPath), store.printStorePath(doia.path), i.first); store.printStorePath(doia.path),
StorePath recomputed = store.makeOutputPath(i.first, *currentOutputHash, drvName); store.printStorePath(recomputed));
if (doia.path != recomputed) },
throw Error("derivation '%s' has incorrect output '%s', should be '%s'", [&](const DrvHashModulo::CaOutputHashes &) {
store.printStorePath(drvPath), store.printStorePath(doia.path), store.printStorePath(recomputed)); /* Shouldn't happen as the original output is
envHasRightPath(doia.path, i.first); input-addressed. */
assert(false);
},
[&](const DrvHashModulo::DeferredDrv &) {
throw Error(
"derivation '%s' has output '%s', but derivation is not yet ready to be input-addressed",
store.printStorePath(drvPath),
store.printStorePath(doia.path));
},
}, hashModulo().raw);
envHasRightPath(doia.path, outputName);
}, },
[&](const DerivationOutput::CAFixed & dof) { [&](const DerivationOutput::CAFixed & dof) {
auto path = dof.path(store, drvName, i.first); auto path = dof.path(store, drvName, outputName);
envHasRightPath(path, i.first); envHasRightPath(path, outputName);
}, },
[&](const DerivationOutput::CAFloating &) { [&](const DerivationOutput::CAFloating &) {
/* Nothing to check */ /* Nothing to check */
}, },
[&](const DerivationOutput::Deferred &) { [&](const DerivationOutput::Deferred &) {
/* Nothing to check */ /* Nothing to check */
std::visit(overloaded {
[&](const DrvHashModulo::DrvHash & drvHash) {
throw Error(
"derivation '%s' has deferred output '%s', yet is ready to be input-addressed",
store.printStorePath(drvPath),
outputName);
},
[&](const DrvHashModulo::CaOutputHashes &) {
/* Shouldn't happen as the original output is
input-addressed. */
assert(false);
},
[&](const DrvHashModulo::DeferredDrv &) {
/* Nothing to check */
},
}, hashModulo().raw);
}, },
[&](const DerivationOutput::Impure &) { [&](const DerivationOutput::Impure &) {
/* Nothing to check */ /* Nothing to check */
}, },
}, i.second.raw); }, output.raw);
} }
} }

2
src/libstore/dummy-store.cc
View file

@ -80,7 +80,7 @@ struct DummyStore : virtual Store
{ unsupported("narFromPath"); } { unsupported("narFromPath"); }
void queryRealisationUncached(const DrvOutput &, void queryRealisationUncached(const DrvOutput &,
Callback<std::shared_ptr<const Realisation>> callback) noexcept override Callback<std::shared_ptr<const UnkeyedRealisation>> callback) noexcept override
{ callback(nullptr); } { callback(nullptr); }
virtual ref<SourceAccessor> getFSAccessor(bool requireValidPath) override virtual ref<SourceAccessor> getFSAccessor(bool requireValidPath) override

22
src/libstore/include/nix/store/binary-cache-store.hh
View file

@ -71,13 +71,27 @@ private:
protected: protected:
// The prefix under which realisation infos will be stored /**
const std::string realisationsPrefix = "realisations"; * The prefix under which realisation infos will be stored
*
* @note The previous (still experimental, though) hash-keyed
* realisations were under "realisations". "build trace" is a better
* name anyways (issue #11895), and this serves as some light
* versioning.
*/
constexpr const static std::string realisationsPrefix = "build-trace";
const std::string cacheInfoFile = "nix-cache-info"; constexpr const static std::string cacheInfoFile = "nix-cache-info";
BinaryCacheStore(Config &); BinaryCacheStore(Config &);
/**
* Compute the path to the given realisation
*
* It's `${realisationsPrefix}/${drvPath}/${outputName}`.
*/
std::string makeRealisationPath(const DrvOutput & id);
public: public:
virtual bool fileExists(const std::string & path) = 0; virtual bool fileExists(const std::string & path) = 0;
@ -161,7 +175,7 @@ public:
void registerDrvOutput(const Realisation & info) override; void registerDrvOutput(const Realisation & info) override;
void queryRealisationUncached(const DrvOutput &, void queryRealisationUncached(const DrvOutput &,
Callback<std::shared_ptr<const Realisation>> callback) noexcept override; Callback<std::shared_ptr<const UnkeyedRealisation>> callback) noexcept override;
void narFromPath(const StorePath & path, Sink & sink) override; void narFromPath(const StorePath & path, Sink & sink) override;

1
src/libstore/include/nix/store/build/derivation-building-misc.hh
View file

@ -44,7 +44,6 @@ struct InitialOutputStatus
struct InitialOutput struct InitialOutput
{ {
bool wanted; bool wanted;
Hash outputHash;
std::optional<InitialOutputStatus> known; std::optional<InitialOutputStatus> known;
}; };

2
src/libstore/include/nix/store/build/derivation-creation-and-realisation-goal.hh
View file

@ -45,7 +45,7 @@ struct DerivationCreationAndRealisationGoal : public Goal
* *
* - For input-addressed derivations, always the precomputed paths * - For input-addressed derivations, always the precomputed paths
* *
* - For content-addressed derivations, calcuated from whatever the * - For content-addressing derivations, calcuated from whatever the
* hash ends up being. (Note that fixed outputs derivations that * hash ends up being. (Note that fixed outputs derivations that
* produce the "wrong" output still install that data under its * produce the "wrong" output still install that data under its
* true content-address.) * true content-address.)

8
src/libstore/include/nix/store/build/drv-output-substitution-goal.hh
View file

@ -35,11 +35,13 @@ public:
RepairFlag repair = NoRepair, RepairFlag repair = NoRepair,
std::optional<ContentAddress> ca = std::nullopt); std::optional<ContentAddress> ca = std::nullopt);
typedef void (DrvOutputSubstitutionGoal::*GoalState)(); /**
GoalState state; * The realisation corresponding to the given output id.
* Will be filled once we can get it.
*/
std::shared_ptr<const UnkeyedRealisation> outputInfo;
Co init(); Co init();
Co realisationFetched(Goals waitees, std::shared_ptr<const Realisation> outputInfo, nix::ref<nix::Store> sub);
void timedOut(Error && ex) override void timedOut(Error && ex) override
{ {

6
src/libstore/include/nix/store/common-protocol-impl.hh
View file

@ -25,14 +25,14 @@ namespace nix {
LengthPrefixedProtoHelper<CommonProto, T >::write(store, conn, t); \ LengthPrefixedProtoHelper<CommonProto, T >::write(store, conn, t); \
} }
#define COMMA_ ,
COMMON_USE_LENGTH_PREFIX_SERIALISER(template<typename T>, std::vector<T>) COMMON_USE_LENGTH_PREFIX_SERIALISER(template<typename T>, std::vector<T>)
#define COMMA_ ,
COMMON_USE_LENGTH_PREFIX_SERIALISER(template<typename T COMMA_ typename Compare>, std::set<T COMMA_ Compare>) COMMON_USE_LENGTH_PREFIX_SERIALISER(template<typename T COMMA_ typename Compare>, std::set<T COMMA_ Compare>)
COMMON_USE_LENGTH_PREFIX_SERIALISER(template<typename... Ts>, std::tuple<Ts...>) COMMON_USE_LENGTH_PREFIX_SERIALISER(template<typename... Ts>, std::tuple<Ts...>)
COMMON_USE_LENGTH_PREFIX_SERIALISER( COMMON_USE_LENGTH_PREFIX_SERIALISER(
template<typename K COMMA_ typename V>, template<typename K COMMA_ typename V COMMA_ typename Compare>,
std::map<K COMMA_ V>) std::map<K COMMA_ V COMMA_ Compare>)
#undef COMMA_ #undef COMMA_

7
src/libstore/include/nix/store/common-protocol.hh
View file

@ -12,7 +12,6 @@ struct Source;
class StorePath; class StorePath;
struct ContentAddress; struct ContentAddress;
struct DrvOutput; struct DrvOutput;
struct Realisation;
/** /**
@ -69,8 +68,6 @@ template<>
DECLARE_COMMON_SERIALISER(ContentAddress); DECLARE_COMMON_SERIALISER(ContentAddress);
template<> template<>
DECLARE_COMMON_SERIALISER(DrvOutput); DECLARE_COMMON_SERIALISER(DrvOutput);
template<>
DECLARE_COMMON_SERIALISER(Realisation);
#define COMMA_ , #define COMMA_ ,
template<typename T> template<typename T>
@ -80,8 +77,8 @@ DECLARE_COMMON_SERIALISER(std::set<T COMMA_ Compare>);
template<typename... Ts> template<typename... Ts>
DECLARE_COMMON_SERIALISER(std::tuple<Ts...>); DECLARE_COMMON_SERIALISER(std::tuple<Ts...>);
template<typename K, typename V> template<typename K, typename V, typename Compare>
DECLARE_COMMON_SERIALISER(std::map<K COMMA_ V>); DECLARE_COMMON_SERIALISER(std::map<K COMMA_ V COMMA_ Compare>);
#undef COMMA_ #undef COMMA_
/** /**

65
src/libstore/include/nix/store/derivations.hh
View file

@ -440,34 +440,44 @@ std::string outputPathName(std::string_view drvName, OutputNameView outputName);
* derivations (fixed-output or not) will have a different hash for each * derivations (fixed-output or not) will have a different hash for each
* output. * output.
*/ */
struct DrvHash { struct DrvHashModulo
{
/** /**
* Map from output names to hashes * Single hash for the derivation
*
* This is for an input-addressed derivation that doesn't
* transitively depend on any floating-CA derivations.
*/ */
std::map<std::string, Hash> hashes; using DrvHash = Hash;
enum struct Kind : bool { /**
/** * Known CA drv's output hashes, for fixed-output derivations whose
* Statically determined derivations. * output hashes are always known since they are fixed up-front.
* This hash will be directly used to compute the output paths */
*/ using CaOutputHashes = std::map<std::string, Hash>;
Regular,
/**
* Floating-output derivations (and their reverse dependencies).
*/
Deferred,
};
/** /**
* The kind of derivation this is, simplified for just "derivation hash * This derivation doesn't yet have known output hashes.
* modulo" purposes. *
* Either because itself is floating CA, or it (transtively) depends
* on a floating CA derivation.
*/ */
Kind kind; using DeferredDrv = std::monostate;
using Raw = std::variant<
DrvHash,
CaOutputHashes,
DeferredDrv
>;
Raw raw;
bool operator == (const DrvHashModulo &) const = default;
//auto operator <=> (const DrvHashModulo &) const = default;
MAKE_WRAPPER_CONSTRUCTOR(DrvHashModulo);
}; };
void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept;
/** /**
* Returns hashes with the details of fixed-output subderivations * Returns hashes with the details of fixed-output subderivations
* expunged. * expunged.
@ -492,20 +502,23 @@ void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept;
* ATerm, after subderivations have been likewise expunged from that * ATerm, after subderivations have been likewise expunged from that
* derivation. * derivation.
*/ */
DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs);
/** /**
* Return a map associating each output to a hash that uniquely identifies its * If a derivation is input addressed and doesn't yet have its input
* derivation (modulo the self-references). * addressed (is deferred) try using `hashDerivationModulo`.
* *
* \todo What is the Hash in this map? * Does nothing if not deferred input-addressed, or
* `hashDerivationModulo` indicates it is missing inputs' output paths
* and is not yet ready (and must stay deferred).
*/ */
std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv); void resolveInputAddressed(Store & store, Derivation & drv);
/** /**
* Memoisation of hashDerivationModulo(). * Memoisation of hashDerivationModulo().
*/ */
typedef std::map<StorePath, DrvHash> DrvHashes; typedef std::map<StorePath, DrvHashModulo> DrvHashes;
// FIXME: global, though at least thread-safe. // FIXME: global, though at least thread-safe.
extern Sync<DrvHashes> drvHashes; extern Sync<DrvHashes> drvHashes;

2
src/libstore/include/nix/store/legacy-ssh-store.hh
View file

@ -193,7 +193,7 @@ public:
std::optional<TrustedFlag> isTrustedClient() override; std::optional<TrustedFlag> isTrustedClient() override;
void queryRealisationUncached(const DrvOutput &, void queryRealisationUncached(const DrvOutput &,
Callback<std::shared_ptr<const Realisation>> callback) noexcept override Callback<std::shared_ptr<const UnkeyedRealisation>> callback) noexcept override
// TODO: Implement // TODO: Implement
{ unsupported("queryRealisation"); } { unsupported("queryRealisation"); }
}; };

20
src/libstore/include/nix/store/length-prefixed-protocol-helper.hh
View file

@ -55,14 +55,14 @@ LENGTH_PREFIXED_PROTO_HELPER(Inner, std::vector<T>);
#define COMMA_ , #define COMMA_ ,
template<class Inner, typename T, typename Compare> template<class Inner, typename T, typename Compare>
LENGTH_PREFIXED_PROTO_HELPER(Inner, std::set<T COMMA_ Compare>); LENGTH_PREFIXED_PROTO_HELPER(Inner, std::set<T COMMA_ Compare>);
#undef COMMA_
template<class Inner, typename... Ts> template<class Inner, typename... Ts>
LENGTH_PREFIXED_PROTO_HELPER(Inner, std::tuple<Ts...>); LENGTH_PREFIXED_PROTO_HELPER(Inner, std::tuple<Ts...>);
template<class Inner, typename K, typename V> template<class Inner, typename K, typename V, typename Compare>
#define LENGTH_PREFIXED_PROTO_HELPER_X std::map<K, V> #define LENGTH_PREFIXED_PROTO_HELPER_X std::map<K, V, Compare>
LENGTH_PREFIXED_PROTO_HELPER(Inner, LENGTH_PREFIXED_PROTO_HELPER_X); LENGTH_PREFIXED_PROTO_HELPER(Inner, LENGTH_PREFIXED_PROTO_HELPER_X);
#undef COMMA_
template<class Inner, typename T> template<class Inner, typename T>
std::vector<T> std::vector<T>
@ -110,12 +110,12 @@ void LengthPrefixedProtoHelper<Inner, std::set<T, Compare>>::write(
} }
} }
template<class Inner, typename K, typename V> template<class Inner, typename K, typename V, typename Compare>
std::map<K, V> std::map<K, V, Compare>
LengthPrefixedProtoHelper<Inner, std::map<K, V>>::read( LengthPrefixedProtoHelper<Inner, std::map<K, V, Compare>>::read(
const StoreDirConfig & store, typename Inner::ReadConn conn) const StoreDirConfig & store, typename Inner::ReadConn conn)
{ {
std::map<K, V> resMap; std::map<K, V, Compare> resMap;
auto size = readNum<size_t>(conn.from); auto size = readNum<size_t>(conn.from);
while (size--) { while (size--) {
auto k = S<K>::read(store, conn); auto k = S<K>::read(store, conn);
@ -125,10 +125,10 @@ LengthPrefixedProtoHelper<Inner, std::map<K, V>>::read(
return resMap; return resMap;
} }
template<class Inner, typename K, typename V> template<class Inner, typename K, typename V, typename Compare>
void void
LengthPrefixedProtoHelper<Inner, std::map<K, V>>::write( LengthPrefixedProtoHelper<Inner, std::map<K, V, Compare>>::write(
const StoreDirConfig & store, typename Inner::WriteConn conn, const std::map<K, V> & resMap) const StoreDirConfig & store, typename Inner::WriteConn conn, const std::map<K, V, Compare> & resMap)
{ {
conn.to << resMap.size(); conn.to << resMap.size();
for (auto & i : resMap) { for (auto & i : resMap) {

2
src/libstore/include/nix/store/local-overlay-store.hh
View file

@ -160,7 +160,7 @@ private:
* Check lower store if upper DB does not have. * Check lower store if upper DB does not have.
*/ */
void queryRealisationUncached(const DrvOutput&, void queryRealisationUncached(const DrvOutput&,
Callback<std::shared_ptr<const Realisation>> callback) noexcept override; Callback<std::shared_ptr<const UnkeyedRealisation>> callback) noexcept override;
/** /**
* Call `remountIfNecessary` after collecting garbage normally. * Call `remountIfNecessary` after collecting garbage normally.

6
src/libstore/include/nix/store/local-store.hh
View file

@ -338,10 +338,10 @@ public:
const std::string & outputName, const std::string & outputName,
const StorePath & output); const StorePath & output);
std::optional<const Realisation> queryRealisation_(State & state, const DrvOutput & id); std::optional<const UnkeyedRealisation> queryRealisation_(State & state, const DrvOutput & id);
std::optional<std::pair<int64_t, Realisation>> queryRealisationCore_(State & state, const DrvOutput & id); std::optional<std::pair<int64_t, UnkeyedRealisation>> queryRealisationCore_(State & state, const DrvOutput & id);
void queryRealisationUncached(const DrvOutput&, void queryRealisationUncached(const DrvOutput&,
Callback<std::shared_ptr<const Realisation>> callback) noexcept override; Callback<std::shared_ptr<const UnkeyedRealisation>> callback) noexcept override;
std::optional<std::string> getVersion() override; std::optional<std::string> getVersion() override;

160
src/libstore/include/nix/store/realisation.hh
View file

@ -18,64 +18,97 @@ struct OutputsSpec;
/** /**
* A general `Realisation` key. * A general `Realisation` key.
* *
* This is similar to a `DerivedPath::Opaque`, but the derivation is * This is similar to a `DerivedPath::Built`, except it is only a single
* identified by its "hash modulo" instead of by its store path. * step: `drvPath` is a `StorePath` rather than a `DerivedPath`.
*/ */
struct DrvOutput { struct DrvOutput
{
/** /**
* The hash modulo of the derivation. * The store path to the derivation
*
* Computed from the derivation itself for most types of
* derivations, but computed from the (fixed) content address of the
* output for fixed-output derivations.
*/ */
Hash drvHash; StorePath drvPath;
/** /**
* The name of the output. * The name of the output.
*/ */
OutputName outputName; OutputName outputName;
/**
* Skips the store dir on the `drvPath`
*/
std::string to_string() const; std::string to_string() const;
std::string strHash() const /**
{ return drvHash.to_string(HashFormat::Base16, true); } * Skips the store dir on the `drvPath`
*/
static DrvOutput from_string(std::string_view);
static DrvOutput parse(const std::string &); /**
* Includes the store dir on `drvPath`
*/
std::string render(const StoreDirConfig & store) const;
GENERATE_CMP(DrvOutput, me->drvHash, me->outputName); /**
* Includes the store dir on `drvPath`
*/
static DrvOutput parse(const StoreDirConfig & store, std::string_view);
nlohmann::json toJSON(const StoreDirConfig & store) const;
static DrvOutput fromJSON(const StoreDirConfig & store, const nlohmann::json & json);
bool operator==(const DrvOutput &) const = default;
auto operator<=>(const DrvOutput &) const = default;
}; };
struct Realisation { struct UnkeyedRealisation
DrvOutput id; {
StorePath outPath; StorePath outPath;
StringSet signatures; StringSet signatures;
nlohmann::json toJSON(const StoreDirConfig & store) const;
static UnkeyedRealisation fromJSON(const StoreDirConfig & store, const nlohmann::json & json);
std::string fingerprint(const StoreDirConfig & store, const DrvOutput & key) const;
void sign(const StoreDirConfig & store, const DrvOutput & key, const Signer &);
bool checkSignature(
const StoreDirConfig & store,
const DrvOutput & key,
const PublicKeys & publicKeys,
const std::string & sig) const;
size_t checkSignatures(const StoreDirConfig & store, const DrvOutput & key, const PublicKeys & publicKeys) const;
/** /**
* The realisations that are required for the current one to be valid. * Just check the `outPath`. Signatures don't matter for this.
* * Callers must ensure that the corresponding key is the same for
* When importing this realisation, the store will first check that all its * most use-cases.
* dependencies exist, and map to the correct output path
*/ */
std::map<DrvOutput, StorePath> dependentRealisations; bool isCompatibleWith(const UnkeyedRealisation & other) const
{
return outPath == other.outPath;
}
nlohmann::json toJSON() const; const StorePath & getPath() const
static Realisation fromJSON(const nlohmann::json& json, const std::string& whence); {
return outPath;
}
std::string fingerprint() const; // TODO sketchy that it avoids signatures
void sign(const Signer &); GENERATE_CMP(UnkeyedRealisation, me->outPath);
bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const; };
size_t checkSignatures(const PublicKeys & publicKeys) const;
static std::set<Realisation> closure(Store &, const std::set<Realisation> &); struct Realisation : UnkeyedRealisation
static void closure(Store &, const std::set<Realisation> &, std::set<Realisation> & res); {
DrvOutput id;
bool isCompatibleWith(const Realisation & other) const; nlohmann::json toJSON(const StoreDirConfig & store) const;
static Realisation fromJSON(const StoreDirConfig & store, const nlohmann::json & json);
StorePath getPath() const { return outPath; } bool operator==(const Realisation &) const = default;
auto operator<=>(const Realisation &) const = default;
GENERATE_CMP(Realisation, me->id, me->outPath);
}; };
/** /**
@ -84,7 +117,7 @@ struct Realisation {
* Since these are the outputs of a single derivation, we know the * Since these are the outputs of a single derivation, we know the
* output names are unique so we can use them as the map key. * output names are unique so we can use them as the map key.
*/ */
typedef std::map<OutputName, Realisation> SingleDrvOutputs; typedef std::map<OutputName, UnkeyedRealisation> SingleDrvOutputs;
/** /**
* Collection type for multiple derivations' outputs' `Realisation`s. * Collection type for multiple derivations' outputs' `Realisation`s.
@ -93,30 +126,34 @@ typedef std::map<OutputName, Realisation> SingleDrvOutputs;
* the same, so we need to identify firstly which derivation, and * the same, so we need to identify firstly which derivation, and
* secondly which output of that derivation. * secondly which output of that derivation.
*/ */
typedef std::map<DrvOutput, Realisation> DrvOutputs; typedef std::map<DrvOutput, UnkeyedRealisation> DrvOutputs;
/** /**
* Filter a SingleDrvOutputs to include only specific output names * Filter a SingleDrvOutputs to include only specific output names
* *
* Moves the `outputs` input. * Moves the `outputs` input.
*/ */
SingleDrvOutputs filterDrvOutputs(const OutputsSpec&, SingleDrvOutputs&&); SingleDrvOutputs filterDrvOutputs(const OutputsSpec &, SingleDrvOutputs && outputs);
struct OpaquePath
struct OpaquePath { {
StorePath path; StorePath path;
StorePath getPath() const { return path; } const StorePath & getPath() const
{
return path;
}
GENERATE_CMP(OpaquePath, me->path); bool operator==(const OpaquePath &) const = default;
auto operator<=>(const OpaquePath &) const = default;
}; };
/** /**
* A store path with all the history of how it went into the store * A store path with all the history of how it went into the store
*/ */
struct RealisedPath { struct RealisedPath
/* {
/**
* A path is either the result of the realisation of a derivation or * A path is either the result of the realisation of a derivation or
* an opaque blob that has been directly added to the store * an opaque blob that has been directly added to the store
*/ */
@ -125,33 +162,38 @@ struct RealisedPath {
using Set = std::set<RealisedPath>; using Set = std::set<RealisedPath>;
RealisedPath(StorePath path) : raw(OpaquePath{path}) {} RealisedPath(StorePath path)
RealisedPath(Realisation r) : raw(r) {} : raw(OpaquePath{path})
{
}
RealisedPath(Realisation r)
: raw(r)
{
}
/** /**
* Get the raw store path associated to this * Get the raw store path associated to this
*/ */
StorePath path() const; const StorePath & path() const;
void closure(Store& store, Set& ret) const; bool operator==(const RealisedPath &) const = default;
static void closure(Store& store, const Set& startPaths, Set& ret); auto operator<=>(const RealisedPath &) const = default;
Set closure(Store& store) const;
GENERATE_CMP(RealisedPath, me->raw);
}; };
class MissingRealisation : public Error class MissingRealisation : public Error
{ {
public: public:
MissingRealisation(DrvOutput & outputId) MissingRealisation(const StoreDirConfig & store, DrvOutput & outputId)
: MissingRealisation(outputId.outputName, outputId.strHash()) : MissingRealisation(store, outputId.drvPath, outputId.outputName)
{} {
MissingRealisation(std::string_view drv, OutputName outputName) }
: Error( "cannot operate on output '%s' of the " MissingRealisation(const StoreDirConfig & store, const StorePath & drvPath, const OutputName & outputName);
"unbuilt derivation '%s'", MissingRealisation(
outputName, const StoreDirConfig & store,
drv) const SingleDerivedPath & drvPath,
{} const StorePath & drvPathResolved,
const OutputName & outputName);
}; };
} }

2
src/libstore/include/nix/store/remote-store.hh
View file

@ -112,7 +112,7 @@ struct RemoteStore :
void registerDrvOutput(const Realisation & info) override; void registerDrvOutput(const Realisation & info) override;
void queryRealisationUncached(const DrvOutput &, void queryRealisationUncached(const DrvOutput &,
Callback<std::shared_ptr<const Realisation>> callback) noexcept override; Callback<std::shared_ptr<const UnkeyedRealisation>> callback) noexcept override;
void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override; void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override;

4
src/libstore/include/nix/store/serve-protocol-impl.hh
View file

@ -33,8 +33,8 @@ SERVE_USE_LENGTH_PREFIX_SERIALISER(template<typename... Ts>, std::tuple<Ts...>)
#define SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA , #define SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA ,
SERVE_USE_LENGTH_PREFIX_SERIALISER( SERVE_USE_LENGTH_PREFIX_SERIALISER(
template<typename K SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA typename V>, template<typename K SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA typename V SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA typename Compare>,
std::map<K SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA V>) std::map<K SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA V SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA Compare>)
/** /**
* Use `CommonProto` where possible. * Use `CommonProto` where possible.

15
src/libstore/include/nix/store/serve-protocol.hh
View file

@ -8,7 +8,7 @@ namespace nix {
#define SERVE_MAGIC_1 0x390c9deb #define SERVE_MAGIC_1 0x390c9deb
#define SERVE_MAGIC_2 0x5452eecb #define SERVE_MAGIC_2 0x5452eecb
#define SERVE_PROTOCOL_VERSION (2 << 8 | 7) #define SERVE_PROTOCOL_VERSION (2 << 8 | 8)
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
@ -19,6 +19,9 @@ struct Source;
// items being serialised // items being serialised
struct BuildResult; struct BuildResult;
struct UnkeyedValidPathInfo; struct UnkeyedValidPathInfo;
struct DrvOutput;
struct UnkeyedRealisation;
struct Realisation;
/** /**
@ -174,6 +177,12 @@ inline std::ostream & operator << (std::ostream & s, ServeProto::Command op)
template<> template<>
DECLARE_SERVE_SERIALISER(BuildResult); DECLARE_SERVE_SERIALISER(BuildResult);
template<> template<>
DECLARE_SERVE_SERIALISER(DrvOutput);
template<>
DECLARE_SERVE_SERIALISER(UnkeyedRealisation);
template<>
DECLARE_SERVE_SERIALISER(Realisation);
template<>
DECLARE_SERVE_SERIALISER(UnkeyedValidPathInfo); DECLARE_SERVE_SERIALISER(UnkeyedValidPathInfo);
template<> template<>
DECLARE_SERVE_SERIALISER(ServeProto::BuildOptions); DECLARE_SERVE_SERIALISER(ServeProto::BuildOptions);
@ -186,8 +195,8 @@ DECLARE_SERVE_SERIALISER(std::set<T COMMA_ Compare>);
template<typename... Ts> template<typename... Ts>
DECLARE_SERVE_SERIALISER(std::tuple<Ts...>); DECLARE_SERVE_SERIALISER(std::tuple<Ts...>);
template<typename K, typename V> template<typename K, typename V, typename Compare>
DECLARE_SERVE_SERIALISER(std::map<K COMMA_ V>); DECLARE_SERVE_SERIALISER(std::map<K COMMA_ V COMMA_ Compare>);
#undef COMMA_ #undef COMMA_
} }

13
src/libstore/include/nix/store/store-api.hh
View file

@ -38,6 +38,7 @@ MakeError(SubstituterDisabled, Error);
MakeError(InvalidStoreReference, Error); MakeError(InvalidStoreReference, Error);
struct UnkeyedRealisation;
struct Realisation; struct Realisation;
struct RealisedPath; struct RealisedPath;
struct DrvOutput; struct DrvOutput;
@ -319,13 +320,13 @@ public:
/** /**
* Query the information about a realisation. * Query the information about a realisation.
*/ */
std::shared_ptr<const Realisation> queryRealisation(const DrvOutput &); std::shared_ptr<const UnkeyedRealisation> queryRealisation(const DrvOutput &);
/** /**
* Asynchronous version of queryRealisation(). * Asynchronous version of queryRealisation().
*/ */
void queryRealisation(const DrvOutput &, void queryRealisation(const DrvOutput &,
Callback<std::shared_ptr<const Realisation>> callback) noexcept; Callback<std::shared_ptr<const UnkeyedRealisation>> callback) noexcept;
/** /**
@ -354,7 +355,7 @@ protected:
virtual void queryPathInfoUncached(const StorePath & path, virtual void queryPathInfoUncached(const StorePath & path,
Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept = 0; Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept = 0;
virtual void queryRealisationUncached(const DrvOutput &, virtual void queryRealisationUncached(const DrvOutput &,
Callback<std::shared_ptr<const Realisation>> callback) noexcept = 0; Callback<std::shared_ptr<const UnkeyedRealisation>> callback) noexcept = 0;
public: public:
@ -902,10 +903,4 @@ std::optional<ValidPathInfo> decodeValidPathInfo(
const ContentAddress * getDerivationCA(const BasicDerivation & drv); const ContentAddress * getDerivationCA(const BasicDerivation & drv);
std::map<DrvOutput, StorePath> drvOutputReferences(
Store & store,
const Derivation & drv,
const StorePath & outputPath,
Store * evalStore = nullptr);
} }

4
src/libstore/include/nix/store/worker-protocol-impl.hh
View file

@ -33,8 +33,8 @@ WORKER_USE_LENGTH_PREFIX_SERIALISER(template<typename... Ts>, std::tuple<Ts...>)
#define WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA , #define WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA ,
WORKER_USE_LENGTH_PREFIX_SERIALISER( WORKER_USE_LENGTH_PREFIX_SERIALISER(
template<typename K WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA typename V>, template<typename K WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA typename V WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA typename Compare>,
std::map<K WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA V>) std::map<K WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA V WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA Compare>)
/** /**
* Use `CommonProto` where possible. * Use `CommonProto` where possible.

17
src/libstore/include/nix/store/worker-protocol.hh
View file

@ -13,7 +13,7 @@ namespace nix {
/* Note: you generally shouldn't change the protocol version. Define a /* Note: you generally shouldn't change the protocol version. Define a
new `WorkerProto::Feature` instead. */ new `WorkerProto::Feature` instead. */
#define PROTOCOL_VERSION (1 << 8 | 38) #define PROTOCOL_VERSION (1 << 8 | 39)
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
@ -37,6 +37,9 @@ struct BuildResult;
struct KeyedBuildResult; struct KeyedBuildResult;
struct ValidPathInfo; struct ValidPathInfo;
struct UnkeyedValidPathInfo; struct UnkeyedValidPathInfo;
struct DrvOutput;
struct UnkeyedRealisation;
struct Realisation;
enum BuildMode : uint8_t; enum BuildMode : uint8_t;
enum TrustedFlag : bool; enum TrustedFlag : bool;
@ -263,6 +266,14 @@ DECLARE_WORKER_SERIALISER(ValidPathInfo);
template<> template<>
DECLARE_WORKER_SERIALISER(UnkeyedValidPathInfo); DECLARE_WORKER_SERIALISER(UnkeyedValidPathInfo);
template<> template<>
DECLARE_WORKER_SERIALISER(DrvOutput);
template<>
DECLARE_WORKER_SERIALISER(UnkeyedRealisation);
template<>
DECLARE_WORKER_SERIALISER(Realisation);
template<>
DECLARE_WORKER_SERIALISER(std::optional<UnkeyedRealisation>);
template<>
DECLARE_WORKER_SERIALISER(BuildMode); DECLARE_WORKER_SERIALISER(BuildMode);
template<> template<>
DECLARE_WORKER_SERIALISER(std::optional<TrustedFlag>); DECLARE_WORKER_SERIALISER(std::optional<TrustedFlag>);
@ -279,8 +290,8 @@ DECLARE_WORKER_SERIALISER(std::set<T COMMA_ Compare>);
template<typename... Ts> template<typename... Ts>
DECLARE_WORKER_SERIALISER(std::tuple<Ts...>); DECLARE_WORKER_SERIALISER(std::tuple<Ts...>);
template<typename K, typename V> template<typename K, typename V, typename Compare>
DECLARE_WORKER_SERIALISER(std::map<K COMMA_ V>); DECLARE_WORKER_SERIALISER(std::map<K COMMA_ V COMMA_ Compare>);
#undef COMMA_ #undef COMMA_
} }

8
src/libstore/local-overlay-store.cc
View file

@ -72,7 +72,7 @@ void LocalOverlayStore::registerDrvOutput(const Realisation & info)
// First do queryRealisation on lower layer to populate DB // First do queryRealisation on lower layer to populate DB
auto res = lowerStore->queryRealisation(info.id); auto res = lowerStore->queryRealisation(info.id);
if (res) if (res)
LocalStore::registerDrvOutput(*res); LocalStore::registerDrvOutput({*res, info.id});
LocalStore::registerDrvOutput(info); LocalStore::registerDrvOutput(info);
} }
@ -106,12 +106,12 @@ void LocalOverlayStore::queryPathInfoUncached(const StorePath & path,
void LocalOverlayStore::queryRealisationUncached(const DrvOutput & drvOutput, void LocalOverlayStore::queryRealisationUncached(const DrvOutput & drvOutput,
Callback<std::shared_ptr<const Realisation>> callback) noexcept Callback<std::shared_ptr<const UnkeyedRealisation>> callback) noexcept
{ {
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
LocalStore::queryRealisationUncached(drvOutput, LocalStore::queryRealisationUncached(drvOutput,
{[this, drvOutput, callbackPtr](std::future<std::shared_ptr<const Realisation>> fut) { {[this, drvOutput, callbackPtr](std::future<std::shared_ptr<const UnkeyedRealisation>> fut) {
try { try {
auto info = fut.get(); auto info = fut.get();
if (info) if (info)
@ -121,7 +121,7 @@ void LocalOverlayStore::queryRealisationUncached(const DrvOutput & drvOutput,
} }
// If we don't have it, check lower store // If we don't have it, check lower store
lowerStore->queryRealisation(drvOutput, lowerStore->queryRealisation(drvOutput,
{[callbackPtr](std::future<std::shared_ptr<const Realisation>> fut) { {[callbackPtr](std::future<std::shared_ptr<const UnkeyedRealisation>> fut) {
try { try {
(*callbackPtr)(fut.get()); (*callbackPtr)(fut.get());
} catch (...) { } catch (...) {

71
src/libstore/local-store.cc
View file

@ -100,8 +100,6 @@ struct LocalStore::State::Stmts {
SQLiteStmt QueryAllRealisedOutputs; SQLiteStmt QueryAllRealisedOutputs;
SQLiteStmt QueryPathFromHashPart; SQLiteStmt QueryPathFromHashPart;
SQLiteStmt QueryValidPaths; SQLiteStmt QueryValidPaths;
SQLiteStmt QueryRealisationReferences;
SQLiteStmt AddRealisationReference;
}; };
LocalStore::LocalStore(ref<const Config> config) LocalStore::LocalStore(ref<const Config> config)
@ -363,19 +361,6 @@ LocalStore::LocalStore(ref<const Config> config)
where drvPath = ? where drvPath = ?
; ;
)"); )");
state->stmts->QueryRealisationReferences.create(state->db,
R"(
select drvPath, outputName from Realisations
join RealisationsRefs on realisationReference = Realisations.id
where referrer = ?;
)");
state->stmts->AddRealisationReference.create(state->db,
R"(
insert or replace into RealisationsRefs (referrer, realisationReference)
values (
(select id from Realisations where drvPath = ? and outputName = ?),
(select id from Realisations where drvPath = ? and outputName = ?));
)");
} }
} }
@ -603,7 +588,7 @@ void LocalStore::registerDrvOutput(const Realisation & info)
info.signatures.end()); info.signatures.end());
state->stmts->UpdateRealisedOutput.use() state->stmts->UpdateRealisedOutput.use()
(concatStringsSep(" ", combinedSignatures)) (concatStringsSep(" ", combinedSignatures))
(info.id.strHash()) (info.id.drvPath.to_string())
(info.id.outputName) (info.id.outputName)
.exec(); .exec();
} else { } else {
@ -618,30 +603,12 @@ void LocalStore::registerDrvOutput(const Realisation & info)
} }
} else { } else {
state->stmts->RegisterRealisedOutput.use() state->stmts->RegisterRealisedOutput.use()
(info.id.strHash()) (info.id.drvPath.to_string())
(info.id.outputName) (info.id.outputName)
(printStorePath(info.outPath)) (printStorePath(info.outPath))
(concatStringsSep(" ", info.signatures)) (concatStringsSep(" ", info.signatures))
.exec(); .exec();
} }
for (auto & [outputId, depPath] : info.dependentRealisations) {
auto localRealisation = queryRealisationCore_(*state, outputId);
if (!localRealisation)
throw Error("unable to register the derivation '%s' as it "
"depends on the non existent '%s'",
info.id.to_string(), outputId.to_string());
if (localRealisation->second.outPath != depPath)
throw Error("unable to register the derivation '%s' as it "
"depends on a realisation of '%s' that doesn’t"
"match what we have locally",
info.id.to_string(), outputId.to_string());
state->stmts->AddRealisationReference.use()
(info.id.strHash())
(info.id.outputName)
(outputId.strHash())
(outputId.outputName)
.exec();
}
}); });
} }
@ -1033,7 +1000,7 @@ bool LocalStore::pathInfoIsUntrusted(const ValidPathInfo & info)
bool LocalStore::realisationIsUntrusted(const Realisation & realisation) bool LocalStore::realisationIsUntrusted(const Realisation & realisation)
{ {
return config->requireSigs && !realisation.checkSignatures(getPublicKeys()); return config->requireSigs && !realisation.checkSignatures(*this, realisation.id, getPublicKeys());
} }
void LocalStore::addToStore(const ValidPathInfo & info, Source & source, void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
@ -1580,13 +1547,13 @@ void LocalStore::addSignatures(const StorePath & storePath, const StringSet & si
} }
std::optional<std::pair<int64_t, Realisation>> LocalStore::queryRealisationCore_( std::optional<std::pair<int64_t, UnkeyedRealisation>> LocalStore::queryRealisationCore_(
LocalStore::State & state, LocalStore::State & state,
const DrvOutput & id) const DrvOutput & id)
{ {
auto useQueryRealisedOutput( auto useQueryRealisedOutput(
state.stmts->QueryRealisedOutput.use() state.stmts->QueryRealisedOutput.use()
(id.strHash()) (id.drvPath.to_string())
(id.outputName)); (id.outputName));
if (!useQueryRealisedOutput.next()) if (!useQueryRealisedOutput.next())
return std::nullopt; return std::nullopt;
@ -1597,15 +1564,14 @@ std::optional<std::pair<int64_t, Realisation>> LocalStore::queryRealisationCore_
return {{ return {{
realisationDbId, realisationDbId,
Realisation{ UnkeyedRealisation{
.id = id,
.outPath = outputPath, .outPath = outputPath,
.signatures = signatures, .signatures = signatures,
} }
}}; }};
} }
std::optional<const Realisation> LocalStore::queryRealisation_( std::optional<const UnkeyedRealisation> LocalStore::queryRealisation_(
LocalStore::State & state, LocalStore::State & state,
const DrvOutput & id) const DrvOutput & id)
{ {
@ -1614,38 +1580,21 @@ std::optional<const Realisation> LocalStore::queryRealisation_(
return std::nullopt; return std::nullopt;
auto [realisationDbId, res] = *maybeCore; auto [realisationDbId, res] = *maybeCore;
std::map<DrvOutput, StorePath> dependentRealisations;
auto useRealisationRefs(
state.stmts->QueryRealisationReferences.use()
(realisationDbId));
while (useRealisationRefs.next()) {
auto depId = DrvOutput {
Hash::parseAnyPrefixed(useRealisationRefs.getStr(0)),
useRealisationRefs.getStr(1),
};
auto dependentRealisation = queryRealisationCore_(state, depId);
assert(dependentRealisation); // Enforced by the db schema
auto outputPath = dependentRealisation->second.outPath;
dependentRealisations.insert({depId, outputPath});
}
res.dependentRealisations = dependentRealisations;
return { res }; return { res };
} }
void LocalStore::queryRealisationUncached(const DrvOutput & id, void LocalStore::queryRealisationUncached(const DrvOutput & id,
Callback<std::shared_ptr<const Realisation>> callback) noexcept Callback<std::shared_ptr<const UnkeyedRealisation>> callback) noexcept
{ {
try { try {
auto maybeRealisation auto maybeRealisation
= retrySQLite<std::optional<const Realisation>>([&]() { = retrySQLite<std::optional<const UnkeyedRealisation>>([&]() {
auto state(_state.lock()); auto state(_state.lock());
return queryRealisation_(*state, id); return queryRealisation_(*state, id);
}); });
if (maybeRealisation) if (maybeRealisation)
callback( callback(
std::make_shared<const Realisation>(maybeRealisation.value())); std::make_shared<const UnkeyedRealisation>(maybeRealisation.value()));
else else
callback(nullptr); callback(nullptr);

75
src/libstore/misc.cc
View file

@ -238,16 +238,15 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
// If there are unknown output paths, attempt to find if the // If there are unknown output paths, attempt to find if the
// paths are known to substituters through a realisation. // paths are known to substituters through a realisation.
auto outputHashes = staticOutputHashes(*this, *drv);
knownOutputPaths = true; knownOutputPaths = true;
for (auto [outputName, hash] : outputHashes) { for (auto & [outputName, _] : drv->outputs) {
if (!bfd.outputs.contains(outputName)) if (!bfd.outputs.contains(outputName))
continue; continue;
bool found = false; bool found = false;
for (auto &sub : getDefaultSubstituters()) { for (auto &sub : getDefaultSubstituters()) {
auto realisation = sub->queryRealisation({hash, outputName}); auto realisation = sub->queryRealisation({drvPath, outputName});
if (!realisation) if (!realisation)
continue; continue;
found = true; found = true;
@ -325,72 +324,6 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths)
}}); }});
} }
std::map<DrvOutput, StorePath> drvOutputReferences(
const std::set<Realisation> & inputRealisations,
const StorePathSet & pathReferences)
{
std::map<DrvOutput, StorePath> res;
for (const auto & input : inputRealisations) {
if (pathReferences.count(input.outPath)) {
res.insert({input.id, input.outPath});
}
}
return res;
}
std::map<DrvOutput, StorePath> drvOutputReferences(
Store & store,
const Derivation & drv,
const StorePath & outputPath,
Store * evalStore_)
{
auto & evalStore = evalStore_ ? *evalStore_ : store;
std::set<Realisation> inputRealisations;
std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumRealisations;
accumRealisations = [&](const StorePath & inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
if (!inputNode.value.empty()) {
auto outputHashes =
staticOutputHashes(evalStore, evalStore.readDerivation(inputDrv));
for (const auto & outputName : inputNode.value) {
auto outputHash = get(outputHashes, outputName);
if (!outputHash)
throw Error(
"output '%s' of derivation '%s' isn't realised", outputName,
store.printStorePath(inputDrv));
auto thisRealisation = store.queryRealisation(
DrvOutput{*outputHash, outputName});
if (!thisRealisation)
throw Error(
"output '%s' of derivation '%s' isn’t built", outputName,
store.printStorePath(inputDrv));
inputRealisations.insert(*thisRealisation);
}
}
if (!inputNode.value.empty()) {
auto d = makeConstantStorePathRef(inputDrv);
for (const auto & [outputName, childNode] : inputNode.childMap) {
SingleDerivedPath next = SingleDerivedPath::Built { d, outputName };
accumRealisations(
// TODO deep resolutions for dynamic derivations, issue #8947, would go here.
resolveDerivedPath(store, next, evalStore_),
childNode);
}
}
};
for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map)
accumRealisations(inputDrv, inputNode);
auto info = store.queryPathInfo(outputPath);
return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references);
}
OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_) OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_)
{ {
auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_); auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_);
@ -420,7 +353,7 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd,
OutputPathMap outputs; OutputPathMap outputs;
for (auto & [outputName, outputPathOpt] : outputsOpt) { for (auto & [outputName, outputPathOpt] : outputsOpt) {
if (!outputPathOpt) if (!outputPathOpt)
throw MissingRealisation(bfd.drvPath->to_string(store), outputName); throw MissingRealisation(store, *bfd.drvPath, drvPath, outputName);
auto & outputPath = *outputPathOpt; auto & outputPath = *outputPathOpt;
outputs.insert_or_assign(outputName, outputPath); outputs.insert_or_assign(outputName, outputPath);
} }
@ -444,7 +377,7 @@ StorePath resolveDerivedPath(Store & store, const SingleDerivedPath & req, Store
store.printStorePath(drvPath), bfd.output); store.printStorePath(drvPath), bfd.output);
auto & optPath = outputPaths.at(bfd.output); auto & optPath = outputPaths.at(bfd.output);
if (!optPath) if (!optPath)
throw MissingRealisation(bfd.drvPath->to_string(store), bfd.output); throw MissingRealisation(store, *bfd.drvPath, drvPath, bfd.output);
return *optPath; return *optPath;
}, },
}, req.raw()); }, req.raw());

61
src/libstore/nar-info-disk-cache.cc
View file

@ -44,10 +44,16 @@ create table if not exists NARs (
create table if not exists Realisations ( create table if not exists Realisations (
cache integer not null, cache integer not null,
outputId text not null,
content blob, -- Json serialisation of the realisation, or null if the realisation is absent drvPath text not null,
outputName text not null,
-- The following are null if the realisation is absent
outputPath text,
sigs text,
timestamp integer not null, timestamp integer not null,
primary key (cache, outputId), primary key (cache, drvPath, outputName),
foreign key (cache) references BinaryCaches(id) on delete cascade foreign key (cache) references BinaryCaches(id) on delete cascade
); );
@ -117,22 +123,22 @@ public:
state->insertRealisation.create(state->db, state->insertRealisation.create(state->db,
R"( R"(
insert or replace into Realisations(cache, outputId, content, timestamp) insert or replace into Realisations(cache, drvPath, outputName, outputPath, sigs, timestamp)
values (?, ?, ?, ?) values (?, ?, ?, ?, ?, ?)
)"); )");
state->insertMissingRealisation.create(state->db, state->insertMissingRealisation.create(state->db,
R"( R"(
insert or replace into Realisations(cache, outputId, timestamp) insert or replace into Realisations(cache, drvPath, outputName, timestamp)
values (?, ?, ?) values (?, ?, ?, ?)
)"); )");
state->queryRealisation.create(state->db, state->queryRealisation.create(state->db,
R"( R"(
select content from Realisations select outputPath, sigs from Realisations
where cache = ? and outputId = ? and where cache = ? and drvPath = ? and outputName = ? and
((content is null and timestamp > ?) or ((outputPath is null and timestamp > ?) or
(content is not null and timestamp > ?)) (outputPath is not null and timestamp > ?))
)"); )");
/* Periodically purge expired entries from the database. */ /* Periodically purge expired entries from the database. */
@ -295,22 +301,31 @@ public:
auto queryRealisation(state->queryRealisation.use() auto queryRealisation(state->queryRealisation.use()
(cache.id) (cache.id)
(id.to_string()) (id.drvPath.to_string())
(id.outputName)
(now - settings.ttlNegativeNarInfoCache) (now - settings.ttlNegativeNarInfoCache)
(now - settings.ttlPositiveNarInfoCache)); (now - settings.ttlPositiveNarInfoCache));
if (!queryRealisation.next()) if (!queryRealisation.next())
return {oUnknown, 0}; return {oUnknown, nullptr};
if (queryRealisation.isNull(0)) if (queryRealisation.isNull(0))
return {oInvalid, 0}; return {oInvalid, nullptr};
auto realisation = try {
std::make_shared<Realisation>(Realisation::fromJSON( return {
nlohmann::json::parse(queryRealisation.getStr(0)), oValid,
"Local disk cache")); std::make_shared<Realisation>(
UnkeyedRealisation{
return {oValid, realisation}; .outPath = StorePath{queryRealisation.getStr(0)},
.signatures = nlohmann::json::parse(queryRealisation.getStr(1)),
},
id),
};
} catch (Error & e) {
e.addTrace({}, "reading build trace key-value from the local disk cache");
throw;
}
}); });
} }
@ -365,8 +380,10 @@ public:
state->insertRealisation.use() state->insertRealisation.use()
(cache.id) (cache.id)
(realisation.id.to_string()) (realisation.id.drvPath.to_string())
(realisation.toJSON().dump()) (realisation.id.outputName)
(realisation.outPath.to_string())
(nlohmann::json(realisation.signatures))
(time(0)).exec(); (time(0)).exec();
}); });

218
src/libstore/realisation.cc
View file

@ -1,130 +1,113 @@
#include "nix/store/realisation.hh" #include "nix/store/realisation.hh"
#include "nix/store/store-api.hh" #include "nix/store/store-api.hh"
#include "nix/util/closure.hh"
#include "nix/util/signature/local-keys.hh" #include "nix/util/signature/local-keys.hh"
#include <nlohmann/json.hpp> #include "nix/util/json-utils.hh"
namespace nix { namespace nix {
MakeError(InvalidDerivationOutputId, Error); MakeError(InvalidDerivationOutputId, Error);
DrvOutput DrvOutput::parse(const std::string &strRep) { DrvOutput DrvOutput::parse(const StoreDirConfig & store, std::string_view s)
size_t n = strRep.find("!"); {
if (n == strRep.npos) size_t n = s.rfind('^');
throw InvalidDerivationOutputId("Invalid derivation output id %s", strRep); if (n == s.npos)
throw InvalidDerivationOutputId("Invalid derivation output id '%s': missing '^'", s);
return DrvOutput{ return DrvOutput{
.drvHash = Hash::parseAnyPrefixed(strRep.substr(0, n)), .drvPath = store.parseStorePath(s.substr(0, n)),
.outputName = strRep.substr(n+1), .outputName = OutputName{s.substr(n + 1)},
}; };
} }
std::string DrvOutput::to_string() const { std::string DrvOutput::render(const StoreDirConfig & store) const
return strHash() + "!" + outputName;
}
std::set<Realisation> Realisation::closure(Store & store, const std::set<Realisation> & startOutputs)
{ {
std::set<Realisation> res; return std::string(store.printStorePath(drvPath)) + "^" + outputName;
Realisation::closure(store, startOutputs, res);
return res;
} }
void Realisation::closure(Store & store, const std::set<Realisation> & startOutputs, std::set<Realisation> & res) std::string DrvOutput::to_string() const
{ {
auto getDeps = [&](const Realisation& current) -> std::set<Realisation> { return std::string(drvPath.to_string()) + "^" + outputName;
std::set<Realisation> res;
for (auto& [currentDep, _] : current.dependentRealisations) {
if (auto currentRealisation = store.queryRealisation(currentDep))
res.insert(*currentRealisation);
else
throw Error(
"Unrealised derivation '%s'", currentDep.to_string());
}
return res;
};
computeClosure<Realisation>(
startOutputs, res,
[&](const Realisation& current,
std::function<void(std::promise<std::set<Realisation>>&)>
processEdges) {
std::promise<std::set<Realisation>> promise;
try {
auto res = getDeps(current);
promise.set_value(res);
} catch (...) {
promise.set_exception(std::current_exception());
}
return processEdges(promise);
});
} }
nlohmann::json Realisation::toJSON() const { nlohmann::json DrvOutput::toJSON(const StoreDirConfig & store) const
auto jsonDependentRealisations = nlohmann::json::object(); {
for (auto & [depId, depOutPath] : dependentRealisations)
jsonDependentRealisations.emplace(depId.to_string(), depOutPath.to_string());
return nlohmann::json{ return nlohmann::json{
{"id", id.to_string()}, {"drvPath", store.printStorePath(drvPath)},
{"outPath", outPath.to_string()}, {"outputName", outputName},
{"signatures", signatures},
{"dependentRealisations", jsonDependentRealisations},
}; };
} }
Realisation Realisation::fromJSON( DrvOutput DrvOutput::fromJSON(const StoreDirConfig & store, const nlohmann::json & json)
const nlohmann::json& json, {
const std::string& whence) { auto obj = getObject(json);
auto getOptionalField = [&](std::string fieldName) -> std::optional<std::string> {
auto fieldIterator = json.find(fieldName); return {
if (fieldIterator == json.end()) .drvPath = store.parseStorePath(getString(valueAt(obj, "drvPath"))),
return std::nullopt; .outputName = getString(valueAt(obj, "outputName")),
return {*fieldIterator};
}; };
auto getField = [&](std::string fieldName) -> std::string { }
if (auto field = getOptionalField(fieldName))
return *field; nlohmann::json UnkeyedRealisation::toJSON(const StoreDirConfig & store) const
else {
throw Error( return nlohmann::json{
"Drv output info file '%1%' is corrupt, missing field %2%", {"outPath", store.printStorePath(outPath)},
whence, fieldName); {"signatures", signatures},
}; };
}
UnkeyedRealisation UnkeyedRealisation::fromJSON(const StoreDirConfig & store, const nlohmann::json & json)
{
auto obj = getObject(json);
StringSet signatures; StringSet signatures;
if (auto signaturesIterator = json.find("signatures"); signaturesIterator != json.end()) if (auto * signaturesJson = get(obj, "signatures"))
signatures.insert(signaturesIterator->begin(), signaturesIterator->end()); signatures = getStringSet(*signaturesJson);
std::map <DrvOutput, StorePath> dependentRealisations; return {
if (auto jsonDependencies = json.find("dependentRealisations"); jsonDependencies != json.end()) .outPath = store.parseStorePath(getString(valueAt(obj, "outPath"))),
for (auto & [jsonDepId, jsonDepOutPath] : jsonDependencies->get<StringMap>())
dependentRealisations.insert({DrvOutput::parse(jsonDepId), StorePath(jsonDepOutPath)});
return Realisation{
.id = DrvOutput::parse(getField("id")),
.outPath = StorePath(getField("outPath")),
.signatures = signatures, .signatures = signatures,
.dependentRealisations = dependentRealisations,
}; };
} }
std::string Realisation::fingerprint() const nlohmann::json Realisation::toJSON(const StoreDirConfig & store) const
{ {
auto serialized = toJSON(); return nlohmann::json{
serialized.erase("signatures"); {"key", id.toJSON(store)},
return serialized.dump(); {"value", static_cast<const UnkeyedRealisation &>(*this).toJSON(store)},
};
} }
void Realisation::sign(const Signer &signer) Realisation Realisation::fromJSON(const StoreDirConfig & store, const nlohmann::json & json)
{ {
signatures.insert(signer.signDetached(fingerprint())); auto obj = getObject(json);
return {
UnkeyedRealisation::fromJSON(store, valueAt(obj, "key")),
DrvOutput::fromJSON(store, valueAt(obj, "value")),
};
} }
bool Realisation::checkSignature(const PublicKeys & publicKeys, const std::string & sig) const std::string UnkeyedRealisation::fingerprint(const StoreDirConfig & store, const DrvOutput & key) const
{ {
return verifyDetached(fingerprint(), sig, publicKeys); auto serialised = Realisation{*this, key}.toJSON(store);
auto value = serialised.find("value");
assert(value != serialised.end());
value->erase("signatures");
return serialised.dump();
} }
size_t Realisation::checkSignatures(const PublicKeys & publicKeys) const void UnkeyedRealisation::sign(const StoreDirConfig & store, const DrvOutput & key, const Signer & signer)
{
signatures.insert(signer.signDetached(fingerprint(store, key)));
}
bool UnkeyedRealisation::checkSignature(
const StoreDirConfig & store, const DrvOutput & key, const PublicKeys & publicKeys, const std::string & sig) const
{
return verifyDetached(fingerprint(store, key), sig, publicKeys);
}
size_t UnkeyedRealisation::checkSignatures(
const StoreDirConfig & store, const DrvOutput & key, const PublicKeys & publicKeys) const
{ {
// FIXME: Maybe we should return `maxSigs` if the realisation corresponds to // FIXME: Maybe we should return `maxSigs` if the realisation corresponds to
// an input-addressed one − because in that case the drv is enough to check // an input-addressed one − because in that case the drv is enough to check
@ -132,16 +115,15 @@ size_t Realisation::checkSignatures(const PublicKeys & publicKeys) const
size_t good = 0; size_t good = 0;
for (auto & sig : signatures) for (auto & sig : signatures)
if (checkSignature(publicKeys, sig)) if (checkSignature(store, key, publicKeys, sig))
good++; good++;
return good; return good;
} }
SingleDrvOutputs filterDrvOutputs(const OutputsSpec & wanted, SingleDrvOutputs && outputs)
SingleDrvOutputs filterDrvOutputs(const OutputsSpec& wanted, SingleDrvOutputs&& outputs)
{ {
SingleDrvOutputs ret = std::move(outputs); SingleDrvOutputs ret = std::move(outputs);
for (auto it = ret.begin(); it != ret.end(); ) { for (auto it = ret.begin(); it != ret.end();) {
if (!wanted.contains(it->first)) if (!wanted.contains(it->first))
it = ret.erase(it); it = ret.erase(it);
else else
@ -150,53 +132,25 @@ SingleDrvOutputs filterDrvOutputs(const OutputsSpec& wanted, SingleDrvOutputs&&
return ret; return ret;
} }
StorePath RealisedPath::path() const { const StorePath & RealisedPath::path() const
return std::visit([](auto && arg) { return arg.getPath(); }, raw); {
return std::visit([](auto && arg) -> auto & { return arg.getPath(); }, raw);
} }
bool Realisation::isCompatibleWith(const Realisation & other) const MissingRealisation::MissingRealisation(
const StoreDirConfig & store, const StorePath & drvPath, const OutputName & outputName)
: Error("cannot operate on output '%s' of the unbuilt derivation '%s'", outputName, store.printStorePath(drvPath))
{ {
assert (id == other.id);
if (outPath == other.outPath) {
if (dependentRealisations.empty() != other.dependentRealisations.empty()) {
warn(
"Encountered a realisation for '%s' with an empty set of "
"dependencies. This is likely an artifact from an older Nix. "
"I’ll try to fix the realisation if I can",
id.to_string());
return true;
} else if (dependentRealisations == other.dependentRealisations) {
return true;
}
}
return false;
} }
void RealisedPath::closure( MissingRealisation::MissingRealisation(
Store& store, const StoreDirConfig & store,
const RealisedPath::Set& startPaths, const SingleDerivedPath & drvPath,
RealisedPath::Set& ret) const StorePath & drvPathResolved,
const OutputName & outputName)
: MissingRealisation{store, drvPathResolved, outputName}
{ {
// FIXME: This only builds the store-path closure, not the real realisation addTrace({}, "looking up realisation for derivation '%s'", drvPath.to_string(store));
// closure
StorePathSet initialStorePaths, pathsClosure;
for (auto& path : startPaths)
initialStorePaths.insert(path.path());
store.computeFSClosure(initialStorePaths, pathsClosure);
ret.insert(startPaths.begin(), startPaths.end());
ret.insert(pathsClosure.begin(), pathsClosure.end());
}
void RealisedPath::closure(Store& store, RealisedPath::Set & ret) const
{
RealisedPath::closure(store, {*this}, ret);
}
RealisedPath::Set RealisedPath::closure(Store& store) const
{
RealisedPath::Set ret;
closure(store, ret);
return ret;
} }
} // namespace nix } // namespace nix

42
src/libstore/remote-store.cc
View file

@ -602,12 +602,12 @@ void RemoteStore::registerDrvOutput(const Realisation & info)
} }
void RemoteStore::queryRealisationUncached(const DrvOutput & id, void RemoteStore::queryRealisationUncached(const DrvOutput & id,
Callback<std::shared_ptr<const Realisation>> callback) noexcept Callback<std::shared_ptr<const UnkeyedRealisation>> callback) noexcept
{ {
try { try {
auto conn(getConnection()); auto conn(getConnection());
if (GET_PROTOCOL_MINOR(conn->protoVersion) < 27) { if (GET_PROTOCOL_MINOR(conn->protoVersion) < 39) {
warn("the daemon is too old to support content-addressing derivations, please upgrade it to 2.4"); warn("the daemon is too old to support content-addressing derivations, please upgrade it to 2.4");
return callback(nullptr); return callback(nullptr);
} }
@ -616,23 +616,13 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id,
conn->to << id.to_string(); conn->to << id.to_string();
conn.processStderr(); conn.processStderr();
auto real = [&]() -> std::shared_ptr<const Realisation> { callback([&]() -> std::shared_ptr<const UnkeyedRealisation> {
if (GET_PROTOCOL_MINOR(conn->protoVersion) < 31) { auto realisation = WorkerProto::Serialise<std::optional<UnkeyedRealisation>>::read(
auto outPaths = WorkerProto::Serialise<std::set<StorePath>>::read( *this, *conn);
*this, *conn); if (!realisation)
if (outPaths.empty()) return nullptr;
return nullptr; return std::make_shared<const UnkeyedRealisation>(*realisation);
return std::make_shared<const Realisation>(Realisation { .id = id, .outPath = *outPaths.begin() }); }());
} else {
auto realisations = WorkerProto::Serialise<std::set<Realisation>>::read(
*this, *conn);
if (realisations.empty())
return nullptr;
return std::make_shared<const Realisation>(*realisations.begin());
}
}();
callback(std::shared_ptr<const Realisation>(real));
} catch (...) { return callback.rethrow(); } } catch (...) { return callback.rethrow(); }
} }
@ -724,27 +714,19 @@ std::vector<KeyedBuildResult> RemoteStore::buildPathsWithResults(
OutputPathMap outputs; OutputPathMap outputs;
auto drvPath = resolveDerivedPath(*evalStore, *bfd.drvPath); auto drvPath = resolveDerivedPath(*evalStore, *bfd.drvPath);
auto drv = evalStore->readDerivation(drvPath);
const auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
auto built = resolveDerivedPath(*this, bfd, &*evalStore); auto built = resolveDerivedPath(*this, bfd, &*evalStore);
for (auto & [output, outputPath] : built) { for (auto & [output, outputPath] : built) {
auto outputHash = get(outputHashes, output); auto outputId = DrvOutput{ drvPath, output };
if (!outputHash)
throw Error(
"the derivation '%s' doesn't have an output named '%s'",
printStorePath(drvPath), output);
auto outputId = DrvOutput{ *outputHash, output };
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
auto realisation = auto realisation =
queryRealisation(outputId); queryRealisation(outputId);
if (!realisation) if (!realisation)
throw MissingRealisation(outputId); throw MissingRealisation(*this, outputId);
res.builtOutputs.emplace(output, *realisation); res.builtOutputs.emplace(output, *realisation);
} else { } else {
res.builtOutputs.emplace( res.builtOutputs.emplace(
output, output,
Realisation { UnkeyedRealisation {
.id = outputId,
.outPath = outputPath, .outPath = outputPath,
}); });
} }

22
src/libstore/restricted-store.cc
View file

@ -111,7 +111,7 @@ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStor
void registerDrvOutput(const Realisation & info) override; void registerDrvOutput(const Realisation & info) override;
void queryRealisationUncached( void queryRealisationUncached(
const DrvOutput & id, Callback<std::shared_ptr<const Realisation>> callback) noexcept override; const DrvOutput & id, Callback<std::shared_ptr<const UnkeyedRealisation>> callback) noexcept override;
void void
buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override; buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override;
@ -254,7 +254,7 @@ void RestrictedStore::registerDrvOutput(const Realisation & info)
} }
void RestrictedStore::queryRealisationUncached( void RestrictedStore::queryRealisationUncached(
const DrvOutput & id, Callback<std::shared_ptr<const Realisation>> callback) noexcept const DrvOutput & id, Callback<std::shared_ptr<const UnkeyedRealisation>> callback) noexcept
// XXX: This should probably be allowed if the realisation corresponds to // XXX: This should probably be allowed if the realisation corresponds to
// an allowed derivation // an allowed derivation
{ {
@ -290,9 +290,19 @@ std::vector<KeyedBuildResult> RestrictedStore::buildPathsWithResults(
auto results = next->buildPathsWithResults(paths, buildMode); auto results = next->buildPathsWithResults(paths, buildMode);
for (auto & result : results) { for (auto & result : results) {
for (auto & [outputName, output] : result.builtOutputs) { if (auto * pathBuilt = std::get_if<DerivedPathBuilt>(&result.path)) {
newPaths.insert(output.outPath); // TODO ugly extra IO
newRealisations.insert(output); auto drvPath = resolveDerivedPath(*next, *pathBuilt->drvPath);
for (auto & [outputName, output] : result.builtOutputs) {
newPaths.insert(output.outPath);
newRealisations.insert({
output,
{
.drvPath = drvPath,
.outputName = outputName,
}
});
}
} }
} }
@ -300,7 +310,7 @@ std::vector<KeyedBuildResult> RestrictedStore::buildPathsWithResults(
next->computeFSClosure(newPaths, closure); next->computeFSClosure(newPaths, closure);
for (auto & path : closure) for (auto & path : closure)
goal.addDependency(path); goal.addDependency(path);
for (auto & real : Realisation::closure(*next, newRealisations)) for (auto & real : newRealisations)
goal.addedDrvOutputs.insert(real.id); goal.addedDrvOutputs.insert(real.id);
return results; return results;

97
src/libstore/serve-protocol.cc
View file

@ -25,12 +25,11 @@ BuildResult ServeProto::Serialise<BuildResult>::read(const StoreDirConfig & stor
>> status.isNonDeterministic >> status.isNonDeterministic
>> status.startTime >> status.startTime
>> status.stopTime; >> status.stopTime;
if (GET_PROTOCOL_MINOR(conn.version) >= 6) { if (GET_PROTOCOL_MINOR(conn.version) >= 8) {
auto builtOutputs = ServeProto::Serialise<DrvOutputs>::read(store, conn); status.builtOutputs = ServeProto::Serialise<std::map<OutputName, UnkeyedRealisation>>::read(store, conn);
for (auto && [output, realisation] : builtOutputs) } else if (GET_PROTOCOL_MINOR(conn.version) >= 6) {
status.builtOutputs.insert_or_assign( // We no longer support these types of realisations
std::move(output.outputName), (void) ServeProto::Serialise<StringMap>::read(store, conn);
std::move(realisation));
} }
return status; return status;
} }
@ -47,11 +46,11 @@ void ServeProto::Serialise<BuildResult>::write(const StoreDirConfig & store, Ser
<< status.isNonDeterministic << status.isNonDeterministic
<< status.startTime << status.startTime
<< status.stopTime; << status.stopTime;
if (GET_PROTOCOL_MINOR(conn.version) >= 6) { if (GET_PROTOCOL_MINOR(conn.version) >= 8) {
DrvOutputs builtOutputs; ServeProto::write(store, conn, status.builtOutputs);
for (auto & [output, realisation] : status.builtOutputs) } else if (GET_PROTOCOL_MINOR(conn.version) >= 6) {
builtOutputs.insert_or_assign(realisation.id, realisation); // We no longer support these types of realisations
ServeProto::write(store, conn, builtOutputs); ServeProto::write(store, conn, StringMap{});
} }
} }
@ -134,4 +133,80 @@ void ServeProto::Serialise<ServeProto::BuildOptions>::write(const StoreDirConfig
} }
} }
UnkeyedRealisation ServeProto::Serialise<UnkeyedRealisation>::read(const StoreDirConfig & store, ReadConn conn)
{
if (GET_PROTOCOL_MINOR(conn.version) < 39) {
throw Error("daemon protocol %d.%d is too old (< 1.29) to understand build trace",
GET_PROTOCOL_MAJOR(conn.version),
GET_PROTOCOL_MINOR(conn.version));
}
auto outPath = ServeProto::Serialise<StorePath>::read(store, conn);
auto signatures = ServeProto::Serialise<StringSet>::read(store, conn);
return UnkeyedRealisation {
.outPath = std::move(outPath),
.signatures = std::move(signatures),
};
}
void ServeProto::Serialise<UnkeyedRealisation>::write(const StoreDirConfig & store, WriteConn conn, const UnkeyedRealisation & info)
{
if (GET_PROTOCOL_MINOR(conn.version) < 39) {
throw Error("daemon protocol %d.%d is too old (< 1.29) to understand build trace",
GET_PROTOCOL_MAJOR(conn.version),
GET_PROTOCOL_MINOR(conn.version));
}
ServeProto::write(store, conn, info.outPath);
ServeProto::write(store, conn, info.signatures);
}
DrvOutput ServeProto::Serialise<DrvOutput>::read(const StoreDirConfig & store, ReadConn conn)
{
if (GET_PROTOCOL_MINOR(conn.version) < 39) {
throw Error("daemon protocol %d.%d is too old (< 1.29) to understand build trace",
GET_PROTOCOL_MAJOR(conn.version),
GET_PROTOCOL_MINOR(conn.version));
}
auto drvPath = ServeProto::Serialise<StorePath>::read(store, conn);
auto outputName = ServeProto::Serialise<std::string>::read(store, conn);
return DrvOutput {
.drvPath = std::move(drvPath),
.outputName = std::move(outputName),
};
}
void ServeProto::Serialise<DrvOutput>::write(const StoreDirConfig & store, WriteConn conn, const DrvOutput & info)
{
if (GET_PROTOCOL_MINOR(conn.version) < 39) {
throw Error("daemon protocol %d.%d is too old (< 1.29) to understand build trace",
GET_PROTOCOL_MAJOR(conn.version),
GET_PROTOCOL_MINOR(conn.version));
}
ServeProto::write(store, conn, info.drvPath);
ServeProto::write(store, conn, info.outputName);
}
Realisation ServeProto::Serialise<Realisation>::read(const StoreDirConfig & store, ReadConn conn)
{
auto id = ServeProto::Serialise<DrvOutput>::read(store, conn);
auto unkeyed = ServeProto::Serialise<UnkeyedRealisation>::read(store, conn);
return Realisation {
std::move(unkeyed),
std::move(id),
};
}
void ServeProto::Serialise<Realisation>::write(const StoreDirConfig & store, WriteConn conn, const Realisation & info)
{
ServeProto::write(store, conn, info.id);
ServeProto::write(store, conn, static_cast<const UnkeyedRealisation &>(info));
}
} }

63
src/libstore/store-api.cc
View file

@ -479,9 +479,8 @@ std::map<std::string, std::optional<StorePath>> Store::queryPartialDerivationOut
return outputs; return outputs;
auto drv = evalStore.readInvalidDerivation(path); auto drv = evalStore.readInvalidDerivation(path);
auto drvHashes = staticOutputHashes(*this, drv); for (auto & [outputName, _] : drv.outputs) {
for (auto & [outputName, hash] : drvHashes) { auto realisation = queryRealisation(DrvOutput{path, outputName});
auto realisation = queryRealisation(DrvOutput{hash, outputName});
if (realisation) { if (realisation) {
outputs.insert_or_assign(outputName, realisation->outPath); outputs.insert_or_assign(outputName, realisation->outPath);
} else { } else {
@ -500,7 +499,7 @@ OutputPathMap Store::queryDerivationOutputMap(const StorePath & path, Store * ev
OutputPathMap result; OutputPathMap result;
for (auto & [outName, optOutPath] : resp) { for (auto & [outName, optOutPath] : resp) {
if (!optOutPath) if (!optOutPath)
throw MissingRealisation(printStorePath(path), outName); throw MissingRealisation(*this, path, outName);
result.insert_or_assign(outName, *optOutPath); result.insert_or_assign(outName, *optOutPath);
} }
return result; return result;
@ -716,7 +715,7 @@ void Store::queryPathInfo(const StorePath & storePath,
} }
void Store::queryRealisation(const DrvOutput & id, void Store::queryRealisation(const DrvOutput & id,
Callback<std::shared_ptr<const Realisation>> callback) noexcept Callback<std::shared_ptr<const UnkeyedRealisation>> callback) noexcept
{ {
try { try {
@ -748,18 +747,18 @@ void Store::queryRealisation(const DrvOutput & id,
queryRealisationUncached( queryRealisationUncached(
id, id,
{ [this, id, callbackPtr]( { [this, id, callbackPtr](
std::future<std::shared_ptr<const Realisation>> fut) { std::future<std::shared_ptr<const UnkeyedRealisation>> fut) {
try { try {
auto info = fut.get(); auto info = fut.get();
if (diskCache) { if (diskCache) {
if (info) if (info)
diskCache->upsertRealisation(getUri(), *info); diskCache->upsertRealisation(getUri(), {*info, id});
else else
diskCache->upsertAbsentRealisation(getUri(), id); diskCache->upsertAbsentRealisation(getUri(), id);
} }
(*callbackPtr)(std::shared_ptr<const Realisation>(info)); (*callbackPtr)(std::shared_ptr<const UnkeyedRealisation>(info));
} catch (...) { } catch (...) {
callbackPtr->rethrow(); callbackPtr->rethrow();
@ -767,9 +766,9 @@ void Store::queryRealisation(const DrvOutput & id,
} }); } });
} }
std::shared_ptr<const Realisation> Store::queryRealisation(const DrvOutput & id) std::shared_ptr<const UnkeyedRealisation> Store::queryRealisation(const DrvOutput & id)
{ {
using RealPtr = std::shared_ptr<const Realisation>; using RealPtr = std::shared_ptr<const UnkeyedRealisation>;
std::promise<RealPtr> promise; std::promise<RealPtr> promise;
queryRealisation(id, queryRealisation(id,
@ -1019,36 +1018,21 @@ std::map<StorePath, StorePath> copyPaths(
SubstituteFlag substitute) SubstituteFlag substitute)
{ {
StorePathSet storePaths; StorePathSet storePaths;
std::set<Realisation> toplevelRealisations; std::vector<const Realisation *> realisations;
for (auto & path : paths) { for (auto & path : paths) {
storePaths.insert(path.path()); storePaths.insert(path.path());
if (auto realisation = std::get_if<Realisation>(&path.raw)) { if (auto * realisation = std::get_if<Realisation>(&path.raw)) {
experimentalFeatureSettings.require(Xp::CaDerivations); experimentalFeatureSettings.require(Xp::CaDerivations);
toplevelRealisations.insert(*realisation); realisations.push_back(realisation);
} }
} }
auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute); auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute);
try { try {
// Copy the realisation closure // Copy the realisation closure
processGraph<Realisation>( for (const auto * realisation : realisations)
Realisation::closure(srcStore, toplevelRealisations), dstStore.registerDrvOutput(*realisation, checkSigs);
[&](const Realisation & current) -> std::set<Realisation> {
std::set<Realisation> children;
for (const auto & [drvOutput, _] : current.dependentRealisations) {
auto currentChild = srcStore.queryRealisation(drvOutput);
if (!currentChild)
throw Error(
"incomplete realisation closure: '%s' is a "
"dependency of '%s' but isn't registered",
drvOutput.to_string(), current.id.to_string());
children.insert(*currentChild);
}
return children;
},
[&](const Realisation& current) -> void {
dstStore.registerDrvOutput(current, checkSigs);
});
} catch (MissingExperimentalFeature & e) { } catch (MissingExperimentalFeature & e) {
// Don't fail if the remote doesn't support CA derivations is it might // Don't fail if the remote doesn't support CA derivations is it might
// not be within our control to change that, and we might still want // not be within our control to change that, and we might still want
@ -1154,8 +1138,19 @@ void copyClosure(
{ {
if (&srcStore == &dstStore) return; if (&srcStore == &dstStore) return;
RealisedPath::Set closure; StorePathSet closure0;
RealisedPath::closure(srcStore, paths, closure); for (auto & path : paths) {
if (auto * opaquePath = std::get_if<OpaquePath>(&path.raw)) {
closure0.insert(opaquePath->path);
}
}
StorePathSet closure1;
srcStore.computeFSClosure(closure0, closure1);
RealisedPath::Set closure = paths;
for (auto && path : closure1)
closure.insert({std::move(path)});
copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute); copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute);
} }
@ -1300,7 +1295,7 @@ void Store::signRealisation(Realisation & realisation)
for (auto & secretKeyFile : secretKeyFiles.get()) { for (auto & secretKeyFile : secretKeyFiles.get()) {
SecretKey secretKey(readFile(secretKeyFile)); SecretKey secretKey(readFile(secretKeyFile));
LocalSigner signer(std::move(secretKey)); LocalSigner signer(std::move(secretKey));
realisation.sign(signer); realisation.sign(*this, realisation.id, signer);
} }
} }

10
src/libstore/unix/build/derivation-builder.cc
View file

@ -2886,11 +2886,13 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs()
auto oldinfo = get(initialOutputs, outputName); auto oldinfo = get(initialOutputs, outputName);
assert(oldinfo); assert(oldinfo);
auto thisRealisation = Realisation { auto thisRealisation = Realisation {
.id = DrvOutput { {
oldinfo->outputHash, .outPath = newInfo.path,
outputName },
DrvOutput {
.drvPath = drvPath,
.outputName = outputName,
}, },
.outPath = newInfo.path
}; };
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)
&& !drv.type().isImpure()) && !drv.type().isImpure())

117
src/libstore/worker-protocol.cc
View file

@ -195,10 +195,11 @@ void WorkerProto::Serialise<BuildResult>::write(const StoreDirConfig & store, Wo
WorkerProto::write(store, conn, res.cpuSystem); WorkerProto::write(store, conn, res.cpuSystem);
} }
if (GET_PROTOCOL_MINOR(conn.version) >= 28) { if (GET_PROTOCOL_MINOR(conn.version) >= 28) {
DrvOutputs builtOutputs; // Don't support those types of realisations anymore.
for (auto & [output, realisation] : res.builtOutputs) WorkerProto::write(store, conn, StringMap{});
builtOutputs.insert_or_assign(realisation.id, realisation); }
WorkerProto::write(store, conn, builtOutputs); if (GET_PROTOCOL_MINOR(conn.version) >= 39) {
WorkerProto::write(store, conn, res.builtOutputs);
} }
} }
@ -260,7 +261,7 @@ WorkerProto::ClientHandshakeInfo WorkerProto::Serialise<WorkerProto::ClientHands
} }
if (GET_PROTOCOL_MINOR(conn.version) >= 35) { if (GET_PROTOCOL_MINOR(conn.version) >= 35) {
res.remoteTrustsUs = WorkerProto::Serialise<std::optional< TrustedFlag>>::read(store, conn); res.remoteTrustsUs = WorkerProto::Serialise<std::optional<TrustedFlag>>::read(store, conn);
} else { } else {
// We don't know the answer; protocol to old. // We don't know the answer; protocol to old.
res.remoteTrustsUs = std::nullopt; res.remoteTrustsUs = std::nullopt;
@ -281,4 +282,110 @@ void WorkerProto::Serialise<WorkerProto::ClientHandshakeInfo>::write(const Store
} }
} }
UnkeyedRealisation WorkerProto::Serialise<UnkeyedRealisation>::read(const StoreDirConfig & store, ReadConn conn)
{
if (GET_PROTOCOL_MINOR(conn.version) < 39) {
throw Error("daemon protocol %d.%d is too old (< 1.29) to understand build trace",
GET_PROTOCOL_MAJOR(conn.version),
GET_PROTOCOL_MINOR(conn.version));
}
auto outPath = WorkerProto::Serialise<StorePath>::read(store, conn);
auto signatures = WorkerProto::Serialise<StringSet>::read(store, conn);
return UnkeyedRealisation {
.outPath = std::move(outPath),
.signatures = std::move(signatures),
};
}
void WorkerProto::Serialise<UnkeyedRealisation>::write(const StoreDirConfig & store, WriteConn conn, const UnkeyedRealisation & info)
{
if (GET_PROTOCOL_MINOR(conn.version) < 39) {
throw Error("daemon protocol %d.%d is too old (< 1.29) to understand build trace",
GET_PROTOCOL_MAJOR(conn.version),
GET_PROTOCOL_MINOR(conn.version));
}
WorkerProto::write(store, conn, info.outPath);
WorkerProto::write(store, conn, info.signatures);
}
std::optional<UnkeyedRealisation> WorkerProto::Serialise<std::optional<UnkeyedRealisation>>::read(const StoreDirConfig & store, ReadConn conn)
{
if (GET_PROTOCOL_MINOR(conn.version) < 39) {
// Hack to improve compat
(void) WorkerProto::Serialise<std::string>::read(store, conn);
return std::nullopt;
} else {
auto temp = readNum<uint8_t>(conn.from);
switch (temp) {
case 0:
return std::nullopt;
case 1:
return WorkerProto::Serialise<UnkeyedRealisation>::read(store, conn);
default:
throw Error("Invalid optional build trace from remote");
}
}
}
void WorkerProto::Serialise<std::optional<UnkeyedRealisation>>::write(const StoreDirConfig & store, WriteConn conn, const std::optional<UnkeyedRealisation> & info)
{
if (!info) {
conn.to << uint8_t{0};
} else {
conn.to << uint8_t{1};
WorkerProto::write(store, conn, *info);
}
}
DrvOutput WorkerProto::Serialise<DrvOutput>::read(const StoreDirConfig & store, ReadConn conn)
{
if (GET_PROTOCOL_MINOR(conn.version) < 39) {
throw Error("daemon protocol %d.%d is too old (< 1.29) to understand build trace",
GET_PROTOCOL_MAJOR(conn.version),
GET_PROTOCOL_MINOR(conn.version));
}
auto drvPath = WorkerProto::Serialise<StorePath>::read(store, conn);
auto outputName = WorkerProto::Serialise<std::string>::read(store, conn);
return DrvOutput {
.drvPath = std::move(drvPath),
.outputName = std::move(outputName),
};
}
void WorkerProto::Serialise<DrvOutput>::write(const StoreDirConfig & store, WriteConn conn, const DrvOutput & info)
{
if (GET_PROTOCOL_MINOR(conn.version) < 39) {
throw Error("daemon protocol %d.%d is too old (< 1.29) to understand build trace",
GET_PROTOCOL_MAJOR(conn.version),
GET_PROTOCOL_MINOR(conn.version));
}
WorkerProto::write(store, conn, info.drvPath);
WorkerProto::write(store, conn, info.outputName);
}
Realisation WorkerProto::Serialise<Realisation>::read(const StoreDirConfig & store, ReadConn conn)
{
auto id = WorkerProto::Serialise<DrvOutput>::read(store, conn);
auto unkeyed = WorkerProto::Serialise<UnkeyedRealisation>::read(store, conn);
return Realisation {
std::move(unkeyed),
std::move(id),
};
}
void WorkerProto::Serialise<Realisation>::write(const StoreDirConfig & store, WriteConn conn, const Realisation & info)
{
WorkerProto::write(store, conn, info.id);
WorkerProto::write(store, conn, static_cast<const UnkeyedRealisation &>(info));
}
} }

10
src/nix/develop.cc
View file

@ -272,16 +272,8 @@ static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore
output.second = DerivationOutput::Deferred { }; output.second = DerivationOutput::Deferred { };
drv.env[output.first] = ""; drv.env[output.first] = "";
} }
auto hashesModulo = hashDerivationModulo(*evalStore, drv, true);
for (auto & output : drv.outputs) { resolveInputAddressed(*evalStore, drv);
Hash h = hashesModulo.hashes.at(output.first);
auto outPath = store->makeOutputPath(output.first, h, drv.name);
output.second = DerivationOutput::InputAddressed {
.path = outPath,
};
drv.env[output.first] = store->printStorePath(outPath);
}
} }
auto shellDrvPath = writeDerivation(*evalStore, drv); auto shellDrvPath = writeDerivation(*evalStore, drv);

2
src/nix/realisation.cc
View file

@ -51,7 +51,7 @@ struct CmdRealisationInfo : BuiltPathsCommand, MixJSON
for (auto & path : realisations) { for (auto & path : realisations) {
nlohmann::json currentPath; nlohmann::json currentPath;
if (auto realisation = std::get_if<Realisation>(&path.raw)) if (auto realisation = std::get_if<Realisation>(&path.raw))
currentPath = realisation->toJSON(); currentPath = realisation->toJSON(*store);
else else
currentPath["opaquePath"] = store->printStorePath(path.path()); currentPath["opaquePath"] = store->printStorePath(path.path());

11
src/perl/lib/Nix/Store.xs
View file

@ -40,7 +40,7 @@ O_OBJECT
} }
else { else {
warn( \"${Package}::$func_name() -- \" warn( \"${Package}::$func_name() -- \"
\"$var not a blessed SV reference\"); \"$var not a blessed SV reference\");
XSRETURN_UNDEF; XSRETURN_UNDEF;
} }
HERE HERE
@ -162,12 +162,15 @@ StoreWrapper::queryPathInfo(char * path, int base32)
} }
SV * SV *
StoreWrapper::queryRawRealisation(char * outputId) StoreWrapper::queryRawRealisation(char * drvPath, char * outputName)
PPCODE: PPCODE:
try { try {
auto realisation = THIS->store->queryRealisation(DrvOutput::parse(outputId)); auto realisation = THIS->store->queryRealisation(DrvOutput{
.drvPath = THIS->store->parseStorePath(drvPath),
.outputName = outputName,
});
if (realisation) if (realisation)
XPUSHs(sv_2mortal(newSVpv(realisation->toJSON().dump().c_str(), 0))); XPUSHs(sv_2mortal(newSVpv(realisation->toJSON(*THIS->store).dump().c_str(), 0)));
else else
XPUSHs(sv_2mortal(newSVpv("", 0))); XPUSHs(sv_2mortal(newSVpv("", 0)));
} catch (Error & e) { } catch (Error & e) {