mirror of
https://github.com/NixOS/nix
synced 2025-06-27 04:21:16 +02:00
Merge remote-tracking branch 'upstream/master' into path-info
This commit is contained in:
John Ericson
commit
f4f3203aa7
303 changed files with 32784 additions and 3308 deletions
|
|
@ -10,6 +10,7 @@
|
|||
#include "references.hh"
|
||||
#include "archive.hh"
|
||||
#include "callback.hh"
|
||||
#include "remote-store.hh"
|
||||
|
||||
#include <regex>
|
||||
|
||||
|
|
@ -258,6 +259,20 @@ StorePath Store::addToStore(const string & name, const Path & _srcPath,
|
|||
}
|
||||
|
||||
|
||||
void Store::addMultipleToStore(
|
||||
Source & source,
|
||||
RepairFlag repair,
|
||||
CheckSigsFlag checkSigs)
|
||||
{
|
||||
auto expected = readNum<uint64_t>(source);
|
||||
for (uint64_t i = 0; i < expected; ++i) {
|
||||
auto info = ValidPathInfo::read(source, *this, 16);
|
||||
info.ultimate = false;
|
||||
addToStore(info, source, repair, checkSigs);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
The aim of this function is to compute in one pass the correct ValidPathInfo for
|
||||
the files that we are trying to add to the store. To accomplish that in one
|
||||
|
|
@ -355,6 +370,13 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
|
|||
return info;
|
||||
}
|
||||
|
||||
StringSet StoreConfig::getDefaultSystemFeatures()
|
||||
{
|
||||
auto res = settings.systemFeatures.get();
|
||||
if (settings.isExperimentalFeatureEnabled("ca-derivations"))
|
||||
res.insert("ca-derivations");
|
||||
return res;
|
||||
}
|
||||
|
||||
Store::Store(const Params & params)
|
||||
: StoreConfig(params)
|
||||
|
|
@ -643,6 +665,42 @@ string Store::makeValidityRegistration(const StorePathSet & paths,
|
|||
}
|
||||
|
||||
|
||||
StorePathSet Store::exportReferences(const StorePathSet & storePaths, const StorePathSet & inputPaths)
|
||||
{
|
||||
StorePathSet paths;
|
||||
|
||||
for (auto & storePath : storePaths) {
|
||||
if (!inputPaths.count(storePath))
|
||||
throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", printStorePath(storePath));
|
||||
|
||||
computeFSClosure({storePath}, paths);
|
||||
}
|
||||
|
||||
/* If there are derivations in the graph, then include their
|
||||
outputs as well. This is useful if you want to do things
|
||||
like passing all build-time dependencies of some path to a
|
||||
derivation that builds a NixOS DVD image. */
|
||||
auto paths2 = paths;
|
||||
|
||||
for (auto & j : paths2) {
|
||||
if (j.isDerivation()) {
|
||||
Derivation drv = derivationFromPath(j);
|
||||
for (auto & k : drv.outputsAndOptPaths(*this)) {
|
||||
if (!k.second.second)
|
||||
/* FIXME: I am confused why we are calling
|
||||
`computeFSClosure` on the output path, rather than
|
||||
derivation itself. That doesn't seem right to me, so I
|
||||
won't try to implemented this for CA derivations. */
|
||||
throw UnimplementedError("exportReferences on CA derivations is not yet implemented");
|
||||
computeFSClosure(*k.second.second, paths);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return paths;
|
||||
}
|
||||
|
||||
|
||||
void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths,
|
||||
bool includeImpureInfo, bool showClosureSize,
|
||||
Base hashBase,
|
||||
|
|
@ -743,31 +801,44 @@ const Store::Stats & Store::getStats()
|
|||
}
|
||||
|
||||
|
||||
void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
|
||||
const StorePath & storePath, RepairFlag repair, CheckSigsFlag checkSigs)
|
||||
static std::string makeCopyPathMessage(
|
||||
std::string_view srcUri,
|
||||
std::string_view dstUri,
|
||||
std::string_view storePath)
|
||||
{
|
||||
auto srcUri = srcStore->getUri();
|
||||
auto dstUri = dstStore->getUri();
|
||||
return srcUri == "local" || srcUri == "daemon"
|
||||
? fmt("copying path '%s' to '%s'", storePath, dstUri)
|
||||
: dstUri == "local" || dstUri == "daemon"
|
||||
? fmt("copying path '%s' from '%s'", storePath, srcUri)
|
||||
: fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri);
|
||||
}
|
||||
|
||||
|
||||
void copyStorePath(
|
||||
Store & srcStore,
|
||||
Store & dstStore,
|
||||
const StorePath & storePath,
|
||||
RepairFlag repair,
|
||||
CheckSigsFlag checkSigs)
|
||||
{
|
||||
auto srcUri = srcStore.getUri();
|
||||
auto dstUri = dstStore.getUri();
|
||||
auto storePathS = srcStore.printStorePath(storePath);
|
||||
Activity act(*logger, lvlInfo, actCopyPath,
|
||||
srcUri == "local" || srcUri == "daemon"
|
||||
? fmt("copying path '%s' to '%s'", srcStore->printStorePath(storePath), dstUri)
|
||||
: dstUri == "local" || dstUri == "daemon"
|
||||
? fmt("copying path '%s' from '%s'", srcStore->printStorePath(storePath), srcUri)
|
||||
: fmt("copying path '%s' from '%s' to '%s'", srcStore->printStorePath(storePath), srcUri, dstUri),
|
||||
{srcStore->printStorePath(storePath), srcUri, dstUri});
|
||||
makeCopyPathMessage(srcUri, dstUri, storePathS),
|
||||
{storePathS, srcUri, dstUri});
|
||||
PushActivity pact(act.id);
|
||||
|
||||
auto info = srcStore->queryPathInfo(storePath);
|
||||
auto info = srcStore.queryPathInfo(storePath);
|
||||
|
||||
uint64_t total = 0;
|
||||
|
||||
// recompute store path on the chance dstStore does it differently
|
||||
if (info->ca && info->references.empty()) {
|
||||
auto info2 = make_ref<ValidPathInfo>(*info);
|
||||
info2->path = dstStore->makeFixedOutputPathFromCA(
|
||||
info2->path = dstStore.makeFixedOutputPathFromCA(
|
||||
info->fullStorePathDescriptorOpt().value());
|
||||
if (dstStore->storeDir == srcStore->storeDir)
|
||||
if (dstStore.storeDir == srcStore.storeDir)
|
||||
assert(info->path == info2->path);
|
||||
info = info2;
|
||||
}
|
||||
|
|
@ -784,32 +855,56 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
|
|||
act.progress(total, info->narSize);
|
||||
});
|
||||
TeeSink tee { sink, progressSink };
|
||||
srcStore->narFromPath(storePath, tee);
|
||||
srcStore.narFromPath(storePath, tee);
|
||||
}, [&]() {
|
||||
throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", srcStore->printStorePath(storePath), srcStore->getUri());
|
||||
throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", srcStore.printStorePath(storePath), srcStore.getUri());
|
||||
});
|
||||
|
||||
dstStore->addToStore(*info, *source, repair, checkSigs);
|
||||
dstStore.addToStore(*info, *source, repair, checkSigs);
|
||||
}
|
||||
|
||||
|
||||
std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, const RealisedPath::Set & paths,
|
||||
RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute)
|
||||
std::map<StorePath, StorePath> copyPaths(
|
||||
Store & srcStore,
|
||||
Store & dstStore,
|
||||
const RealisedPath::Set & paths,
|
||||
RepairFlag repair,
|
||||
CheckSigsFlag checkSigs,
|
||||
SubstituteFlag substitute)
|
||||
{
|
||||
StorePathSet storePaths;
|
||||
std::set<Realisation> realisations;
|
||||
std::set<Realisation> toplevelRealisations;
|
||||
for (auto & path : paths) {
|
||||
storePaths.insert(path.path());
|
||||
if (auto realisation = std::get_if<Realisation>(&path.raw)) {
|
||||
settings.requireExperimentalFeature("ca-derivations");
|
||||
realisations.insert(*realisation);
|
||||
toplevelRealisations.insert(*realisation);
|
||||
}
|
||||
}
|
||||
auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute);
|
||||
|
||||
ThreadPool pool;
|
||||
|
||||
try {
|
||||
for (auto & realisation : realisations) {
|
||||
dstStore->registerDrvOutput(realisation, checkSigs);
|
||||
}
|
||||
// Copy the realisation closure
|
||||
processGraph<Realisation>(
|
||||
pool, Realisation::closure(srcStore, toplevelRealisations),
|
||||
[&](const Realisation & current) -> std::set<Realisation> {
|
||||
std::set<Realisation> children;
|
||||
for (const auto & [drvOutput, _] : current.dependentRealisations) {
|
||||
auto currentChild = srcStore.queryRealisation(drvOutput);
|
||||
if (!currentChild)
|
||||
throw Error(
|
||||
"incomplete realisation closure: '%s' is a "
|
||||
"dependency of '%s' but isn't registered",
|
||||
drvOutput.to_string(), current.id.to_string());
|
||||
children.insert(*currentChild);
|
||||
}
|
||||
return children;
|
||||
},
|
||||
[&](const Realisation& current) -> void {
|
||||
dstStore.registerDrvOutput(current, checkSigs);
|
||||
});
|
||||
} catch (MissingExperimentalFeature & e) {
|
||||
// Don't fail if the remote doesn't support CA derivations is it might
|
||||
// not be within our control to change that, and we might still want
|
||||
|
|
@ -823,10 +918,15 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
|
|||
return pathsMap;
|
||||
}
|
||||
|
||||
std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths,
|
||||
RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute)
|
||||
std::map<StorePath, StorePath> copyPaths(
|
||||
Store & srcStore,
|
||||
Store & dstStore,
|
||||
const StorePathSet & storePaths,
|
||||
RepairFlag repair,
|
||||
CheckSigsFlag checkSigs,
|
||||
SubstituteFlag substitute)
|
||||
{
|
||||
auto valid = dstStore->queryValidPaths(storePaths, substitute);
|
||||
auto valid = dstStore.queryValidPaths(storePaths, substitute);
|
||||
|
||||
StorePathSet missing;
|
||||
for (auto & path : storePaths)
|
||||
|
|
@ -836,9 +936,31 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
|
|||
for (auto & path : storePaths)
|
||||
pathsMap.insert_or_assign(path, path);
|
||||
|
||||
|
||||
Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
|
||||
|
||||
auto sorted = srcStore.topoSortPaths(missing);
|
||||
std::reverse(sorted.begin(), sorted.end());
|
||||
|
||||
auto source = sinkToSource([&](Sink & sink) {
|
||||
sink << sorted.size();
|
||||
for (auto & storePath : sorted) {
|
||||
auto srcUri = srcStore.getUri();
|
||||
auto dstUri = dstStore.getUri();
|
||||
auto storePathS = srcStore.printStorePath(storePath);
|
||||
Activity act(*logger, lvlInfo, actCopyPath,
|
||||
makeCopyPathMessage(srcUri, dstUri, storePathS),
|
||||
{storePathS, srcUri, dstUri});
|
||||
PushActivity pact(act.id);
|
||||
|
||||
auto info = srcStore.queryPathInfo(storePath);
|
||||
info->write(sink, srcStore, 16);
|
||||
srcStore.narFromPath(storePath, sink);
|
||||
}
|
||||
});
|
||||
|
||||
dstStore.addMultipleToStore(*source, repair, checkSigs);
|
||||
|
||||
#if 0
|
||||
std::atomic<size_t> nrDone{0};
|
||||
std::atomic<size_t> nrFailed{0};
|
||||
std::atomic<uint64_t> bytesExpected{0};
|
||||
|
|
@ -854,19 +976,22 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
|
|||
StorePathSet(missing.begin(), missing.end()),
|
||||
|
||||
[&](const StorePath & storePath) {
|
||||
auto info = srcStore->queryPathInfo(storePath);
|
||||
auto info = srcStore.queryPathInfo(storePath);
|
||||
auto storePathForDst = storePath;
|
||||
if (info->ca && info->references.empty()) {
|
||||
storePathForDst = dstStore->makeFixedOutputPathFromCA(
|
||||
storePathForDst = dstStore.makeFixedOutputPathFromCA(
|
||||
info->fullStorePathDescriptorOpt().value());
|
||||
if (dstStore->storeDir == srcStore->storeDir)
|
||||
if (dstStore.storeDir == srcStore.storeDir)
|
||||
assert(storePathForDst == storePath);
|
||||
if (storePathForDst != storePath)
|
||||
debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri());
|
||||
debug("replaced path '%s' to '%s' for substituter '%s'",
|
||||
srcStore.printStorePath(storePath),
|
||||
dstStore.printStorePath(storePathForDst),
|
||||
dstStore.getUri());
|
||||
}
|
||||
pathsMap.insert_or_assign(storePath, storePathForDst);
|
||||
|
||||
if (dstStore->isValidPath(storePath)) {
|
||||
if (dstStore.isValidPath(storePath)) {
|
||||
nrDone++;
|
||||
showProgress();
|
||||
return StorePathSet();
|
||||
|
|
@ -881,20 +1006,23 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
|
|||
[&](const StorePath & storePath) {
|
||||
checkInterrupt();
|
||||
|
||||
auto info = srcStore->queryPathInfo(storePath);
|
||||
auto info = srcStore.queryPathInfo(storePath);
|
||||
|
||||
auto storePathForDst = storePath;
|
||||
if (info->ca && info->references.empty()) {
|
||||
storePathForDst = dstStore->makeFixedOutputPathFromCA(
|
||||
storePathForDst = dstStore.makeFixedOutputPathFromCA(
|
||||
info->fullStorePathDescriptorOpt().value());
|
||||
if (dstStore->storeDir == srcStore->storeDir)
|
||||
assert(storePathForDst == storePath);
|
||||
if (storePathForDst != storePath)
|
||||
debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri());
|
||||
debug("replaced path '%s' to '%s' for substituter '%s'",
|
||||
srcStore.printStorePath(storePath),
|
||||
dstStore.printStorePath(storePathForDst),
|
||||
dstStore.getUri());
|
||||
}
|
||||
pathsMap.insert_or_assign(storePath, storePathForDst);
|
||||
|
||||
if (!dstStore->isValidPath(storePathForDst)) {
|
||||
if (!dstStore.isValidPath(storePathForDst)) {
|
||||
MaintainCount<decltype(nrRunning)> mc(nrRunning);
|
||||
showProgress();
|
||||
try {
|
||||
|
|
@ -903,7 +1031,7 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
|
|||
nrFailed++;
|
||||
if (!settings.keepGoing)
|
||||
throw e;
|
||||
logger->log(lvlError, fmt("could not copy %s: %s", dstStore->printStorePath(storePath), e.what()));
|
||||
logger->log(lvlError, fmt("could not copy %s: %s", dstStore.printStorePath(storePath), e.what()));
|
||||
showProgress();
|
||||
return;
|
||||
}
|
||||
|
|
@ -912,9 +1040,27 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
|
|||
nrDone++;
|
||||
showProgress();
|
||||
});
|
||||
#endif
|
||||
|
||||
return pathsMap;
|
||||
}
|
||||
|
||||
void copyClosure(
|
||||
Store & srcStore,
|
||||
Store & dstStore,
|
||||
const RealisedPath::Set & paths,
|
||||
RepairFlag repair,
|
||||
CheckSigsFlag checkSigs,
|
||||
SubstituteFlag substitute)
|
||||
{
|
||||
if (&srcStore == &dstStore) return;
|
||||
|
||||
RealisedPath::Set closure;
|
||||
RealisedPath::closure(srcStore, paths, closure);
|
||||
|
||||
copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute);
|
||||
}
|
||||
|
||||
std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istream & str, std::optional<HashResult> hashGiven)
|
||||
{
|
||||
std::string path;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue