wroclaw-git-backups/nix
1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-06-25 14:51:16 +02:00
Code Activity

Low-latency closure copy

Browse source 
This adds a new store operation 'addMultipleToStore' that reads a
number of NARs and ValidPathInfos from a Source, allowing any number
of store paths to be copied in a single call. This is much faster on
high-latency links when copying a lot of small files, like .drv
closures.

For example, on a connection with an 50 ms delay:

Before:

  $ nix copy --to 'unix:///tmp/proxy-socket?root=/tmp/dest-chroot' \
    /nix/store/90jjw94xiyg5drj70whm9yll6xjj0ca9-hello-2.10.drv \
    --derivation --no-check-sigs
  real    0m57.868s
  user    0m0.103s
  sys     0m0.056s

After:

  real    0m0.690s
  user    0m0.017s
  sys     0m0.011s
This commit is contained in:
Eelco Dolstra 2021-07-26 13:31:09 +02:00
parent 9957315ce0
commit fe1f34fa60
8 changed files with 162 additions and 90 deletions
Download patch file Download diff file Expand all files Collapse all files

73
src/libstore/store-api.cc
View file

@ -250,6 +250,20 @@ StorePath Store::addToStore(const string & name, const Path & _srcPath,
}
void Store::addMultipleToStore(
Source & source,
RepairFlag repair,
CheckSigsFlag checkSigs)
{
auto expected = readNum<uint64_t>(source);
for (uint64_t i = 0; i < expected; ++i) {
auto info = ValidPathInfo::read(source, *this, 16);
info.ultimate = false;
addToStore(info, source, repair, checkSigs);
}
}
/*
The aim of this function is to compute in one pass the correct ValidPathInfo for
the files that we are trying to add to the store. To accomplish that in one
@ -771,6 +785,19 @@ const Store::Stats & Store::getStats()
}
static std::string makeCopyPathMessage(
std::string_view srcUri,
std::string_view dstUri,
std::string_view storePath)
{
return srcUri == "local" || srcUri == "daemon"
? fmt("copying path '%s' to '%s'", storePath, dstUri)
: dstUri == "local" || dstUri == "daemon"
? fmt("copying path '%s' from '%s'", storePath, srcUri)
: fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri);
}
void copyStorePath(
Store & srcStore,
Store & dstStore,
@ -780,14 +807,10 @@ void copyStorePath(
{
auto srcUri = srcStore.getUri();
auto dstUri = dstStore.getUri();
auto storePathS = srcStore.printStorePath(storePath);
Activity act(*logger, lvlInfo, actCopyPath,
srcUri == "local" || srcUri == "daemon"
? fmt("copying path '%s' to '%s'", srcStore.printStorePath(storePath), dstUri)
: dstUri == "local" || dstUri == "daemon"
? fmt("copying path '%s' from '%s'", srcStore.printStorePath(storePath), srcUri)
: fmt("copying path '%s' from '%s' to '%s'", srcStore.printStorePath(storePath), srcUri, dstUri),
{srcStore.printStorePath(storePath), srcUri, dstUri});
makeCopyPathMessage(srcUri, dstUri, storePathS),
{storePathS, srcUri, dstUri});
PushActivity pact(act.id);
auto info = srcStore.queryPathInfo(storePath);
@ -896,19 +919,31 @@ std::map<StorePath, StorePath> copyPaths(
for (auto & path : storePaths)
pathsMap.insert_or_assign(path, path);
// FIXME: Temporary hack to copy closures in a single round-trip.
if (dynamic_cast<RemoteStore *>(&dstStore)) {
if (!missing.empty()) {
auto source = sinkToSource([&](Sink & sink) {
srcStore.exportPaths(missing, sink);
});
dstStore.importPaths(*source, NoCheckSigs);
}
return pathsMap;
}
Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
auto sorted = srcStore.topoSortPaths(missing);
std::reverse(sorted.begin(), sorted.end());
auto source = sinkToSource([&](Sink & sink) {
sink << sorted.size();
for (auto & storePath : sorted) {
auto srcUri = srcStore.getUri();
auto dstUri = dstStore.getUri();
auto storePathS = srcStore.printStorePath(storePath);
Activity act(*logger, lvlInfo, actCopyPath,
makeCopyPathMessage(srcUri, dstUri, storePathS),
{storePathS, srcUri, dstUri});
PushActivity pact(act.id);
auto info = srcStore.queryPathInfo(storePath);
info->write(sink, srcStore, 16);
srcStore.narFromPath(storePath, sink);
}
});
dstStore.addMultipleToStore(*source, repair, checkSigs);
#if 0
std::atomic<size_t> nrDone{0};
std::atomic<size_t> nrFailed{0};
std::atomic<uint64_t> bytesExpected{0};
@ -986,6 +1021,8 @@ std::map<StorePath, StorePath> copyPaths(
nrDone++;
showProgress();
});
#endif
return pathsMap;
}