wroclaw-git-backups/nix
1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-06-26 11:41:15 +02:00
Code Activity

Make LocalBinaryCacheStore::narFromPath() run in constant memory

Browse source 
This reduces memory consumption of

  nix copy --from file://... --to ~/my-nix /nix/store/95cwv4q54dc6giaqv6q6p4r02ia2km35-blender-2.79

from 514 MiB to 18 MiB for an uncompressed binary cache, and from 192
MiB to 53 MiB for a bzipped binary cache. It may also be faster
because fetching can happen concurrently with decompression/writing.

Continuation of 48662d151b.

Issue https://github.com/NixOS/nix/issues/1681.
This commit is contained in:
Eelco Dolstra 2018-03-27 23:12:31 +02:00
parent 81ea8bd5ce
commit 08ec757726
No known key found for this signature in database
GPG key ID: 8170B4726D7198DE
6 changed files with 65 additions and 31 deletions
Download patch file Download diff file Expand all files Collapse all files

36
src/libstore/binary-cache-store.cc
View file

@ -54,7 +54,15 @@ void BinaryCacheStore::init()
}
}
std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
void BinaryCacheStore::getFile(const std::string & path,
Callback<std::shared_ptr<std::string>> callback)
{
try {
callback(getFile(path));
} catch (...) { callback.rethrow(); }
}
void BinaryCacheStore::getFile(const std::string & path, Sink & sink)
{
std::promise<std::shared_ptr<std::string>> promise;
getFile(path,
@ -65,7 +73,19 @@ std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
promise.set_exception(std::current_exception());
}
}});
return promise.get_future().get();
auto data = promise.get_future().get();
sink((unsigned char *) data->data(), data->size());
}
std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
{
StringSink sink;
try {
getFile(path, sink);
} catch (NoSuchBinaryCacheFile &) {
return nullptr;
}
return sink.s;
}
Path BinaryCacheStore::narInfoFileFor(const Path & storePath)
@ -197,23 +217,21 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink)
{
auto info = queryPathInfo(storePath).cast<const NarInfo>();
auto nar = getFile(info->url);
if (!nar) throw Error(format("file '%s' missing from binary cache") % info->url);
auto source = sinkToSource([this, url{info->url}](Sink & sink) {
getFile(url, sink);
});
stats.narRead++;
stats.narReadCompressedBytes += nar->size();
//stats.narReadCompressedBytes += nar->size(); // FIXME
uint64_t narSize = 0;
StringSource source(*nar);
LambdaSink wrapperSink([&](const unsigned char * data, size_t len) {
sink(data, len);
narSize += len;
});
decompress(info->compression, source, wrapperSink);
decompress(info->compression, *source, wrapperSink);
stats.narReadBytes += narSize;
}