wroclaw-git-backups/nix
1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-06-28 17:51:15 +02:00
Code Activity

Reduce substitution memory consumption

Browse source 
copyStorePath() now pipes the output of srcStore->narFromPath()
directly into dstStore->addToStore(). The sink used by the former is
converted into a source usable by the latter using
boost::coroutine2. This is based on [1].

This reduces the maximum resident size of

  $ nix build --store ~/my-nix/ /nix/store/b0zlxla7dmy1iwc3g459rjznx59797xy-binutils-2.28.1 --substituters file:///tmp/binary-cache-xz/ --no-require-sigs

from 418592 KiB to 53416 KiB. (The previous commit also reduced the
runtime from ~4.2s to ~3.4s, not sure why.) A further improvement will
be to download files into a Sink.

[1] https://github.com/NixOS/nix/compare/master...Mathnerd314:dump-fix-coroutine#diff-dcbcac55a634031f9cc73707da6e4b18

Issue #1969.
This commit is contained in:
Eelco Dolstra 2018-03-16 20:22:34 +01:00
parent 3e6b194d78
commit 48662d151b
No known key found for this signature in database
GPG key ID: 8170B4726D7198DE
8 changed files with 145 additions and 37 deletions
Download patch file Download diff file Expand all files Collapse all files

49
src/libstore/store-api.cc
View file

@ -590,32 +590,15 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
uint64_t total = 0;
auto progress = [&](size_t len) {
total += len;
act.progress(total, info->narSize);
};
struct MyStringSink : StringSink
{
typedef std::function<void(size_t)> Callback;
Callback callback;
MyStringSink(Callback callback) : callback(callback) { }
void operator () (const unsigned char * data, size_t len) override
{
StringSink::operator ()(data, len);
callback(len);
};
};
MyStringSink sink(progress);
srcStore->narFromPath({storePath}, sink);
// FIXME
#if 0
if (!info->narHash) {
auto info2 = make_ref<ValidPathInfo>(*info);
info2->narHash = hashString(htSHA256, *sink.s);
if (!info->narSize) info2->narSize = sink.s->size();
info = info2;
}
#endif
if (info->ultimate) {
auto info2 = make_ref<ValidPathInfo>(*info);
@ -623,7 +606,16 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
info = info2;
}
dstStore->addToStore(*info, sink.s, repair, checkSigs);
auto source = sinkToSource([&](Sink & sink) {
LambdaSink wrapperSink([&](const unsigned char * data, size_t len) {
sink(data, len);
total += len;
act.progress(total, info->narSize);
});
srcStore->narFromPath({storePath}, wrapperSink);
});
dstStore->addToStore(*info, *source, repair, checkSigs);
}
@ -808,6 +800,21 @@ std::string makeFixedOutputCA(bool recursive, const Hash & hash)
}
void Store::addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor)
{
addToStore(info, make_ref<std::string>(narSource.drain()), repair, checkSigs, accessor);
}
void Store::addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor)
{
StringSource source(*nar);
addToStore(info, source, repair, checkSigs, accessor);
}
}