wroclaw-git-backups/nix
1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-06-26 15:51:15 +02:00
Code Activity

BinaryCacheStore: Optionally write a NAR listing

Browse source 
The store parameter "write-nar-listing=1" will cause BinaryCacheStore
to write a file ‘<store-hash>.ls.xz’ for each ‘<store-hash>.narinfo’
added to the binary cache. This file contains an XZ-compressed JSON
file describing the contents of the NAR, excluding the contents of
regular files.

E.g.

  {
    "version": 1,
    "root": {
      "type": "directory",
      "entries": {
        "lib": {
          "type": "directory",
          "entries": {
            "Mcrt1.o": {
              "type": "regular",
              "size": 1288
            },
            "Scrt1.o": {
              "type": "regular",
              "size": 3920
            },
          }
        }
      }
      ...
    }
  }

(The actual file has no indentation.)

This is intended to speed up the NixOS channels programs index
generator [1], since fetching gazillions of large NARs from
cache.nixos.org is currently a bottleneck for updating the regular
(non-small) channel.

[1] https://github.com/NixOS/nixos-channel-scripts/blob/master/generate-programs-index.cc
This commit is contained in:
Eelco Dolstra 2016-10-21 16:50:28 +02:00
parent 307cc8c33d
commit 542ae5c8f8
10 changed files with 81 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

73
src/libstore/binary-cache-store.cc
View file

@ -9,6 +9,7 @@
#include "worker-protocol.hh"
#include "nar-accessor.hh"
#include "nar-info-disk-cache.hh"
#include "json.hh"
#include <chrono>
@ -19,6 +20,7 @@ namespace nix {
BinaryCacheStore::BinaryCacheStore(const Params & params)
: Store(params)
, compression(get(params, "compression", "xz"))
, writeNARListing(get(params, "write-nar-listing", "0") == "1")
{
auto secretKeyFile = get(params, "secret-key", "");
if (secretKeyFile != "")
@ -79,7 +81,7 @@ Path BinaryCacheStore::narInfoFileFor(const Path & storePath)
return storePathToHash(storePath) + ".narinfo";
}
void BinaryCacheStore::addToStore(const ValidPathInfo & info, const std::string & nar,
void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
bool repair, bool dontCheckSigs)
{
if (!repair && isValidPath(info.path)) return;
@ -97,20 +99,73 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const std::string
auto narInfoFile = narInfoFileFor(info.path);
assert(nar.compare(0, narMagic.size(), narMagic) == 0);
assert(nar->compare(0, narMagic.size(), narMagic) == 0);
auto narInfo = make_ref<NarInfo>(info);
narInfo->narSize = nar.size();
narInfo->narHash = hashString(htSHA256, nar);
narInfo->narSize = nar->size();
narInfo->narHash = hashString(htSHA256, *nar);
if (info.narHash && info.narHash != narInfo->narHash)
throw Error(format("refusing to copy corrupted path ‘%1%’ to binary cache") % info.path);
/* Optionally write a JSON file containing a listing of the
contents of the NAR. */
if (writeNARListing) {
std::ostringstream jsonOut;
{
JSONObject jsonRoot(jsonOut);
jsonRoot.attr("version", 1);
auto accessor = makeNarAccessor(nar);
std::function<void(const Path &, JSONPlaceholder &)> recurse;
recurse = [&](const Path & path, JSONPlaceholder & res) {
auto st = accessor->stat(path);
auto obj = res.object();
switch (st.type) {
case FSAccessor::Type::tRegular:
obj.attr("type", "regular");
obj.attr("size", st.fileSize);
if (st.isExecutable)
obj.attr("executable", true);
break;
case FSAccessor::Type::tDirectory:
obj.attr("type", "directory");
{
auto res2 = obj.object("entries");
for (auto & name : accessor->readDirectory(path)) {
auto res3 = res2.placeholder(name);
recurse(path + "/" + name, res3);
}
}
break;
case FSAccessor::Type::tSymlink:
obj.attr("type", "symlink");
obj.attr("target", accessor->readLink(path));
break;
default:
abort();
}
};
{
auto res = jsonRoot.placeholder("root");
recurse("", res);
}
}
upsertFile(storePathToHash(info.path) + ".ls.xz", *compress("xz", jsonOut.str()));
}
/* Compress the NAR. */
narInfo->compression = compression;
auto now1 = std::chrono::steady_clock::now();
auto narCompressed = compress(compression, nar);
auto narCompressed = compress(compression, *nar);
auto now2 = std::chrono::steady_clock::now();
narInfo->fileHash = hashString(htSHA256, *narCompressed);
narInfo->fileSize = narCompressed->size();
@ -118,7 +173,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const std::string
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache")
% narInfo->path % narInfo->narSize
% ((1.0 - (double) narCompressed->size() / nar.size()) * 100.0)
% ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0)
% duration);
/* Atomically write the NAR file. */
@ -132,7 +187,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const std::string
} else
stats.narWriteAverted++;
stats.narWriteBytes += nar.size();
stats.narWriteBytes += nar->size();
stats.narWriteCompressedBytes += narCompressed->size();
stats.narWriteCompressionTimeMs += duration;
@ -231,7 +286,7 @@ Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
ValidPathInfo info;
info.path = makeFixedOutputPath(recursive, h, name);
addToStore(info, *sink.s, repair);
addToStore(info, sink.s, repair);
return info.path;
}
@ -246,7 +301,7 @@ Path BinaryCacheStore::addTextToStore(const string & name, const string & s,
if (repair || !isValidPath(info.path)) {
StringSink sink;
dumpString(s, sink);
addToStore(info, *sink.s, repair);
addToStore(info, sink.s, repair);
}
return info.path;