1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-07-05 20:41:47 +02:00

Add setting to warn about copying/hashing large paths

This is useful for diagnosing whether an evaluation is copying large
paths to the store. Example:

   $ nix build .#packages.x86_64-linux.default --large-path-warning-threshold 1000000
   warning: copied large path '/home/eelco/Dev/nix-master/' to the store (6271792 bytes)
   warning: copied large path '«github:NixOS/nixpkgs/b550fe4b4776908ac2a861124307045f8e717c8e?narHash=sha256-7kkJQd4rZ%2BvFrzWu8sTRtta5D1kBG0LSRYAfhtmMlSo%3D»/' to the store (155263768 bytes)
   warning: copied large path '«github:libgit2/libgit2/45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5?narHash=sha256-oX4Z3S9WtJlwvj0uH9HlYcWv%2Bx1hqp8mhXl7HsLu2f0%3D»/' to the store (22175416 bytes)
   warning: copied large path '/nix/store/z985088mcd6w23qwdlirsinnyzayagki-source' to the store (5885872 bytes)
This commit is contained in:
Eelco Dolstra 2024-05-07 19:25:44 +02:00
parent d8559cad8d
commit dbe1b51580
9 changed files with 54 additions and 15 deletions

View file

@ -112,17 +112,19 @@ HashResult hashPath(
}
Hash hashPath(
std::pair<Hash, std::optional<uint64_t>> hashPath(
const SourcePath & path,
FileIngestionMethod method, HashAlgorithm ht,
PathFilter & filter)
{
switch (method) {
case FileIngestionMethod::Flat:
case FileIngestionMethod::Recursive:
return hashPath(path, (FileSerialisationMethod) method, ht, filter).first;
case FileIngestionMethod::Recursive: {
auto res = hashPath(path, (FileSerialisationMethod) method, ht, filter);
return {res.first, {res.second}};
}
case FileIngestionMethod::Git:
return git::dumpHash(ht, path, filter).hash;
return {git::dumpHash(ht, path, filter).hash, std::nullopt};
}
assert(false);
}

View file

@ -132,14 +132,15 @@ std::string_view renderFileIngestionMethod(FileIngestionMethod method);
/**
* Compute the hash of the given file system object according to the
* given method.
* given method, and for some ingestion methods, the size of the
* serialisation.
*
* Unlike the other `hashPath`, this works on an arbitrary
* `FileIngestionMethod` instead of `FileSerialisationMethod`, but
* doesn't return the size as this is this is not a both simple and
* may not return the size as this is this is not a both simple and
* useful defined for a merkle format.
*/
Hash hashPath(
std::pair<Hash, std::optional<uint64_t>> hashPath(
const SourcePath & path,
FileIngestionMethod method, HashAlgorithm ha,
PathFilter & filter = defaultPathFilter);

View file

@ -283,6 +283,26 @@ struct LengthSink : Sink
}
};
/**
* A wrapper source that counts the number of bytes read from it.
*/
struct LengthSource : Source
{
Source & next;
LengthSource(Source & next) : next(next)
{ }
uint64_t total = 0;
size_t read(char * data, size_t len) override
{
auto n = next.read(data, len);
total += n;
return n;
}
};
/**
* Convert a function into a sink.
*/