wroclaw-git-backups/nix
1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-06-25 10:41:16 +02:00
Code Activity

Merge pull request #12676 from silvanshade/blake3-tbb

Browse source 
Implement memory-mapped IO and multi-threading for BLAKE3 hashing
This commit is contained in:
John Ericson 2025-05-04 10:58:53 -04:00 • committed by GitHub
commit 2676ae7ca6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 46 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

22
src/libutil/hash.cc
View file

@ -307,11 +307,31 @@ static void start(HashAlgorithm ha, Ctx & ctx)
else if (ha == HashAlgorithm::SHA512) SHA512_Init(&ctx.sha512);
}
// BLAKE3 data size threshold beyond which parallel hashing with TBB is likely faster.
//
// NOTE: This threshold is based on the recommended rule-of-thumb from the official BLAKE3 documentation for typical
// x86_64 hardware as of 2025. In the future it may make sense to allow the user to tune this through nix.conf.
const size_t blake3TbbThreshold = 128000;
// Decide which BLAKE3 update strategy to use based on some heuristics. Currently this just checks the data size but in
// the future it might also take into consideration available system resources or the presence of a shared-memory
// capable GPU for a heterogenous compute implementation.
void blake3_hasher_update_with_heuristics(blake3_hasher * blake3, std::string_view data)
{
#ifdef BLAKE3_USE_TBB
if (data.size() >= blake3TbbThreshold) {
blake3_hasher_update_tbb(blake3, data.data(), data.size());
} else
#endif
{
blake3_hasher_update(blake3, data.data(), data.size());
}
}
static void update(HashAlgorithm ha, Ctx & ctx,
std::string_view data)
{
if (ha == HashAlgorithm::BLAKE3) blake3_hasher_update(&ctx.blake3, data.data(), data.size());
if (ha == HashAlgorithm::BLAKE3) blake3_hasher_update_with_heuristics(&ctx.blake3, data);
else if (ha == HashAlgorithm::MD5) MD5_Update(&ctx.md5, data.data(), data.size());
else if (ha == HashAlgorithm::SHA1) SHA1_Update(&ctx.sha1, data.data(), data.size());
else if (ha == HashAlgorithm::SHA256) SHA256_Update(&ctx.sha256, data.data(), data.size());