Merge pull request #12676 from silvanshade/blake3-tbb

Implement memory-mapped IO and multi-threading for BLAKE3 hashing

flake.lock

@@ -63,11 +63,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1745391562,
+        "lastModified": 1746141548,
-        "narHash": "sha256-sPwcCYuiEopaafePqlG826tBhctuJsLx/mhKKM5Fmjo=",
+        "narHash": "sha256-IgBWhX7A2oJmZFIrpRuMnw5RAufVnfvOgHWgIdds+hc=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "8a2f738d9d1f1d986b5a4cd2fd2061a7127237d7",
+        "rev": "f02fddb8acef29a8b32f10a335d44828d7825b78",
         "type": "github"
       },
       "original": {

packaging/dependencies.nix

@@ -61,6 +61,7 @@ scope: {
         "--with-container"
         "--with-context"
         "--with-coroutine"
+        "--with-iostreams"
       ];
     }).overrideAttrs
       (old: {

src/libutil/file-system.cc

@@ -21,6 +21,8 @@
 #include <sys/time.h>
 #include <unistd.h>
 
+#include <boost/iostreams/device/mapped_file.hpp>
+
 #ifdef _WIN32
 # include <io.h>
 #endif
@@ -273,9 +275,22 @@ std::string readFile(const std::filesystem::path & path)
     return readFile(os_string_to_string(PathViewNG { path }));
 }
 
-
+void readFile(const Path & path, Sink & sink, bool memory_map)
-void readFile(const Path & path, Sink & sink)
 {
+    // Memory-map the file for faster processing where possible.
+    if (memory_map) {
+        try {
+            boost::iostreams::mapped_file_source mmap(path);
+            if (mmap.is_open()) {
+                sink({mmap.data(), mmap.size()});
+                return;
+            }
+        } catch (const boost::exception & e) {
+        }
+        debug("memory-mapping failed for path: %s", path);
+    }
+
+    // Stream the file instead if memory-mapping fails or is disabled.
     AutoCloseFD fd = toDescriptor(open(path.c_str(), O_RDONLY
 // TODO
 #ifndef _WIN32

src/libutil/hash.cc

@@ -307,11 +307,31 @@ static void start(HashAlgorithm ha, Ctx & ctx)
     else if (ha == HashAlgorithm::SHA512) SHA512_Init(&ctx.sha512);
 }
 
+// BLAKE3 data size threshold beyond which parallel hashing with TBB is likely faster.
+//
+// NOTE: This threshold is based on the recommended rule-of-thumb from the official BLAKE3 documentation for typical
+// x86_64 hardware as of 2025. In the future it may make sense to allow the user to tune this through nix.conf.
+const size_t blake3TbbThreshold = 128000;
+
+// Decide which BLAKE3 update strategy to use based on some heuristics. Currently this just checks the data size but in
+// the future it might also take into consideration available system resources or the presence of a shared-memory
+// capable GPU for a heterogenous compute implementation.
+void blake3_hasher_update_with_heuristics(blake3_hasher * blake3, std::string_view data)
+{
+#ifdef BLAKE3_USE_TBB
+    if (data.size() >= blake3TbbThreshold) {
+        blake3_hasher_update_tbb(blake3, data.data(), data.size());
+    } else
+#endif
+    {
+        blake3_hasher_update(blake3, data.data(), data.size());
+    }
+}
 
 static void update(HashAlgorithm ha, Ctx & ctx,
                    std::string_view data)
 {
-    if (ha == HashAlgorithm::BLAKE3) blake3_hasher_update(&ctx.blake3, data.data(), data.size());
+    if (ha == HashAlgorithm::BLAKE3) blake3_hasher_update_with_heuristics(&ctx.blake3, data);
     else if (ha == HashAlgorithm::MD5) MD5_Update(&ctx.md5, data.data(), data.size());
     else if (ha == HashAlgorithm::SHA1) SHA1_Update(&ctx.sha1, data.data(), data.size());
     else if (ha == HashAlgorithm::SHA256) SHA256_Update(&ctx.sha256, data.data(), data.size());

src/libutil/include/nix/util/file-system.hh

@@ -173,7 +173,7 @@ Descriptor openDirectory(const std::filesystem::path & path);
  */
 std::string readFile(const Path & path);
 std::string readFile(const std::filesystem::path & path);
-void readFile(const Path & path, Sink & sink);
+void readFile(const Path & path, Sink & sink, bool memory_map = true);
 
 /**
  * Write a string to a file.

src/libutil/meson.build

@@ -50,13 +50,14 @@ endif
 
 blake3 = dependency(
   'libblake3',
-  version: '>= 1.5.5',
+  version: '>= 1.8.2',
+  method : 'pkg-config',
 )
 deps_private += blake3
 
 boost = dependency(
   'boost',
-  modules : ['context', 'coroutine'],
+  modules : ['context', 'coroutine', 'iostreams'],
   include_type: 'system',
 )
 # boost is a public dependency, but not a pkg-config dependency unfortunately, so we