Merge pull request #12379 from silvanshade/blake3-c

Add BLAKE3 hashing algorithm

doc/manual/source/command-ref/nix-hash.md

@@ -67,7 +67,7 @@ md5sum`.
 - `--type` *hashAlgo*
 
   Use the specified cryptographic hash algorithm, which can be one of
-  `md5`, `sha1`, `sha256`, and `sha512`.
+  `blake3`, `md5`, `sha1`, `sha256`, and `sha512`.
 
 - `--to-base16`
 

doc/manual/source/command-ref/nix-prefetch-url.md

@@ -42,7 +42,7 @@ the path of the downloaded file in the Nix store is also printed.
 - `--type` *hashAlgo*
 
   Use the specified cryptographic hash algorithm,
-  which can be one of `md5`, `sha1`, `sha256`, and `sha512`.
+  which can be one of `blake3`, `md5`, `sha1`, `sha256`, and `sha512`.
   The default is `sha256`.
 
 - `--print-path`

doc/manual/source/language/advanced-attributes.md

@@ -192,7 +192,7 @@ Derivations can declare some infrequently used optional attributes.
     The [`convertHash`](@docroot@/language/builtins.md#builtins-convertHash) function shows how to convert between different encodings, and the [`nix-hash` command](../command-ref/nix-hash.md) has information about obtaining the hash for some contents, as well as converting to and from encodings.
 
     The `outputHashAlgo` attribute specifies the hash algorithm used to compute the hash.
-    It can currently be `"sha1"`, `"sha256"`, `"sha512"`, or `null`.
+    It can currently be `"blake3", "sha1"`, `"sha256"`, `"sha512"`, or `null`.
     `outputHashAlgo` can only be `null` when `outputHash` follows the SRI format.
 
     The `outputHashMode` attribute determines how the hash is computed.

doc/manual/source/protocols/json/derivation.md

@@ -38,6 +38,7 @@ is a JSON object with the following fields:
     For an output which will be [content addresed], the name of the hash algorithm used.
     Valid algorithm strings are:
 
+    - `blake3`
     - `md5`
     - `sha1`
     - `sha256`

src/libcmd/misc-store-flags.cc

@@ -50,7 +50,7 @@ Args::Flag hashAlgo(std::string && longName, HashAlgorithm * ha)
 {
     return Args::Flag {
             .longName = std::move(longName),
-            .description = "Hash algorithm (`md5`, `sha1`, `sha256`, or `sha512`).",
+            .description = "Hash algorithm (`blake3`, `md5`, `sha1`, `sha256`, or `sha512`).",
             .labels = {"hash-algo"},
             .handler = {[ha](std::string s) {
                 *ha = parseHashAlgo(s);
@@ -63,7 +63,7 @@ Args::Flag hashAlgoOpt(std::string && longName, std::optional<HashAlgorithm> * o
 {
     return Args::Flag {
             .longName = std::move(longName),
-            .description = "Hash algorithm (`md5`, `sha1`, `sha256`, or `sha512`). Can be omitted for SRI hashes.",
+            .description = "Hash algorithm (`blake3`, `md5`, `sha1`, `sha256`, or `sha512`). Can be omitted for SRI hashes.",
             .labels = {"hash-algo"},
             .handler = {[oha](std::string s) {
                 *oha = std::optional<HashAlgorithm>{parseHashAlgo(s)};

src/libexpr-tests/error_traces.cc

@@ -1152,7 +1152,7 @@ namespace nix {
 
         ASSERT_TRACE1("hashString \"foo\" \"content\"",
                       UsageError,
-                      HintFmt("unknown hash algorithm '%s', expect 'md5', 'sha1', 'sha256', or 'sha512'", "foo"));
+                      HintFmt("unknown hash algorithm '%s', expect 'blake3', 'md5', 'sha1', 'sha256', or 'sha512'", "foo"));
 
         ASSERT_TRACE2("hashString \"sha256\" {}",
                       TypeError,

src/libutil-tests/hash.cc

@@ -6,10 +6,52 @@
 
 namespace nix {
 
+class BLAKE3HashTest : public virtual ::testing::Test
+{
+public:
+
+    /**
+     * We set these in tests rather than the regular globals so we don't have
+     * to worry about race conditions if the tests run concurrently.
+     */
+    ExperimentalFeatureSettings mockXpSettings;
+
+private:
+
+    void SetUp() override
+    {
+        mockXpSettings.set("experimental-features", "blake3-hashes");
+    }
+};
+
     /* ----------------------------------------------------------------------------
      * hashString
      * --------------------------------------------------------------------------*/
 
+    TEST_F(BLAKE3HashTest, testKnownBLAKE3Hashes1) {
+        // values taken from: https://tools.ietf.org/html/rfc4634
+        auto s = "abc";
+        auto hash = hashString(HashAlgorithm::BLAKE3, s, mockXpSettings);
+        ASSERT_EQ(hash.to_string(HashFormat::Base16, true),
+                "blake3:6437b3ac38465133ffb63b75273a8db548c558465d79db03fd359c6cd5bd9d85");
+    }
+
+    TEST_F(BLAKE3HashTest, testKnownBLAKE3Hashes2) {
+        // values taken from: https://tools.ietf.org/html/rfc4634
+        auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq";
+        auto hash = hashString(HashAlgorithm::BLAKE3, s, mockXpSettings);
+        ASSERT_EQ(hash.to_string(HashFormat::Base16, true),
+                "blake3:c19012cc2aaf0dc3d8e5c45a1b79114d2df42abb2a410bf54be09e891af06ff8");
+    }
+
+    TEST_F(BLAKE3HashTest, testKnownBLAKE3Hashes3) {
+        // values taken from: https://www.ietf.org/archive/id/draft-aumasson-blake3-00.txt
+        auto s = "IETF";
+        auto hash = hashString(HashAlgorithm::BLAKE3, s, mockXpSettings);
+        ASSERT_EQ(hash.to_string(HashFormat::Base16, true),
+                "blake3:83a2de1ee6f4e6ab686889248f4ec0cf4cc5709446a682ffd1cbb4d6165181e2");
+    }
+
     TEST(hashString, testKnownMD5Hashes1) {
         // values taken from: https://tools.ietf.org/html/rfc1321
         auto s1 = "";

src/libutil/experimental-features.cc

@@ -24,7 +24,7 @@ struct ExperimentalFeatureDetails
  * feature, we either have no issue at all if few features are not added
  * at the end of the list, or a proper merge conflict if they are.
  */
-constexpr size_t numXpFeatures = 1 + static_cast<size_t>(Xp::PipeOperators);
+constexpr size_t numXpFeatures = 1 + static_cast<size_t>(Xp::BLAKE3Hashes);
 
 constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails = {{
     {
@@ -302,6 +302,14 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
         )",
         .trackingUrl = "https://github.com/NixOS/nix/milestone/55",
     },
+    {
+        .tag = Xp::BLAKE3Hashes,
+        .name = "blake3-hashes",
+        .description = R"(
+            Enables support for BLAKE3 hashes.
+        )",
+        .trackingUrl = "",
+    },
 }};
 
 static_assert(

src/libutil/experimental-features.hh

@@ -37,6 +37,7 @@ enum struct ExperimentalFeature
     MountedSSHStore,
     VerifiedFetches,
     PipeOperators,
+    BLAKE3Hashes,
 };
 
 /**

src/libutil/hash.cc

@@ -1,6 +1,7 @@
 #include <iostream>
 #include <cstring>
 
+#include <blake3.h>
 #include <openssl/crypto.h>
 #include <openssl/md5.h>
 #include <openssl/sha.h>
@@ -8,6 +9,7 @@
 #include "args.hh"
 #include "hash.hh"
 #include "archive.hh"
+#include "config.hh"
 #include "split.hh"
 
 #include <sys/types.h>
@@ -20,6 +22,7 @@ namespace nix {
 
 static size_t regularHashSize(HashAlgorithm type) {
     switch (type) {
+    case HashAlgorithm::BLAKE3: return blake3HashSize;
     case HashAlgorithm::MD5: return md5HashSize;
     case HashAlgorithm::SHA1: return sha1HashSize;
     case HashAlgorithm::SHA256: return sha256HashSize;
@@ -29,12 +32,15 @@ static size_t regularHashSize(HashAlgorithm type) {
 }
 
 
-const std::set<std::string> hashAlgorithms = {"md5", "sha1", "sha256", "sha512" };
+const std::set<std::string> hashAlgorithms = {"blake3", "md5", "sha1", "sha256", "sha512" };
 
 const std::set<std::string> hashFormats = {"base64", "nix32", "base16", "sri" };
 
-Hash::Hash(HashAlgorithm algo) : algo(algo)
+Hash::Hash(HashAlgorithm algo, const ExperimentalFeatureSettings & xpSettings) : algo(algo)
 {
+    if (algo == HashAlgorithm::BLAKE3) {
+        xpSettings.require(Xp::BLAKE3Hashes);
+    }
     hashSize = regularHashSize(algo);
     assert(hashSize <= maxHashSize);
     memset(hash, 0, maxHashSize);
@@ -284,6 +290,7 @@ Hash newHashAllowEmpty(std::string_view hashStr, std::optional<HashAlgorithm> ha
 
 union Ctx
 {
+    blake3_hasher blake3;
     MD5_CTX md5;
     SHA_CTX sha1;
     SHA256_CTX sha256;
@@ -293,7 +300,8 @@ union Ctx
 
 static void start(HashAlgorithm ha, Ctx & ctx)
 {
-    if (ha == HashAlgorithm::MD5) MD5_Init(&ctx.md5);
+    if (ha == HashAlgorithm::BLAKE3) blake3_hasher_init(&ctx.blake3);
+    else if (ha == HashAlgorithm::MD5) MD5_Init(&ctx.md5);
     else if (ha == HashAlgorithm::SHA1) SHA1_Init(&ctx.sha1);
     else if (ha == HashAlgorithm::SHA256) SHA256_Init(&ctx.sha256);
     else if (ha == HashAlgorithm::SHA512) SHA512_Init(&ctx.sha512);
@@ -303,7 +311,8 @@ static void start(HashAlgorithm ha, Ctx & ctx)
 static void update(HashAlgorithm ha, Ctx & ctx,
                    std::string_view data)
 {
-    if (ha == HashAlgorithm::MD5) MD5_Update(&ctx.md5, data.data(), data.size());
+    if (ha == HashAlgorithm::BLAKE3) blake3_hasher_update(&ctx.blake3, data.data(), data.size());
+    else if (ha == HashAlgorithm::MD5) MD5_Update(&ctx.md5, data.data(), data.size());
     else if (ha == HashAlgorithm::SHA1) SHA1_Update(&ctx.sha1, data.data(), data.size());
     else if (ha == HashAlgorithm::SHA256) SHA256_Update(&ctx.sha256, data.data(), data.size());
     else if (ha == HashAlgorithm::SHA512) SHA512_Update(&ctx.sha512, data.data(), data.size());
@@ -312,24 +321,24 @@ static void update(HashAlgorithm ha, Ctx & ctx,
 
 static void finish(HashAlgorithm ha, Ctx & ctx, unsigned char * hash)
 {
-    if (ha == HashAlgorithm::MD5) MD5_Final(hash, &ctx.md5);
+    if (ha == HashAlgorithm::BLAKE3) blake3_hasher_finalize(&ctx.blake3, hash, BLAKE3_OUT_LEN);
+    else if (ha == HashAlgorithm::MD5) MD5_Final(hash, &ctx.md5);
     else if (ha == HashAlgorithm::SHA1) SHA1_Final(hash, &ctx.sha1);
     else if (ha == HashAlgorithm::SHA256) SHA256_Final(hash, &ctx.sha256);
     else if (ha == HashAlgorithm::SHA512) SHA512_Final(hash, &ctx.sha512);
 }
 
-
+Hash hashString(
-Hash hashString(HashAlgorithm ha, std::string_view s)
+    HashAlgorithm ha, std::string_view s, const ExperimentalFeatureSettings & xpSettings)
 {
     Ctx ctx;
-    Hash hash(ha);
+    Hash hash(ha, xpSettings);
     start(ha, ctx);
     update(ha, ctx, s);
     finish(ha, ctx, hash.hash);
     return hash;
 }
 
-
 Hash hashFile(HashAlgorithm ha, const Path & path)
 {
     HashSink sink(ha);
@@ -426,6 +435,7 @@ std::string_view printHashFormat(HashFormat HashFormat)
 
 std::optional<HashAlgorithm> parseHashAlgoOpt(std::string_view s)
 {
+    if (s == "blake3") return HashAlgorithm::BLAKE3;
     if (s == "md5") return HashAlgorithm::MD5;
     if (s == "sha1") return HashAlgorithm::SHA1;
     if (s == "sha256") return HashAlgorithm::SHA256;
@@ -439,12 +449,13 @@ HashAlgorithm parseHashAlgo(std::string_view s)
     if (opt_h)
         return *opt_h;
     else
-        throw UsageError("unknown hash algorithm '%1%', expect 'md5', 'sha1', 'sha256', or 'sha512'", s);
+        throw UsageError("unknown hash algorithm '%1%', expect 'blake3', 'md5', 'sha1', 'sha256', or 'sha512'", s);
 }
 
 std::string_view printHashAlgo(HashAlgorithm ha)
 {
     switch (ha) {
+    case HashAlgorithm::BLAKE3: return "blake3";
     case HashAlgorithm::MD5: return "md5";
     case HashAlgorithm::SHA1: return "sha1";
     case HashAlgorithm::SHA256: return "sha256";

src/libutil/hash.hh

@@ -1,6 +1,7 @@
 #pragma once
 ///@file
 
+#include "config.hh"
 #include "types.hh"
 #include "serialise.hh"
 #include "file-system.hh"
@@ -11,9 +12,9 @@ namespace nix {
 MakeError(BadHash, Error);
 
 
-enum struct HashAlgorithm : char { MD5 = 42, SHA1, SHA256, SHA512 };
+enum struct HashAlgorithm : char { MD5 = 42, SHA1, SHA256, SHA512, BLAKE3 };
-
 
+const int blake3HashSize = 32;
 const int md5HashSize = 16;
 const int sha1HashSize = 20;
 const int sha256HashSize = 32;
@@ -52,7 +53,7 @@ struct Hash
     /**
      * Create a zero-filled hash object.
      */
-    explicit Hash(HashAlgorithm algo);
+    explicit Hash(HashAlgorithm algo, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
     /**
      * Parse the hash from a string representation in the format
@@ -157,7 +158,7 @@ std::string printHash16or32(const Hash & hash);
 /**
  * Compute the hash of the given string.
  */
-Hash hashString(HashAlgorithm ha, std::string_view s);
+Hash hashString(HashAlgorithm ha, std::string_view s, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
 /**
  * Compute the hash of the given file, hashing its contents directly.

src/libutil/meson.build

@@ -62,6 +62,12 @@ elif host_machine.system() == 'sunos'
   deps_other += [socket, network_service_library]
 endif
 
+blake3 = dependency(
+  'libblake3',
+  version: '>= 1.5.5',
+)
+deps_private += blake3
+
 boost = dependency(
   'boost',
   modules : ['context', 'coroutine'],

src/libutil/package.nix

@@ -6,6 +6,7 @@
   boost,
   brotli,
   libarchive,
+  libblake3,
   libcpuid,
   libsodium,
   nlohmann_json,
@@ -42,6 +43,7 @@ mkMesonLibrary (finalAttrs: {
 
   buildInputs = [
     brotli
+    libblake3
     libsodium
     openssl
   ] ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid;