From 241c539f6fab7177e37f3736e7942dc8a0d0a09e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 3 Jul 2024 16:28:24 +0200 Subject: [PATCH 1/3] Tarball fetcher: Fix fetchToStore() and eval caching (cherry picked from commit 9d95c228eeb2750d37f86228905d11eea5fb1e05) --- src/libfetchers/fetchers.cc | 1 + src/libfetchers/tarball.cc | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 73923907c..86d13dd7b 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -260,6 +260,7 @@ std::pair, Input> Input::getAccessorUnchecked(ref sto auto [accessor, final] = scheme->getAccessor(store, *this); + assert(!accessor->fingerprint); accessor->fingerprint = scheme->getFingerprint(store, final); return {accessor, std::move(final)}; diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 5de367052..aa8ff652f 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -365,6 +365,16 @@ struct TarballInputScheme : CurlInputScheme return {result.accessor, input}; } + + std::optional getFingerprint(ref store, const Input & input) const override + { + if (auto narHash = input.getNarHash()) + return narHash->to_string(HashFormat::SRI, true); + else if (auto rev = input.getRev()) + return rev->gitRev(); + else + return std::nullopt; + } }; static auto rTarballInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); From e38f45b19fc721e06ccf87bf2134c98601db96cc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 3 Jul 2024 16:37:26 +0200 Subject: [PATCH 2/3] nix flake metadata: Show flake fingerprint This is useful for testing/debugging and maybe for sharing eval caches (since it tells you what file in ~/.cache/nix/eval-cache-v5 to copy). (cherry picked from commit 1ff186fc6e99e57501fee9291e7013ea88d8720a) --- src/nix/flake.cc | 6 ++++++ tests/functional/flakes/flakes.sh | 1 + tests/nixos/tarball-flakes.nix | 3 +++ 3 files changed, 10 insertions(+) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 6bf694e08..6a9aacb91 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -232,6 +232,8 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON j["lastModified"] = *lastModified; j["path"] = storePath; j["locks"] = lockedFlake.lockFile.toJSON().first; + if (auto fingerprint = lockedFlake.getFingerprint(store)) + j["fingerprint"] = fingerprint->to_string(HashFormat::Base16, false); logger->cout("%s", j.dump()); } else { logger->cout( @@ -264,6 +266,10 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON logger->cout( ANSI_BOLD "Last modified:" ANSI_NORMAL " %s", std::put_time(std::localtime(&*lastModified), "%F %T")); + if (auto fingerprint = lockedFlake.getFingerprint(store)) + logger->cout( + ANSI_BOLD "Fingerprint:" ANSI_NORMAL " %s", + fingerprint->to_string(HashFormat::Base16, false)); if (!lockedFlake.lockFile.root->inputs.empty()) logger->cout(ANSI_BOLD "Inputs:" ANSI_NORMAL); diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index 3057c0293..3c7bbcafe 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -189,6 +189,7 @@ json=$(nix flake metadata flake1 --json | jq .) [[ -d $(echo "$json" | jq -r .path) ]] [[ $(echo "$json" | jq -r .lastModified) = $(git -C "$flake1Dir" log -n1 --format=%ct) ]] hash1=$(echo "$json" | jq -r .revision) +[[ -n $(echo "$json" | jq -r .fingerprint) ]] echo foo > "$flake1Dir/foo" git -C "$flake1Dir" add $flake1Dir/foo diff --git a/tests/nixos/tarball-flakes.nix b/tests/nixos/tarball-flakes.nix index e30d15739..01d8bcadb 100644 --- a/tests/nixos/tarball-flakes.nix +++ b/tests/nixos/tarball-flakes.nix @@ -66,6 +66,9 @@ in # Check that we got redirected to the immutable URL. assert info["locked"]["url"] == "http://localhost/stable/${nixpkgs.rev}.tar.gz" + # Check that we got a fingerprint for caching. + assert info["fingerprint"] + # Check that we got the rev and revCount attributes. assert info["revision"] == "${nixpkgs.rev}" assert info["revCount"] == 1234 From b91c7cf077718ea85f30f2665c61a359f31cfabd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 3 Jul 2024 21:54:54 +0200 Subject: [PATCH 3/3] Tarball fetcher: Include revCount/lastModified in the fingerprint This can influence the evaluation result so they should be included in the fingerprint. (cherry picked from commit 5b4102c3b25350872d8fefffac3547a63863f0c1) --- src/libexpr/flake/flake.cc | 12 +++++++++++- src/libfetchers/fetchers.cc | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 3af9ef14e..e97867443 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -949,10 +949,20 @@ std::optional LockedFlake::getFingerprint(ref store) const auto fingerprint = flake.lockedRef.input.getFingerprint(store); if (!fingerprint) return std::nullopt; + *fingerprint += fmt(";%s;%s", flake.lockedRef.subdir, lockFile); + + /* Include revCount and lastModified because they're not + necessarily implied by the content fingerprint (e.g. for + tarball flakes) but can influence the evaluation result. */ + if (auto revCount = flake.lockedRef.input.getRevCount()) + *fingerprint += fmt(";revCount=%d", *revCount); + if (auto lastModified = flake.lockedRef.input.getLastModified()) + *fingerprint += fmt(";lastModified=%d", *lastModified); + // FIXME: as an optimization, if the flake contains a lock file // and we haven't changed it, then it's sufficient to use // flake.sourceInfo.storePath for the fingerprint. - return hashString(HashAlgorithm::SHA256, fmt("%s;%s;%s", *fingerprint, flake.lockedRef.subdir, lockFile)); + return hashString(HashAlgorithm::SHA256, *fingerprint); } Flake::~Flake() { } diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 86d13dd7b..c11ddc467 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -419,7 +419,7 @@ namespace nlohmann { using namespace nix; fetchers::PublicKey adl_serializer::from_json(const json & json) { - fetchers::PublicKey res = { }; + fetchers::PublicKey res = { }; if (auto type = optionalValueAt(json, "type")) res.type = getString(*type);