From 76f75e76915329c4f3502abcbea1df8e3ee658c9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 8 Oct 2024 15:28:49 +0200
Subject: [PATCH 01/73] nix copy: Add --profile flag

This allows `nix copy` to atomically copy a store path and point a
profile to it, without the risk that the store path might be GC'ed in
between. This is useful for instance when deploying a new NixOS system
profile from a remote store.
---
 src/libcmd/command.cc    | 35 ++++++++++++++++++++++-------------
 src/libcmd/command.hh    | 12 ++++++++----
 src/nix/build.cc         |  2 +-
 src/nix/copy.cc          |  8 +++++---
 src/nix/copy.md          |  9 +++++++++
 src/nix/develop.cc       |  2 +-
 src/nix/profile.cc       |  6 +++---
 src/nix/realisation.cc   |  2 +-
 tests/functional/zstd.sh |  4 +++-
 9 files changed, 53 insertions(+), 27 deletions(-)

diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc
index 6d8bfc19b..e38f982d8 100644
--- a/src/libcmd/command.cc
+++ b/src/libcmd/command.cc
@@ -179,30 +179,34 @@ BuiltPathsCommand::BuiltPathsCommand(bool recursive)
 
 void BuiltPathsCommand::run(ref<Store> store, Installables && installables)
 {
-    BuiltPaths paths;
+    BuiltPaths rootPaths, allPaths;
+
     if (all) {
         if (installables.size())
             throw UsageError("'--all' does not expect arguments");
         // XXX: Only uses opaque paths, ignores all the realisations
         for (auto & p : store->queryAllValidPaths())
-            paths.emplace_back(BuiltPath::Opaque{p});
+            rootPaths.emplace_back(BuiltPath::Opaque{p});
+        allPaths = rootPaths;
     } else {
-        paths = Installable::toBuiltPaths(getEvalStore(), store, realiseMode, operateOn, installables);
+        rootPaths = Installable::toBuiltPaths(getEvalStore(), store, realiseMode, operateOn, installables);
+        allPaths = rootPaths;
+
         if (recursive) {
             // XXX: This only computes the store path closure, ignoring
             // intermediate realisations
             StorePathSet pathsRoots, pathsClosure;
-            for (auto & root : paths) {
+            for (auto & root : rootPaths) {
                 auto rootFromThis = root.outPaths();
                 pathsRoots.insert(rootFromThis.begin(), rootFromThis.end());
             }
             store->computeFSClosure(pathsRoots, pathsClosure);
             for (auto & path : pathsClosure)
-                paths.emplace_back(BuiltPath::Opaque{path});
+                allPaths.emplace_back(BuiltPath::Opaque{path});
         }
     }
 
-    run(store, std::move(paths));
+    run(store, std::move(allPaths), std::move(rootPaths));
 }
 
 StorePathsCommand::StorePathsCommand(bool recursive)
@@ -210,10 +214,10 @@ StorePathsCommand::StorePathsCommand(bool recursive)
 {
 }
 
-void StorePathsCommand::run(ref<Store> store, BuiltPaths && paths)
+void StorePathsCommand::run(ref<Store> store, BuiltPaths && allPaths, BuiltPaths && rootPaths)
 {
     StorePathSet storePaths;
-    for (auto & builtPath : paths)
+    for (auto & builtPath : allPaths)
         for (auto & p : builtPath.outPaths())
             storePaths.insert(p);
 
@@ -242,17 +246,21 @@ MixProfile::MixProfile()
     });
 }
 
-void MixProfile::updateProfile(const StorePath & storePath)
+void MixProfile::updateProfile(
+    const StorePath & storePath,
+    ref<Store> store_)
 {
     if (!profile) return;
-    auto store = getStore().dynamic_pointer_cast<LocalFSStore>();
+    auto store = store_.dynamic_pointer_cast<LocalFSStore>();
     if (!store) throw Error("'--profile' is not supported for this Nix store");
     auto profile2 = absPath(*profile);
     switchLink(profile2,
         createGeneration(*store, profile2, storePath));
 }
 
-void MixProfile::updateProfile(const BuiltPaths & buildables)
+void MixProfile::updateProfile(
+    const BuiltPaths & buildables,
+    ref<Store> store)
 {
     if (!profile) return;
 
@@ -274,7 +282,7 @@ void MixProfile::updateProfile(const BuiltPaths & buildables)
     if (result.size() != 1)
         throw UsageError("'--profile' requires that the arguments produce a single store path, but there are %d", result.size());
 
-    updateProfile(result[0]);
+    updateProfile(result[0], store);
 }
 
 MixDefaultProfile::MixDefaultProfile()
@@ -308,7 +316,8 @@ MixEnvironment::MixEnvironment() : ignoreEnvironment(false)
     });
 }
 
-void MixEnvironment::setEnviron() {
+void MixEnvironment::setEnviron()
+{
     if (ignoreEnvironment) {
         if (!unset.empty())
             throw UsageError("--unset does not make sense with --ignore-environment");
diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh
index 4a72627ed..8ada78782 100644
--- a/src/libcmd/command.hh
+++ b/src/libcmd/command.hh
@@ -238,7 +238,7 @@ public:
 
     BuiltPathsCommand(bool recursive = false);
 
-    virtual void run(ref<Store> store, BuiltPaths && paths) = 0;
+    virtual void run(ref<Store> store, BuiltPaths && allPaths, BuiltPaths && rootPaths) = 0;
 
     void run(ref<Store> store, Installables && installables) override;
 
@@ -251,7 +251,7 @@ struct StorePathsCommand : public BuiltPathsCommand
 
     virtual void run(ref<Store> store, StorePaths && storePaths) = 0;
 
-    void run(ref<Store> store, BuiltPaths && paths) override;
+    void run(ref<Store> store, BuiltPaths && allPaths, BuiltPaths && rootPaths) override;
 };
 
 /**
@@ -301,11 +301,15 @@ struct MixProfile : virtual StoreCommand
     MixProfile();
 
     /* If 'profile' is set, make it point at 'storePath'. */
-    void updateProfile(const StorePath & storePath);
+    void updateProfile(
+        const StorePath & storePath,
+        ref<Store> store);
 
     /* If 'profile' is set, make it point at the store path produced
        by 'buildables'. */
-    void updateProfile(const BuiltPaths & buildables);
+    void updateProfile(
+        const BuiltPaths & buildables,
+        ref<Store> store);
 };
 
 struct MixDefaultProfile : MixProfile
diff --git a/src/nix/build.cc b/src/nix/build.cc
index da9132d02..cffbeccb6 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -161,7 +161,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
         BuiltPaths buildables2;
         for (auto & b : buildables)
             buildables2.push_back(b.path);
-        updateProfile(buildables2);
+        updateProfile(buildables2, store);
     }
 };
 
diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index 151d28277..c058a7446 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -4,7 +4,7 @@
 
 using namespace nix;
 
-struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand
+struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand, MixProfile
 {
     CheckSigsFlag checkSigs = CheckSigs;
 
@@ -43,19 +43,21 @@ struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand
 
     Category category() override { return catSecondary; }
 
-    void run(ref<Store> srcStore, BuiltPaths && paths) override
+    void run(ref<Store> srcStore, BuiltPaths && allPaths, BuiltPaths && rootPaths) override
     {
         auto dstStore = getDstStore();
 
         RealisedPath::Set stuffToCopy;
 
-        for (auto & builtPath : paths) {
+        for (auto & builtPath : allPaths) {
             auto theseRealisations = builtPath.toRealisedPaths(*srcStore);
             stuffToCopy.insert(theseRealisations.begin(), theseRealisations.end());
         }
 
         copyPaths(
             *srcStore, *dstStore, stuffToCopy, NoRepair, checkSigs, substitute);
+
+        updateProfile(rootPaths, dstStore);
     }
 };
 
diff --git a/src/nix/copy.md b/src/nix/copy.md
index 6ab7cdee3..813050fcb 100644
--- a/src/nix/copy.md
+++ b/src/nix/copy.md
@@ -55,6 +55,15 @@ R""(
   # nix copy --to /tmp/nix nixpkgs#hello --no-check-sigs
   ```
 
+* Update the NixOS system profile to point to a closure copied from a
+  remote machine:
+
+  ```console
+  # nix copy --from ssh://server \
+      --profile /nix/var/nix/profiles/system \
+      /nix/store/r14v3km89zm3prwsa521fab5kgzvfbw4-nixos-system-foobar-24.05.20240925.759537f
+  ```
+
 # Description
 
 `nix copy` copies store path closures between two Nix stores. The
diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index c7a733025..ca9f6a4af 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -502,7 +502,7 @@ struct Common : InstallableCommand, MixProfile
 
         auto strPath = store->printStorePath(shellOutPath);
 
-        updateProfile(shellOutPath);
+        updateProfile(shellOutPath, store);
 
         debug("reading environment file '%s'", strPath);
 
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index 324fd6330..ae1f3e6e9 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -424,7 +424,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
         }
 
         try {
-            updateProfile(manifest.build(store));
+            updateProfile(manifest.build(store), store);
         } catch (BuildEnvFileConflictError & conflictError) {
             // FIXME use C++20 std::ranges once macOS has it
             //       See https://github.com/NixOS/nix/compare/3efa476c5439f8f6c1968a6ba20a31d1239c2f04..1fe5d172ece51a619e879c4b86f603d9495cc102
@@ -669,7 +669,7 @@ struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElem
             removedCount,
             newManifest.elements.size());
 
-        updateProfile(newManifest.build(store));
+        updateProfile(newManifest.build(store), store);
     }
 };
 
@@ -779,7 +779,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
                 builtPaths.find(&*installable)->second.first);
         }
 
-        updateProfile(manifest.build(store));
+        updateProfile(manifest.build(store), store);
     }
 };
 
diff --git a/src/nix/realisation.cc b/src/nix/realisation.cc
index e1f231222..a386d98ea 100644
--- a/src/nix/realisation.cc
+++ b/src/nix/realisation.cc
@@ -36,7 +36,7 @@ struct CmdRealisationInfo : BuiltPathsCommand, MixJSON
 
     Category category() override { return catSecondary; }
 
-    void run(ref<Store> store, BuiltPaths && paths) override
+    void run(ref<Store> store, BuiltPaths && paths, BuiltPaths && rootPaths) override
     {
         experimentalFeatureSettings.require(Xp::CaDerivations);
         RealisedPath::Set realisations;
diff --git a/tests/functional/zstd.sh b/tests/functional/zstd.sh
index 90fe58539..cdc30c66e 100755
--- a/tests/functional/zstd.sh
+++ b/tests/functional/zstd.sh
@@ -18,7 +18,9 @@ HASH=$(nix hash path $outPath)
 clearStore
 clearCacheCache
 
-nix copy --from $cacheURI $outPath --no-check-sigs
+nix copy --from $cacheURI $outPath --no-check-sigs --profile $TEST_ROOT/profile
+
+[[ -e $TEST_ROOT/profile ]]
 
 if ls $cacheDir/nar/*.zst &> /dev/null; then
     echo "files do exist"

From 43ad8c5eb2bdf4995467595f83efd6d9a747b440 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 8 Oct 2024 15:36:21 +0200
Subject: [PATCH 02/73] Make getDstStore() a virtual method in StoreCommand

---
 src/libcmd/command.cc | 12 ++++--------
 src/libcmd/command.hh | 23 ++++++++++++++++-------
 src/nix/build.cc      |  2 +-
 src/nix/copy.cc       |  2 +-
 src/nix/develop.cc    |  2 +-
 src/nix/profile.cc    |  6 +++---
 6 files changed, 26 insertions(+), 21 deletions(-)

diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc
index e38f982d8..0fb4f5680 100644
--- a/src/libcmd/command.cc
+++ b/src/libcmd/command.cc
@@ -246,21 +246,17 @@ MixProfile::MixProfile()
     });
 }
 
-void MixProfile::updateProfile(
-    const StorePath & storePath,
-    ref<Store> store_)
+void MixProfile::updateProfile(const StorePath & storePath)
 {
     if (!profile) return;
-    auto store = store_.dynamic_pointer_cast<LocalFSStore>();
+    auto store = getDstStore().dynamic_pointer_cast<LocalFSStore>();
     if (!store) throw Error("'--profile' is not supported for this Nix store");
     auto profile2 = absPath(*profile);
     switchLink(profile2,
         createGeneration(*store, profile2, storePath));
 }
 
-void MixProfile::updateProfile(
-    const BuiltPaths & buildables,
-    ref<Store> store)
+void MixProfile::updateProfile(const BuiltPaths & buildables)
 {
     if (!profile) return;
 
@@ -282,7 +278,7 @@ void MixProfile::updateProfile(
     if (result.size() != 1)
         throw UsageError("'--profile' requires that the arguments produce a single store path, but there are %d", result.size());
 
-    updateProfile(result[0], store);
+    updateProfile(result[0]);
 }
 
 MixDefaultProfile::MixDefaultProfile()
diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh
index 8ada78782..73aaab741 100644
--- a/src/libcmd/command.hh
+++ b/src/libcmd/command.hh
@@ -45,7 +45,20 @@ struct StoreCommand : virtual Command
 {
     StoreCommand();
     void run() override;
+
+    /**
+     * Return the default Nix store.
+     */
     ref<Store> getStore();
+
+    /**
+     * Return the destination Nix store.
+     */
+    virtual ref<Store> getDstStore()
+    {
+        return getStore();
+    }
+
     virtual ref<Store> createStore();
     /**
      * Main entry point, with a `Store` provided
@@ -68,7 +81,7 @@ struct CopyCommand : virtual StoreCommand
 
     ref<Store> createStore() override;
 
-    ref<Store> getDstStore();
+    ref<Store> getDstStore() override;
 };
 
 /**
@@ -301,15 +314,11 @@ struct MixProfile : virtual StoreCommand
     MixProfile();
 
     /* If 'profile' is set, make it point at 'storePath'. */
-    void updateProfile(
-        const StorePath & storePath,
-        ref<Store> store);
+    void updateProfile(const StorePath & storePath);
 
     /* If 'profile' is set, make it point at the store path produced
        by 'buildables'. */
-    void updateProfile(
-        const BuiltPaths & buildables,
-        ref<Store> store);
+    void updateProfile(const BuiltPaths & buildables);
 };
 
 struct MixDefaultProfile : MixProfile
diff --git a/src/nix/build.cc b/src/nix/build.cc
index cffbeccb6..da9132d02 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -161,7 +161,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
         BuiltPaths buildables2;
         for (auto & b : buildables)
             buildables2.push_back(b.path);
-        updateProfile(buildables2, store);
+        updateProfile(buildables2);
     }
 };
 
diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index c058a7446..60a64723c 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -57,7 +57,7 @@ struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand, MixProfile
         copyPaths(
             *srcStore, *dstStore, stuffToCopy, NoRepair, checkSigs, substitute);
 
-        updateProfile(rootPaths, dstStore);
+        updateProfile(rootPaths);
     }
 };
 
diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index ca9f6a4af..c7a733025 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -502,7 +502,7 @@ struct Common : InstallableCommand, MixProfile
 
         auto strPath = store->printStorePath(shellOutPath);
 
-        updateProfile(shellOutPath, store);
+        updateProfile(shellOutPath);
 
         debug("reading environment file '%s'", strPath);
 
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index ae1f3e6e9..324fd6330 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -424,7 +424,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
         }
 
         try {
-            updateProfile(manifest.build(store), store);
+            updateProfile(manifest.build(store));
         } catch (BuildEnvFileConflictError & conflictError) {
             // FIXME use C++20 std::ranges once macOS has it
             //       See https://github.com/NixOS/nix/compare/3efa476c5439f8f6c1968a6ba20a31d1239c2f04..1fe5d172ece51a619e879c4b86f603d9495cc102
@@ -669,7 +669,7 @@ struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElem
             removedCount,
             newManifest.elements.size());
 
-        updateProfile(newManifest.build(store), store);
+        updateProfile(newManifest.build(store));
     }
 };
 
@@ -779,7 +779,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
                 builtPaths.find(&*installable)->second.first);
         }
 
-        updateProfile(manifest.build(store), store);
+        updateProfile(manifest.build(store));
     }
 };
 

From 7f6d006bebfb56b2967165c7f2e8c940abafac24 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 8 Oct 2024 16:35:53 +0200
Subject: [PATCH 03/73] nix copy: Add --out-link

---
 src/libcmd/command.cc      | 25 +++++++++++++++++++++++++
 src/libcmd/command.hh      | 10 ++++++++++
 src/libcmd/installables.cc |  8 ++++++++
 src/libcmd/installables.hh |  2 ++
 src/nix/build.cc           | 25 +------------------------
 src/nix/copy.cc            | 18 ++++++++++++++++++
 tests/functional/zstd.sh   |  3 ++-
 7 files changed, 66 insertions(+), 25 deletions(-)

diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc
index 0fb4f5680..8053e1da6 100644
--- a/src/libcmd/command.cc
+++ b/src/libcmd/command.cc
@@ -334,4 +334,29 @@ void MixEnvironment::setEnviron()
     }
 }
 
+void createOutLinks(
+    const std::filesystem::path & outLink,
+    const BuiltPaths & buildables,
+    LocalFSStore & store)
+{
+    for (const auto & [_i, buildable] : enumerate(buildables)) {
+        auto i = _i;
+        std::visit(overloaded {
+            [&](const BuiltPath::Opaque & bo) {
+                auto symlink = outLink;
+                if (i) symlink += fmt("-%d", i);
+                store.addPermRoot(bo.path, absPath(symlink.string()));
+            },
+            [&](const BuiltPath::Built & bfd) {
+                for (auto & output : bfd.outputs) {
+                    auto symlink = outLink;
+                    if (i) symlink += fmt("-%d", i);
+                    if (output.first != "out") symlink += fmt("-%s", output.first);
+                    store.addPermRoot(output.second, absPath(symlink.string()));
+                }
+            },
+        }, buildable.raw());
+    }
+}
+
 }
diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh
index 73aaab741..e9fcf3df9 100644
--- a/src/libcmd/command.hh
+++ b/src/libcmd/command.hh
@@ -18,6 +18,7 @@ extern char * * savedArgv;
 class EvalState;
 struct Pos;
 class Store;
+class LocalFSStore;
 
 static constexpr Command::Category catHelp = -1;
 static constexpr Command::Category catSecondary = 100;
@@ -367,4 +368,13 @@ void printClosureDiff(
     const StorePath & afterPath,
     std::string_view indent);
 
+/**
+ * Create symlinks prefixed by `outLink` to the store paths in
+ * `buildables`.
+ */
+void createOutLinks(
+    const std::filesystem::path & outLink,
+    const BuiltPaths & buildables,
+    LocalFSStore & store);
+
 }
diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index f9d6d8ce8..86d26f6c4 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -917,4 +917,12 @@ void BuiltPathsCommand::applyDefaultInstallables(std::vector<std::string> & rawI
         rawInstallables.push_back(".");
 }
 
+BuiltPaths toBuiltPaths(const std::vector<BuiltPathWithResult> & builtPathsWithResult)
+{
+    BuiltPaths res;
+    for (auto & i : builtPathsWithResult)
+        res.push_back(i.path);
+    return res;
+}
+
 }
diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh
index bf5759230..c995c3019 100644
--- a/src/libcmd/installables.hh
+++ b/src/libcmd/installables.hh
@@ -86,6 +86,8 @@ struct BuiltPathWithResult
     std::optional<BuildResult> result;
 };
 
+BuiltPaths toBuiltPaths(const std::vector<BuiltPathWithResult> & builtPathsWithResult);
+
 /**
  * Shorthand, for less typing and helping us keep the choice of
  * collection in sync.
diff --git a/src/nix/build.cc b/src/nix/build.cc
index da9132d02..3569b0cde 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -42,29 +42,6 @@ static nlohmann::json builtPathsWithResultToJSON(const std::vector<BuiltPathWith
     return res;
 }
 
-// TODO deduplicate with other code also setting such out links.
-static void createOutLinks(const std::filesystem::path& outLink, const std::vector<BuiltPathWithResult>& buildables, LocalFSStore& store2)
-{
-    for (const auto & [_i, buildable] : enumerate(buildables)) {
-        auto i = _i;
-        std::visit(overloaded {
-            [&](const BuiltPath::Opaque & bo) {
-                auto symlink = outLink;
-                if (i) symlink += fmt("-%d", i);
-                store2.addPermRoot(bo.path, absPath(symlink.string()));
-            },
-            [&](const BuiltPath::Built & bfd) {
-                for (auto & output : bfd.outputs) {
-                    auto symlink = outLink;
-                    if (i) symlink += fmt("-%d", i);
-                    if (output.first != "out") symlink += fmt("-%s", output.first);
-                    store2.addPermRoot(output.second, absPath(symlink.string()));
-                }
-            },
-        }, buildable.path.raw());
-    }
-}
-
 struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
 {
     Path outLink = "result";
@@ -140,7 +117,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
 
         if (outLink != "")
             if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
-                createOutLinks(outLink, buildables, *store2);
+                createOutLinks(outLink, toBuiltPaths(buildables), *store2);
 
         if (printOutputPaths) {
             stopProgressBar();
diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index 60a64723c..399a6c0fd 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -1,11 +1,13 @@
 #include "command.hh"
 #include "shared.hh"
 #include "store-api.hh"
+#include "local-fs-store.hh"
 
 using namespace nix;
 
 struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand, MixProfile
 {
+    std::optional<std::filesystem::path> outLink;
     CheckSigsFlag checkSigs = CheckSigs;
 
     SubstituteFlag substitute = NoSubstitute;
@@ -13,6 +15,15 @@ struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand, MixProfile
     CmdCopy()
         : BuiltPathsCommand(true)
     {
+        addFlag({
+            .longName = "out-link",
+            .shortName = 'o',
+            .description = "Create symlinks prefixed with *path* to the top-level store paths fetched from the source store.",
+            .labels = {"path"},
+            .handler = {&outLink},
+            .completer = completePath
+        });
+
         addFlag({
             .longName = "no-check-sigs",
             .description = "Do not require that paths are signed by trusted keys.",
@@ -58,6 +69,13 @@ struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand, MixProfile
             *srcStore, *dstStore, stuffToCopy, NoRepair, checkSigs, substitute);
 
         updateProfile(rootPaths);
+
+        if (outLink) {
+            if (auto store2 = dstStore.dynamic_pointer_cast<LocalFSStore>())
+                createOutLinks(*outLink, rootPaths, *store2);
+            else
+                throw Error("'--out-link' is not supported for this Nix store");
+        }
     }
 };
 
diff --git a/tests/functional/zstd.sh b/tests/functional/zstd.sh
index cdc30c66e..1eb1b5343 100755
--- a/tests/functional/zstd.sh
+++ b/tests/functional/zstd.sh
@@ -18,9 +18,10 @@ HASH=$(nix hash path $outPath)
 clearStore
 clearCacheCache
 
-nix copy --from $cacheURI $outPath --no-check-sigs --profile $TEST_ROOT/profile
+nix copy --from $cacheURI $outPath --no-check-sigs --profile $TEST_ROOT/profile --out-link $TEST_ROOT/result
 
 [[ -e $TEST_ROOT/profile ]]
+[[ -e $TEST_ROOT/result ]]
 
 if ls $cacheDir/nar/*.zst &> /dev/null; then
     echo "files do exist"

From e9b5704d1c3bdead2d9d259b3c443a391306b6d7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 8 Oct 2024 16:49:35 +0200
Subject: [PATCH 04/73] Add release note

---
 doc/manual/rl-next/nix-copy-flags.md | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 doc/manual/rl-next/nix-copy-flags.md

diff --git a/doc/manual/rl-next/nix-copy-flags.md b/doc/manual/rl-next/nix-copy-flags.md
new file mode 100644
index 000000000..f5b2b9716
--- /dev/null
+++ b/doc/manual/rl-next/nix-copy-flags.md
@@ -0,0 +1,18 @@
+---
+synopsis: "`nix copy` supports `--profile` and `--out-link`"
+prs: [11657]
+---
+
+The `nix copy` command now has flags `--profile` and `--out-link`, similar to `nix build`. `--profile` makes a profile point to the
+top-level store path, while `--out-link` create symlinks to the top-level store paths.
+
+For example, when updating the local NixOS system profile from a NixOS system closure on a remote machine, instead of
+```
+# nix copy --from ssh://server $path
+# nix build --profile /nix/var/nix/profiles/system $path
+```
+you can now do
+```
+# nix copy --from ssh://server --profile /nix/var/nix/profiles/system $path
+```
+The advantage is that this avoids a time window where *path* is not a garbage collector root, and so could be deleted by a concurrent `nix store gc` process.

From 27ea43781371cad717077ae723b11a79c0d0fc78 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 10 Oct 2024 16:54:11 +0200
Subject: [PATCH 05/73] Support fine-grained database schema migrations

Backward-compatible schema changes (e.g. those that add tables or
nullable columns) now no longer need a change to the global schema
file (/nix/var/nix/db/schema). Thus, old Nix versions can continue to
access the database.

This is especially useful for schema changes required by experimental
features. In particular, it replaces the ad-hoc handling of the schema
changes for CA derivations (i.e. the file /nix/var/nix/db/ca-schema).

Schema versions 8 and 10 could have been handled by this mechanism in
a backward-compatible way as well.
---
 src/libstore/local-store.cc | 105 ++++++++++++++++++------------------
 src/libstore/local-store.hh |   2 +
 2 files changed, 54 insertions(+), 53 deletions(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index eafdac0cd..f708bd1b0 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -95,51 +95,6 @@ struct LocalStore::State::Stmts {
     SQLiteStmt AddRealisationReference;
 };
 
-static int getSchema(Path schemaPath)
-{
-    int curSchema = 0;
-    if (pathExists(schemaPath)) {
-        auto s = readFile(schemaPath);
-        auto n = string2Int<int>(s);
-        if (!n)
-            throw Error("'%1%' is corrupt", schemaPath);
-        curSchema = *n;
-    }
-    return curSchema;
-}
-
-void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd)
-{
-    const int nixCASchemaVersion = 4;
-    int curCASchema = getSchema(schemaPath);
-    if (curCASchema != nixCASchemaVersion) {
-        if (curCASchema > nixCASchemaVersion) {
-            throw Error("current Nix store ca-schema is version %1%, but I only support %2%",
-                 curCASchema, nixCASchemaVersion);
-        }
-
-        if (!lockFile(lockFd.get(), ltWrite, false)) {
-            printInfo("waiting for exclusive access to the Nix store for ca drvs...");
-            lockFile(lockFd.get(), ltNone, false); // We have acquired a shared lock; release it to prevent deadlocks
-            lockFile(lockFd.get(), ltWrite, true);
-        }
-
-        if (curCASchema == 0) {
-            static const char schema[] =
-              #include "ca-specific-schema.sql.gen.hh"
-                ;
-            db.exec(schema);
-            curCASchema = nixCASchemaVersion;
-        }
-
-        if (curCASchema < 4)
-            throw Error("experimental CA schema version %d is no longer supported", curCASchema);
-
-        writeFile(schemaPath, fmt("%d", nixCASchemaVersion), 0666, true);
-        lockFile(lockFd.get(), ltRead, true);
-    }
-}
-
 LocalStore::LocalStore(
     std::string_view scheme,
     PathView path,
@@ -316,6 +271,10 @@ LocalStore::LocalStore(
 
         openDB(*state, false);
 
+        /* Legacy database schema migrations. Don't bump 'schema' for
+           new migrations; instead, add a migration to
+           upgradeDBSchema(). */
+
         if (curSchema < 8) {
             SQLiteTxn txn(state->db);
             state->db.exec("alter table ValidPaths add column ultimate integer");
@@ -342,13 +301,7 @@ LocalStore::LocalStore(
 
     else openDB(*state, false);
 
-    if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
-        if (!readOnly) {
-            migrateCASchema(state->db, dbDir + "/ca-schema", globalLock);
-        } else {
-            throw Error("need to migrate to content-addressed schema, but this cannot be done in read-only mode");
-        }
-    }
+    upgradeDBSchema(*state);
 
     /* Prepare SQL statements. */
     state->stmts->RegisterValidPath.create(state->db,
@@ -483,7 +436,17 @@ std::string LocalStore::getUri()
 
 
 int LocalStore::getSchema()
-{ return nix::getSchema(schemaPath); }
+{
+    int curSchema = 0;
+    if (pathExists(schemaPath)) {
+        auto s = readFile(schemaPath);
+        auto n = string2Int<int>(s);
+        if (!n)
+            throw Error("'%1%' is corrupt", schemaPath);
+        curSchema = *n;
+    }
+    return curSchema;
+}
 
 void LocalStore::openDB(State & state, bool create)
 {
@@ -566,6 +529,42 @@ void LocalStore::openDB(State & state, bool create)
 }
 
 
+void LocalStore::upgradeDBSchema(State & state)
+{
+    state.db.exec("create table if not exists SchemaMigrations (migration text primary key not null);");
+
+    std::set<std::string> schemaMigrations;
+
+    {
+        SQLiteStmt querySchemaMigrations;
+        querySchemaMigrations.create(state.db, "select migration from SchemaMigrations;");
+        auto useQuerySchemaMigrations(querySchemaMigrations.use());
+        while (useQuerySchemaMigrations.next())
+            schemaMigrations.insert(useQuerySchemaMigrations.getStr(0));
+    }
+
+    auto doUpgrade = [&](const std::string & migrationName, const std::string & stmt)
+    {
+        if (schemaMigrations.contains(migrationName))
+            return;
+
+        debug("executing Nix database schema migration '%s'...", migrationName);
+
+        SQLiteTxn txn(state.db);
+        state.db.exec(stmt + fmt(";\ninsert into SchemaMigrations values('%s')", migrationName));
+        txn.commit();
+
+        schemaMigrations.insert(migrationName);
+    };
+
+    if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations))
+        doUpgrade(
+            "20220326-ca-derivations",
+            #include "ca-specific-schema.sql.gen.hh"
+            );
+}
+
+
 /* To improve purity, users may want to make the Nix store a read-only
    bind mount.  So make the Nix store writable for this process. */
 void LocalStore::makeStoreWritable()
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 21848cc4d..83154d651 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -356,6 +356,8 @@ private:
 
     void openDB(State & state, bool create);
 
+    void upgradeDBSchema(State & state);
+
     void makeStoreWritable();
 
     uint64_t queryValidPathId(State & state, const StorePath & path);

From 59246349d5b227cb3e35fbaef6b5468fbd2d3ec7 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 8 Nov 2024 20:13:07 +0100
Subject: [PATCH 06/73] Make nix log command easier to copy

---
 src/libstore/build/derivation-goal.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 34ed16a38..3c0445fb6 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -991,7 +991,8 @@ Goal::Co DerivationGoal::buildDone()
                 auto nixLogCommand = experimentalFeatureSettings.isEnabled(Xp::NixCommand)
                     ? "nix log"
                     : "nix-store -l";
-                msg += fmt("For full logs, run '" ANSI_BOLD "%s %s" ANSI_NORMAL "'.",
+                // Don't put quotes around the command. This is copy-pasted a ton, so don't make that error prone.
+                msg += fmt("For full logs, run " ANSI_BOLD "%s %s" ANSI_NORMAL,
                     nixLogCommand,
                     worker.store.printStorePath(drvPath));
             }

From 4b44fa0f06d74caf7664a23a771774d093598824 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 8 Nov 2024 20:17:13 +0100
Subject: [PATCH 07/73] Make nix log command easy to copy on its own line

---
 src/libstore/build/derivation-goal.cc | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 3c0445fb6..794be1568 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -991,8 +991,10 @@ Goal::Co DerivationGoal::buildDone()
                 auto nixLogCommand = experimentalFeatureSettings.isEnabled(Xp::NixCommand)
                     ? "nix log"
                     : "nix-store -l";
-                // Don't put quotes around the command. This is copy-pasted a ton, so don't make that error prone.
-                msg += fmt("For full logs, run " ANSI_BOLD "%s %s" ANSI_NORMAL,
+                // The command is on a separate line for easy copying, such as with triple click.
+                // This message will be indented elsewhere, so removing the indentation before the
+                // command will not put it at the start of the line unfortunately.
+                msg += fmt("For full logs, run:\n  " ANSI_BOLD "%s %s" ANSI_NORMAL,
                     nixLogCommand,
                     worker.store.printStorePath(drvPath));
             }

From e194e27f8574c193b1979cad8beef9ba5d7d584d Mon Sep 17 00:00:00 2001
From: WxNzEMof <143541718+WxNzEMof@users.noreply.github.com>
Date: Thu, 25 Jan 2024 21:56:09 +0000
Subject: [PATCH 08/73] docker: Allow building for non-root user

Add options uid, gid, uname, and gname to docker.nix.

Setting these to e.g. 1000, 1000, "user", "user" will build an image
which runs and allows using Nix as that user.
---
 docker.nix | 50 +++++++++++++++++++++++++++++++++++---------------
 1 file changed, 35 insertions(+), 15 deletions(-)

diff --git a/docker.nix b/docker.nix
index bd16b71cd..19479e57c 100644
--- a/docker.nix
+++ b/docker.nix
@@ -9,6 +9,10 @@
 , maxLayers ? 100
 , nixConf ? {}
 , flake-registry ? null
+, uid ? 0
+, gid ? 0
+, uname ? "root"
+, gname ? "root"
 }:
 let
   defaultPkgs = with pkgs; [
@@ -50,6 +54,15 @@ let
       description = "Unprivileged account (don't use!)";
     };
 
+  } // lib.optionalAttrs (uid != 0) {
+    "${uname}" = {
+      uid = uid;
+      shell = "${pkgs.bashInteractive}/bin/bash";
+      home = "/home/${uname}";
+      gid = gid;
+      groups = [ "${gname}" ];
+      description = "Nix user";
+    };
   } // lib.listToAttrs (
     map
       (
@@ -70,6 +83,8 @@ let
     root.gid = 0;
     nixbld.gid = 30000;
     nobody.gid = 65534;
+  } // lib.optionalAttrs (gid != 0) {
+    "${gname}".gid = gid;
   };
 
   userToPasswd = (
@@ -150,6 +165,8 @@ let
     in
       "${n} = ${vStr}") (defaultNixConf // nixConf))) + "\n";
 
+  userHome = if uid == 0 then "/root" else "/home/${uname}";
+
   baseSystem =
     let
       nixpkgs = pkgs.path;
@@ -237,26 +254,26 @@ let
       mkdir -p $out/etc/nix
       cat $nixConfContentsPath > $out/etc/nix/nix.conf
 
-      mkdir -p $out/root
-      mkdir -p $out/nix/var/nix/profiles/per-user/root
+      mkdir -p $out${userHome}
+      mkdir -p $out/nix/var/nix/profiles/per-user/${uname}
 
       ln -s ${profile} $out/nix/var/nix/profiles/default-1-link
       ln -s $out/nix/var/nix/profiles/default-1-link $out/nix/var/nix/profiles/default
-      ln -s /nix/var/nix/profiles/default $out/root/.nix-profile
+      ln -s /nix/var/nix/profiles/default $out${userHome}/.nix-profile
 
-      ln -s ${channel} $out/nix/var/nix/profiles/per-user/root/channels-1-link
-      ln -s $out/nix/var/nix/profiles/per-user/root/channels-1-link $out/nix/var/nix/profiles/per-user/root/channels
+      ln -s ${channel} $out/nix/var/nix/profiles/per-user/${uname}/channels-1-link
+      ln -s $out/nix/var/nix/profiles/per-user/${uname}/channels-1-link $out/nix/var/nix/profiles/per-user/${uname}/channels
 
-      mkdir -p $out/root/.nix-defexpr
-      ln -s $out/nix/var/nix/profiles/per-user/root/channels $out/root/.nix-defexpr/channels
-      echo "${channelURL} ${channelName}" > $out/root/.nix-channels
+      mkdir -p $out${userHome}/.nix-defexpr
+      ln -s $out/nix/var/nix/profiles/per-user/${uname}/channels $out${userHome}/.nix-defexpr/channels
+      echo "${channelURL} ${channelName}" > $out${userHome}/.nix-channels
 
       mkdir -p $out/bin $out/usr/bin
       ln -s ${pkgs.coreutils}/bin/env $out/usr/bin/env
       ln -s ${pkgs.bashInteractive}/bin/bash $out/bin/sh
 
     '' + (lib.optionalString (flake-registry-path != null) ''
-      nixCacheDir="/root/.cache/nix"
+      nixCacheDir="${userHome}/.cache/nix"
       mkdir -p $out$nixCacheDir
       globalFlakeRegistryPath="$nixCacheDir/flake-registry.json"
       ln -s ${flake-registry-path} $out$globalFlakeRegistryPath
@@ -268,7 +285,7 @@ let
 in
 pkgs.dockerTools.buildLayeredImageWithNixDb {
 
-  inherit name tag maxLayers;
+  inherit name tag maxLayers uid gid uname gname;
 
   contents = [ baseSystem ];
 
@@ -279,25 +296,28 @@ pkgs.dockerTools.buildLayeredImageWithNixDb {
   fakeRootCommands = ''
     chmod 1777 tmp
     chmod 1777 var/tmp
+    chown -R ${toString uid}:${toString gid} .${userHome}
+    chown -R ${toString uid}:${toString gid} nix
   '';
 
   config = {
-    Cmd = [ "/root/.nix-profile/bin/bash" ];
+    Cmd = [ "${userHome}/.nix-profile/bin/bash" ];
+    User = "${toString uid}:${toString gid}";
     Env = [
-      "USER=root"
+      "USER=${uname}"
       "PATH=${lib.concatStringsSep ":" [
-        "/root/.nix-profile/bin"
+        "${userHome}/.nix-profile/bin"
         "/nix/var/nix/profiles/default/bin"
         "/nix/var/nix/profiles/default/sbin"
       ]}"
       "MANPATH=${lib.concatStringsSep ":" [
-        "/root/.nix-profile/share/man"
+        "${userHome}/.nix-profile/share/man"
         "/nix/var/nix/profiles/default/share/man"
       ]}"
       "SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"
       "GIT_SSL_CAINFO=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"
       "NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"
-      "NIX_PATH=/nix/var/nix/profiles/per-user/root/channels:/root/.nix-defexpr/channels"
+      "NIX_PATH=/nix/var/nix/profiles/per-user/${uname}/channels:${userHome}/.nix-defexpr/channels"
     ];
   };
 

From 1cfb226b7269b14c34b2ef42e4c501e1ba851bb4 Mon Sep 17 00:00:00 2001
From: WxNzEMof <143541718+WxNzEMof@users.noreply.github.com>
Date: Sun, 10 Nov 2024 21:31:02 +0000
Subject: [PATCH 09/73] tests/nixos: add nix-docker test

---
 tests/nixos/default.nix    |  2 ++
 tests/nixos/nix-docker.nix | 39 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+)
 create mode 100644 tests/nixos/nix-docker.nix

diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix
index 17bfdea38..c5f4a23aa 100644
--- a/tests/nixos/default.nix
+++ b/tests/nixos/default.nix
@@ -124,6 +124,8 @@ in
 
   nix-copy = runNixOSTestFor "x86_64-linux" ./nix-copy.nix;
 
+  nix-docker = runNixOSTestFor "x86_64-linux" ./nix-docker.nix;
+
   nssPreload = runNixOSTestFor "x86_64-linux" ./nss-preload.nix;
 
   githubFlakes = runNixOSTestFor "x86_64-linux" ./github-flakes.nix;
diff --git a/tests/nixos/nix-docker.nix b/tests/nixos/nix-docker.nix
new file mode 100644
index 000000000..5c21dfff6
--- /dev/null
+++ b/tests/nixos/nix-docker.nix
@@ -0,0 +1,39 @@
+# Test the container built by ../../docker.nix.
+
+{ lib, config, nixpkgs, hostPkgs, ... }:
+
+let
+  pkgs = config.nodes.machine.nixpkgs.pkgs;
+
+  nixImage = import ../../docker.nix {
+    inherit (config.nodes.machine.nixpkgs) pkgs;
+  };
+  nixUserImage = import ../../docker.nix {
+    inherit (config.nodes.machine.nixpkgs) pkgs;
+    name = "nix-user";
+    uid = 1000;
+    gid = 1000;
+    uname = "user";
+    gname = "user";
+  };
+
+in {
+  name = "nix-docker";
+
+  nodes.machine =
+    { config, lib, pkgs, ... }:
+    { virtualisation.diskSize = 4096;
+    };
+
+  testScript = { nodes }: ''
+    machine.succeed("mkdir -p /etc/containers")
+    machine.succeed("""echo '{"default":[{"type":"insecureAcceptAnything"}]}' > /etc/containers/policy.json""")
+
+    machine.succeed("${pkgs.podman}/bin/podman load -i ${nixImage}")
+    machine.succeed("${pkgs.podman}/bin/podman run --rm nix nix --version")
+
+    machine.succeed("${pkgs.podman}/bin/podman load -i ${nixUserImage}")
+    machine.succeed("${pkgs.podman}/bin/podman run --rm nix-user nix --version")
+    machine.succeed("[[ $(${pkgs.podman}/bin/podman run --rm nix-user stat -c %u /nix/store) = 1000 ]]")
+  '';
+}

From 1dda18ef0a3c6d109b6e9fc2e1c7f93c9c1a4471 Mon Sep 17 00:00:00 2001
From: WxNzEMof <143541718+WxNzEMof@users.noreply.github.com>
Date: Sun, 10 Nov 2024 21:31:32 +0000
Subject: [PATCH 10/73] doc/manual: add documentation for non-root container
 images

---
 .../source/installation/installing-docker.md   | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/doc/manual/source/installation/installing-docker.md b/doc/manual/source/installation/installing-docker.md
index 6f77d6a57..9354c1a72 100644
--- a/doc/manual/source/installation/installing-docker.md
+++ b/doc/manual/source/installation/installing-docker.md
@@ -57,3 +57,21 @@ $ nix build ./\#hydraJobs.dockerImage.x86_64-linux
 $ docker load -i ./result/image.tar.gz
 $ docker run -ti nix:2.5pre20211105
 ```
+
+# Docker image with non-root Nix
+
+If you would like to run Nix in a container under a user other than `root`,
+you can build an image with a non-root single-user installation of Nix
+by specifying the `uid`, `gid`, `uname`, and `gname` arguments to `docker.nix`:
+
+```console
+$ nix build --file docker.nix \
+    --arg uid 1000 \
+    --arg gid 1000 \
+    --argstr uname user \
+    --argstr gname user \
+    --argstr name nix-user \
+    --out-link nix-user.tar.gz
+$ docker load -i nix-user.tar.gz
+$ docker run -ti nix-user
+```

From 11d3b017cfdce506bf46edf7f11ba923a67adee9 Mon Sep 17 00:00:00 2001
From: WxNzEMof <143541718+WxNzEMof@users.noreply.github.com>
Date: Sun, 10 Nov 2024 23:20:31 +0000
Subject: [PATCH 11/73] tests/nixos: add more thorough nix-docker tests

---
 tests/nixos/nix-docker-test.sh | 47 ++++++++++++++++++++++++++++++++++
 tests/nixos/nix-docker.nix     | 20 ++++++++++++---
 2 files changed, 64 insertions(+), 3 deletions(-)
 create mode 100644 tests/nixos/nix-docker-test.sh

diff --git a/tests/nixos/nix-docker-test.sh b/tests/nixos/nix-docker-test.sh
new file mode 100644
index 000000000..1f65e1a94
--- /dev/null
+++ b/tests/nixos/nix-docker-test.sh
@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+# docker.nix test script. Runs inside a built docker.nix container.
+
+set -eEuo pipefail
+
+export NIX_CONFIG='substituters = http://cache:5000?trusted=1'
+
+cd /tmp
+
+# Test getting a fetched derivation
+test "$("$(nix-build -E '(import <nixpkgs> {}).hello')"/bin/hello)" = "Hello, world!"
+
+# Test building a simple derivation
+# shellcheck disable=SC2016
+nix-build -E '
+let
+  pkgs = import <nixpkgs> {};
+in
+builtins.derivation {
+  name = "test";
+  system = builtins.currentSystem;
+  builder = "${pkgs.bash}/bin/bash";
+  args = ["-c" "echo OK > $out"];
+}'
+test "$(cat result)" = OK
+
+# Ensure #!/bin/sh shebang works
+echo '#!/bin/sh' > ./shebang-test
+echo 'echo OK' >> ./shebang-test
+chmod +x ./shebang-test
+test "$(./shebang-test)" = OK
+
+# Ensure #!/usr/bin/env shebang works
+echo '#!/usr/bin/env bash' > ./shebang-test
+echo 'echo OK' >> ./shebang-test
+chmod +x ./shebang-test
+test "$(./shebang-test)" = OK
+
+# Test nix-shell
+{
+    echo '#!/usr/bin/env nix-shell'
+    echo '#! nix-shell -i bash'
+    echo '#! nix-shell -p hello'
+    echo 'hello'
+} > ./nix-shell-test
+chmod +x ./nix-shell-test
+test "$(./nix-shell-test)" = "Hello, world!"
diff --git a/tests/nixos/nix-docker.nix b/tests/nixos/nix-docker.nix
index 5c21dfff6..dfd508988 100644
--- a/tests/nixos/nix-docker.nix
+++ b/tests/nixos/nix-docker.nix
@@ -17,23 +17,37 @@ let
     gname = "user";
   };
 
+  containerTestScript = ./nix-docker-test.sh;
+
 in {
   name = "nix-docker";
 
-  nodes.machine =
-    { config, lib, pkgs, ... }:
-    { virtualisation.diskSize = 4096;
+  nodes =
+    { machine =
+        { config, lib, pkgs, ... }:
+        { virtualisation.diskSize = 4096;
+        };
+      cache =
+        { config, lib, pkgs, ... }:
+        { virtualisation.additionalPaths = [ pkgs.stdenv pkgs.hello ];
+          services.harmonia.enable = true;
+          networking.firewall.allowedTCPPorts = [ 5000 ];
+        };
     };
 
   testScript = { nodes }: ''
+    cache.wait_for_unit("harmonia.service")
+
     machine.succeed("mkdir -p /etc/containers")
     machine.succeed("""echo '{"default":[{"type":"insecureAcceptAnything"}]}' > /etc/containers/policy.json""")
 
     machine.succeed("${pkgs.podman}/bin/podman load -i ${nixImage}")
     machine.succeed("${pkgs.podman}/bin/podman run --rm nix nix --version")
+    machine.succeed("${pkgs.podman}/bin/podman run --rm -i nix < ${containerTestScript}")
 
     machine.succeed("${pkgs.podman}/bin/podman load -i ${nixUserImage}")
     machine.succeed("${pkgs.podman}/bin/podman run --rm nix-user nix --version")
+    machine.succeed("${pkgs.podman}/bin/podman run --rm -i nix-user < ${containerTestScript}")
     machine.succeed("[[ $(${pkgs.podman}/bin/podman run --rm nix-user stat -c %u /nix/store) = 1000 ]]")
   '';
 }

From a2e4a4c2384789d92b300959995a7d9d0e9d725f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 12 Nov 2024 19:26:39 +0100
Subject: [PATCH 12/73] callFunction: Use std::span

This is a bit safer than having a separate nrArgs argument.
---
 src/libexpr-c/nix_api_expr.cc |  2 +-
 src/libexpr/eval.cc           | 32 ++++++++++++++------------------
 src/libexpr/eval.hh           |  5 ++---
 src/libexpr/primops.cc        |  8 ++++----
 src/libflake/flake/flake.cc   |  2 +-
 5 files changed, 22 insertions(+), 27 deletions(-)

diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc
index 333e99460..6144a7986 100644
--- a/src/libexpr-c/nix_api_expr.cc
+++ b/src/libexpr-c/nix_api_expr.cc
@@ -67,7 +67,7 @@ nix_err nix_value_call_multi(nix_c_context * context, EvalState * state, nix_val
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        state->state.callFunction(fn->value, nargs, (nix::Value * *)args, value->value, nix::noPos);
+        state->state.callFunction(fn->value, {(nix::Value * *) args, nargs}, value->value, nix::noPos);
         state->state.forceValue(value->value, nix::noPos);
     }
     NIXC_CATCH_ERRS
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index e21f70553..6e82af1d8 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -588,14 +588,14 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
     if (isFunctor(v)) {
         try {
             Value & functor = *v.attrs()->find(sFunctor)->value;
-            Value * vp = &v;
+            Value * vp[] = {&v};
             Value partiallyApplied;
             // The first paramater is not user-provided, and may be
             // handled by code that is opaque to the user, like lib.const = x: y: y;
             // So preferably we show docs that are relevant to the
             // "partially applied" function returned by e.g. `const`.
             // We apply the first argument:
-            callFunction(functor, 1, &vp, partiallyApplied, noPos);
+            callFunction(functor, vp, partiallyApplied, noPos);
             auto _level = addCallDepth(noPos);
             return getDoc(partiallyApplied);
         }
@@ -1460,7 +1460,7 @@ void ExprLambda::eval(EvalState & state, Env & env, Value & v)
     v.mkLambda(&env, this);
 }
 
-void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const PosIdx pos)
+void EvalState::callFunction(Value & fun, std::span<Value *> args, Value & vRes, const PosIdx pos)
 {
     auto _level = addCallDepth(pos);
 
@@ -1475,7 +1475,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
     auto makeAppChain = [&]()
     {
         vRes = vCur;
-        for (size_t i = 0; i < nrArgs; ++i) {
+        for (size_t i = 0; i < args.size(); ++i) {
             auto fun2 = allocValue();
             *fun2 = vRes;
             vRes.mkPrimOpApp(fun2, args[i]);
@@ -1484,7 +1484,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
 
     const Attr * functor;
 
-    while (nrArgs > 0) {
+    while (args.size() > 0) {
 
         if (vCur.isLambda()) {
 
@@ -1587,15 +1587,14 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                 throw;
             }
 
-            nrArgs--;
-            args += 1;
+            args = args.subspan(1);
         }
 
         else if (vCur.isPrimOp()) {
 
             size_t argsLeft = vCur.primOp()->arity;
 
-            if (nrArgs < argsLeft) {
+            if (args.size() < argsLeft) {
                 /* We don't have enough arguments, so create a tPrimOpApp chain. */
                 makeAppChain();
                 return;
@@ -1607,15 +1606,14 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                 if (countCalls) primOpCalls[fn->name]++;
 
                 try {
-                    fn->fun(*this, vCur.determinePos(noPos), args, vCur);
+                    fn->fun(*this, vCur.determinePos(noPos), args.data(), vCur);
                 } catch (Error & e) {
                     if (fn->addTrace)
                         addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name);
                     throw;
                 }
 
-                nrArgs -= argsLeft;
-                args += argsLeft;
+                args = args.subspan(argsLeft);
             }
         }
 
@@ -1631,7 +1629,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
             auto arity = primOp->primOp()->arity;
             auto argsLeft = arity - argsDone;
 
-            if (nrArgs < argsLeft) {
+            if (args.size() < argsLeft) {
                 /* We still don't have enough arguments, so extend the tPrimOpApp chain. */
                 makeAppChain();
                 return;
@@ -1663,8 +1661,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                     throw;
                 }
 
-                nrArgs -= argsLeft;
-                args += argsLeft;
+                args = args.subspan(argsLeft);
             }
         }
 
@@ -1675,13 +1672,12 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
             Value * args2[] = {allocValue(), args[0]};
             *args2[0] = vCur;
             try {
-                callFunction(*functor->value, 2, args2, vCur, functor->pos);
+                callFunction(*functor->value, args2, vCur, functor->pos);
             } catch (Error & e) {
                 e.addTrace(positions[pos], "while calling a functor (an attribute set with a '__functor' attribute)");
                 throw;
             }
-            nrArgs--;
-            args++;
+            args = args.subspan(1);
         }
 
         else
@@ -1724,7 +1720,7 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v)
     for (size_t i = 0; i < args.size(); ++i)
         vArgs[i] = args[i]->maybeThunk(state, env);
 
-    state.callFunction(vFun, args.size(), vArgs.data(), v, pos);
+    state.callFunction(vFun, vArgs, v, pos);
 }
 
 
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 7fe70af31..b0c79ab86 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -690,13 +690,12 @@ public:
 
     bool isFunctor(Value & fun);
 
-    // FIXME: use std::span
-    void callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const PosIdx pos);
+    void callFunction(Value & fun, std::span<Value *> args, Value & vRes, const PosIdx pos);
 
     void callFunction(Value & fun, Value & arg, Value & vRes, const PosIdx pos)
     {
         Value * args[] = {&arg};
-        callFunction(fun, 1, args, vRes, pos);
+        callFunction(fun, args, vRes, pos);
     }
 
     /**
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 45d9f86ac..aea623435 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -724,7 +724,7 @@ static void prim_genericClosure(EvalState & state, const PosIdx pos, Value * * a
 
         /* Call the `operator' function with `e' as argument. */
         Value newElements;
-        state.callFunction(*op->value, 1, &e, newElements, noPos);
+        state.callFunction(*op->value, {&e, 1}, newElements, noPos);
         state.forceList(newElements, noPos, "while evaluating the return value of the `operator` passed to builtins.genericClosure");
 
         /* Add the values returned by the operator to the work set. */
@@ -2450,7 +2450,7 @@ bool EvalState::callPathFilter(
     // assert that type is not "unknown"
     Value * args []{&arg1, fileTypeToString(*this, st.type)};
     Value res;
-    callFunction(*filterFun, 2, args, res, pos);
+    callFunction(*filterFun, args, res, pos);
 
     return forceBool(res, pos, "while evaluating the return value of the path filter function");
 }
@@ -3487,7 +3487,7 @@ static void prim_foldlStrict(EvalState & state, const PosIdx pos, Value * * args
         for (auto [n, elem] : enumerate(args[2]->listItems())) {
             Value * vs []{vCur, elem};
             vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue();
-            state.callFunction(*args[0], 2, vs, *vCur, pos);
+            state.callFunction(*args[0], vs, *vCur, pos);
         }
         state.forceValue(v, pos);
     } else {
@@ -3637,7 +3637,7 @@ static void prim_sort(EvalState & state, const PosIdx pos, Value * * args, Value
 
         Value * vs[] = {a, b};
         Value vBool;
-        state.callFunction(*args[0], 2, vs, vBool, noPos);
+        state.callFunction(*args[0], vs, vBool, noPos);
         return state.forceBool(vBool, pos, "while evaluating the return value of the sorting function passed to builtins.sort");
     };
 
diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc
index edb76f861..19b622a34 100644
--- a/src/libflake/flake/flake.cc
+++ b/src/libflake/flake/flake.cc
@@ -816,7 +816,7 @@ void callFlake(EvalState & state,
     assert(vFetchFinalTree);
 
     Value * args[] = {vLocks, &vOverrides, *vFetchFinalTree};
-    state.callFunction(*vCallFlake, 3, args, vRes, noPos);
+    state.callFunction(*vCallFlake, args, vRes, noPos);
 }
 
 void initLib(const Settings & settings)

From 3e4a83f53b343a7fe2ba4faaf4e3932c99ee438b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 14 Nov 2024 16:12:14 +0100
Subject: [PATCH 13/73] Use range-based for

---
 src/libexpr/eval.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 6e82af1d8..0283a14d6 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -1475,10 +1475,10 @@ void EvalState::callFunction(Value & fun, std::span<Value *> args, Value & vRes,
     auto makeAppChain = [&]()
     {
         vRes = vCur;
-        for (size_t i = 0; i < args.size(); ++i) {
+        for (auto arg : args) {
             auto fun2 = allocValue();
             *fun2 = vRes;
-            vRes.mkPrimOpApp(fun2, args[i]);
+            vRes.mkPrimOpApp(fun2, arg);
         }
     };
 

From 2f3764acbb96ab687e612fdd81bdf58ac5f0e605 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 15 Nov 2024 11:35:27 +0100
Subject: [PATCH 14/73] .github/ci: Add nix-docker test

We still have room to spare in vm_tests, as it's quicker than `nix flake check`
---
 .github/workflows/ci.yml | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 27f60574e..a3b7b06d4 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -194,7 +194,13 @@ jobs:
       - uses: actions/checkout@v4
       - uses: DeterminateSystems/nix-installer-action@main
       - uses: DeterminateSystems/magic-nix-cache-action@main
-      - run: nix build -L .#hydraJobs.tests.githubFlakes .#hydraJobs.tests.tarballFlakes .#hydraJobs.tests.functional_user
+      - run: |
+          nix build -L \
+            .#hydraJobs.tests.functional_user \
+            .#hydraJobs.tests.githubFlakes \
+            .#hydraJobs.tests.nix-docker \
+            .#hydraJobs.tests.tarballFlakes \
+            ;
 
   flake_regressions:
     needs: vm_tests

From c9433c0d1834362305c4f5ca2802fca97d999044 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 15 Nov 2024 11:37:17 +0100
Subject: [PATCH 15/73] .github/ci: Push docker only when test succeeds

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a3b7b06d4..cb97fd211 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -128,7 +128,7 @@ jobs:
     - run: exec bash -c "nix-channel --update && nix-env -iA nixpkgs.hello && hello"
 
   docker_push_image:
-    needs: [check_secrets, tests]
+    needs: [check_secrets, tests, vm_tests]
     permissions:
       contents: read
       packages: write

From 3f6855c31b8d35b39148a86c2c28c7e2f367b739 Mon Sep 17 00:00:00 2001
From: myclevorname <140354451+myclevorname@users.noreply.github.com>
Date: Sun, 17 Nov 2024 14:12:27 -0500
Subject: [PATCH 16/73] doc/nix fmt: Mention nixfmt-rfc-style instead of
 nixfmt(-classic)

---
 src/nix/fmt.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/nix/fmt.md b/src/nix/fmt.md
index a2afde61c..b4693eb65 100644
--- a/src/nix/fmt.md
+++ b/src/nix/fmt.md
@@ -22,13 +22,13 @@ With [nixpkgs-fmt](https://github.com/nix-community/nixpkgs-fmt):
 }
 ```
 
-With [nixfmt](https://github.com/serokell/nixfmt):
+With [nixfmt](https://github.com/NixOS/nixfmt):
 
 ```nix
 # flake.nix
 {
   outputs = { nixpkgs, self }: {
-    formatter.x86_64-linux = nixpkgs.legacyPackages.x86_64-linux.nixfmt;
+    formatter.x86_64-linux = nixpkgs.legacyPackages.x86_64-linux.nixfmt-rfc-style;
   };
 }
 ```

From d65fac0fc461e5b62373b86401b845df6c698177 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 18 Nov 2024 15:08:32 +0100
Subject: [PATCH 17/73] Add --print-errorlogs to mesonCheckFlags

This prints the error logs in the tests, including when they're run
with `checkPhase` in the dev shell.
---
 packaging/dependencies.nix | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix
index 13766f2c0..a8005ce16 100644
--- a/packaging/dependencies.nix
+++ b/packaging/dependencies.nix
@@ -70,6 +70,9 @@ let
         pkgs.buildPackages.meson
         pkgs.buildPackages.ninja
       ] ++ prevAttrs.nativeBuildInputs or [];
+      mesonCheckFlags = prevAttrs.mesonCheckFlags or [] ++ [
+        "--print-errorlogs"
+      ];
     };
 
   mesonBuildLayer = finalAttrs: prevAttrs:

From 428af8c66f0918f5852080b06268498e5021bfe8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 18 Nov 2024 16:28:12 +0100
Subject: [PATCH 18/73] tests/functional/flakes/develop.sh: Don't hang

The bash shell started by `nix develop` waited forever for stdin
input.

Fixes #11827.
---
 tests/functional/flakes/develop.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/functional/flakes/develop.sh b/tests/functional/flakes/develop.sh
index b3e438e99..c222f0fbb 100755
--- a/tests/functional/flakes/develop.sh
+++ b/tests/functional/flakes/develop.sh
@@ -122,7 +122,7 @@ expectStderr 1 nix develop --unset-env-var FOO --set-env-var FOO 'BAR' --no-writ
   grepQuiet "error: Cannot set environment variable 'FOO' that is unset with '--unset-env-var'"
 
 # Check that multiple `--ignore-env`'s are okay.
-expectStderr 0 nix develop --ignore-env --set-env-var FOO 'BAR' --ignore-env .#hello
+expectStderr 0 nix develop --ignore-env --set-env-var FOO 'BAR' --ignore-env .#hello < /dev/null
 
 # Determine the bashInteractive executable.
 nix build --no-write-lock-file './nixpkgs#bashInteractive' --out-link ./bash-interactive

From c4b95dbdd1fb45bbc7ad1fc921ebdf81789e22b3 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 18 Nov 2024 16:45:18 +0100
Subject: [PATCH 19/73] Fix issue 11892

It seems that I copied the expression for baseDir thoughtlessly and
did not come back to it.

- `baseDir` was only used in the `fromArgs` branch.
- `fromArgs` is true when `packages` is true.
---
 src/nix-build/nix-build.cc    | 10 ++++++----
 tests/functional/nix-shell.sh | 29 +++++++++++++++++++++++++++++
 tests/functional/shell.nix    |  6 +++++-
 3 files changed, 40 insertions(+), 5 deletions(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index c394836da..de01e1afc 100644
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -340,13 +340,15 @@ static void main_nix_build(int argc, char * * argv)
         exprs = {state->parseStdin()};
     else
         for (auto i : remainingArgs) {
-            auto baseDir = inShebang && !packages ? absPath(dirOf(script)) : i;
-
-            if (fromArgs)
+            if (fromArgs) {
+                auto shebangBaseDir = absPath(dirOf(script));
                 exprs.push_back(state->parseExprFromString(
                     std::move(i),
-                    (inShebang && compatibilitySettings.nixShellShebangArgumentsRelativeToScript) ? lookupFileArg(*state, baseDir) : state->rootPath(".")
+                    (inShebang && compatibilitySettings.nixShellShebangArgumentsRelativeToScript)
+                        ? lookupFileArg(*state, shebangBaseDir)
+                        : state->rootPath(".")
                 ));
+            }
             else {
                 auto absolute = i;
                 try {
diff --git a/tests/functional/nix-shell.sh b/tests/functional/nix-shell.sh
index 2b78216f4..b054b7f75 100755
--- a/tests/functional/nix-shell.sh
+++ b/tests/functional/nix-shell.sh
@@ -167,6 +167,35 @@ EOF
 chmod a+x $TEST_ROOT/marco/polo/default.nix
 (cd $TEST_ROOT/marco && ./polo/default.nix | grepQuiet "Polo")
 
+# https://github.com/NixOS/nix/issues/11892
+mkdir $TEST_ROOT/issue-11892
+cat >$TEST_ROOT/issue-11892/shebangscript <<EOF
+#!$(type -P env) nix-shell
+#! nix-shell -I nixpkgs=$shellDotNix
+#! nix-shell -p 'callPackage (import ./my_package.nix) {}'
+#! nix-shell -i bash
+set -euxo pipefail
+my_package
+EOF
+cat >$TEST_ROOT/issue-11892/my_package.nix <<EOF
+{ stdenv, shell, ... }:
+stdenv.mkDerivation {
+  name = "my_package";
+  buildCommand = ''
+    mkdir -p \$out/bin
+    ( echo "#!\${shell}"
+      echo "echo 'ok' 'baz11892'"
+    ) > \$out/bin/my_package
+    cat \$out/bin/my_package
+    chmod a+x \$out/bin/my_package
+  '';
+}
+EOF
+chmod a+x $TEST_ROOT/issue-11892/shebangscript
+$TEST_ROOT/issue-11892/shebangscript \
+  | tee /dev/stderr \
+  | grepQuiet "ok baz11892"
+
 
 #####################
 # Flake equivalents #
diff --git a/tests/functional/shell.nix b/tests/functional/shell.nix
index 9cae14b78..4b1a0623a 100644
--- a/tests/functional/shell.nix
+++ b/tests/functional/shell.nix
@@ -37,7 +37,7 @@ let pkgs = rec {
       mkdir -p $out
       ln -s ${setupSh} $out/setup
     '';
-  };
+  } // { inherit mkDerivation; };
 
   shellDrv = mkDerivation {
     name = "shellDrv";
@@ -94,5 +94,9 @@ let pkgs = rec {
     chmod a+rx $out/bin/ruby
   '';
 
+  inherit (cfg) shell;
+
+  callPackage = f: args: f (pkgs // args);
+
   inherit pkgs;
 }; in pkgs

From e224a35a77b588aed2bc29bf36cc4274098e7a5f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Tue, 19 Nov 2024 11:25:26 +0100
Subject: [PATCH 20/73] docs/flake: document how to build a pull request

It's not so common knowledge that forges also expose pull requests as
git refs. But it's actually a cool way of quickly testing someones
contribution, so I found it worth specifically mentioning it.
---
 src/nix/flake.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/nix/flake.md b/src/nix/flake.md
index 2b999431c..8f0f9936c 100644
--- a/src/nix/flake.md
+++ b/src/nix/flake.md
@@ -84,6 +84,8 @@ Here are some examples of flake references in their URL-like representation:
   repository on GitHub.
 * `github:NixOS/nixpkgs/nixos-20.09`: The `nixos-20.09` branch of the
   `nixpkgs` repository.
+* `github:NixOS/nixpkgs/pull/357207/head`: The `357207` pull request
+   of the nixpkgs repository.
 * `github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293`: A
   specific revision of the `nixpkgs` repository.
 * `github:edolstra/nix-warez?dir=blender`: A flake in a subdirectory

From dd4838974eb8afbde4cf3dc60ffd327084affb06 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Tue, 19 Nov 2024 13:24:11 +0100
Subject: [PATCH 21/73] document shallow clone options in git fetchers

---
 src/nix/flake.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/nix/flake.md b/src/nix/flake.md
index 8f0f9936c..5412afcdd 100644
--- a/src/nix/flake.md
+++ b/src/nix/flake.md
@@ -245,6 +245,9 @@ Currently the `type` attribute can be one of the following:
   * `./sub/dir` (when used on the command line and `dir/flake.nix` is in a git repository)
   * `git+https://example.org/my/repo`
   * `git+https://example.org/my/repo?dir=flake1`
+  * `git+https://example.org/my/repo?shallow=1` A shallow clone of the repository.
+     For large repositories, the shallow clone option can significantly speed up fresh clones compared
+     to non-shallow clones, while still providing faster updates than other fetch methods such as `tarball:` or `github:`.
   * `git+ssh://git@github.com/NixOS/nix?ref=v1.2.3`
   * `git://github.com/edolstra/dwarffs?ref=unstable&rev=e486d8d40e626a20e06d792db8cc5ac5aba9a5b4`
   * `git+file:///home/my-user/some-repo/some-repo`

From 850281908cd65b7ccfdfe17b1e4a43f8ec59ef9a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 19 Nov 2024 16:50:13 +0100
Subject: [PATCH 22/73] Clean up flakeref parsing

This factors out some commonality in calling fromURL() and handling
the "dir" parameter into a fromParsedURL() helper function.
---
 src/libflake/flake/flakeref.cc | 43 ++++++++++++++++++----------------
 1 file changed, 23 insertions(+), 20 deletions(-)

diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc
index 01fe747f9..ed47ad737 100644
--- a/src/libflake/flake/flakeref.cc
+++ b/src/libflake/flake/flakeref.cc
@@ -67,6 +67,20 @@ std::optional<FlakeRef> maybeParseFlakeRef(
     }
 }
 
+static std::pair<FlakeRef, std::string> fromParsedURL(
+    const fetchers::Settings & fetchSettings,
+    ParsedURL && parsedURL,
+    bool isFlake)
+{
+    auto dir = getOr(parsedURL.query, "dir", "");
+    parsedURL.query.erase("dir");
+
+    std::string fragment;
+    std::swap(fragment, parsedURL.fragment);
+
+    return std::make_pair(FlakeRef(fetchers::Input::fromURL(fetchSettings, parsedURL, isFlake), dir), fragment);
+}
+
 std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
     const fetchers::Settings & fetchSettings,
     const std::string & url,
@@ -89,7 +103,7 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
         fragment = percentDecode(url.substr(fragmentStart+1));
     }
     if (pathEnd != std::string::npos && fragmentStart != std::string::npos && url[pathEnd] == '?') {
-        query = decodeQuery(url.substr(pathEnd+1, fragmentStart-pathEnd-1));
+        query = decodeQuery(url.substr(pathEnd + 1, fragmentStart - pathEnd - 1));
     }
 
     if (baseDir) {
@@ -153,6 +167,7 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
                         .authority = "",
                         .path = flakeRoot,
                         .query = query,
+                        .fragment = fragment,
                     };
 
                     if (subdir != "") {
@@ -164,9 +179,7 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
                     if (pathExists(flakeRoot + "/.git/shallow"))
                         parsedURL.query.insert_or_assign("shallow", "1");
 
-                    return std::make_pair(
-                        FlakeRef(fetchers::Input::fromURL(fetchSettings, parsedURL), getOr(parsedURL.query, "dir", "")),
-                        fragment);
+                    return fromParsedURL(fetchSettings, std::move(parsedURL), isFlake);
                 }
 
                 subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir);
@@ -185,11 +198,12 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
     attrs.insert_or_assign("path", path);
 
     return std::make_pair(FlakeRef(fetchers::Input::fromAttrs(fetchSettings, std::move(attrs)), ""), fragment);
-};
+}
 
-
-/* Check if 'url' is a flake ID. This is an abbreviated syntax for
-    'flake:<flake-id>?ref=<ref>&rev=<rev>'. */
+/**
+ * Check if `url` is a flake ID. This is an abbreviated syntax for
+ * `flake:<flake-id>?ref=<ref>&rev=<rev>`.
+ */
 static std::optional<std::pair<FlakeRef, std::string>> parseFlakeIdRef(
     const fetchers::Settings & fetchSettings,
     const std::string & url,
@@ -227,22 +241,11 @@ std::optional<std::pair<FlakeRef, std::string>> parseURLFlakeRef(
     bool isFlake
 )
 {
-    ParsedURL parsedURL;
     try {
-        parsedURL = parseURL(url);
+        return fromParsedURL(fetchSettings, parseURL(url), isFlake);
     } catch (BadURL &) {
         return std::nullopt;
     }
-
-    std::string fragment;
-    std::swap(fragment, parsedURL.fragment);
-
-    auto input = fetchers::Input::fromURL(fetchSettings, parsedURL, isFlake);
-    input.parent = baseDir;
-
-    return std::make_pair(
-        FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")),
-        fragment);
 }
 
 std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(

From 868b4d37ea8e59d76afbaef4c82315e23b5fa8f4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 19 Nov 2024 16:59:38 +0100
Subject: [PATCH 23/73] nix flake init: Operate on a SourcePath

Cherry-picked from lazy-trees.
---
 src/nix/flake.cc | 45 ++++++++++++++++++++-------------------------
 1 file changed, 20 insertions(+), 25 deletions(-)

diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index 3a54763a1..ce2faacb0 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -891,37 +891,32 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand
 
         auto cursor = installable.getCursor(*evalState);
 
-        auto templateDirAttr = cursor->getAttr("path");
-        auto templateDir = templateDirAttr->getString();
-
-        if (!store->isInStore(templateDir))
-            evalState->error<TypeError>(
-                "'%s' was not found in the Nix store\n"
-                "If you've set '%s' to a string, try using a path instead.",
-                templateDir, templateDirAttr->getAttrPathStr()).debugThrow();
+        auto templateDirAttr = cursor->getAttr("path")->forceValue();
+        NixStringContext context;
+        auto templateDir = evalState->coerceToPath(noPos, templateDirAttr, context, "");
 
         std::vector<fs::path> changedFiles;
         std::vector<fs::path> conflictedFiles;
 
-        std::function<void(const fs::path & from, const fs::path & to)> copyDir;
-        copyDir = [&](const fs::path & from, const fs::path & to)
+        std::function<void(const SourcePath & from, const fs::path & to)> copyDir;
+        copyDir = [&](const SourcePath & from, const fs::path & to)
         {
             fs::create_directories(to);
 
-            for (auto & entry : fs::directory_iterator{from}) {
+            for (auto & [name, entry] : from.readDirectory()) {
                 checkInterrupt();
-                auto from2 = entry.path();
-                auto to2 = to / entry.path().filename();
-                auto st = entry.symlink_status();
+                auto from2 = from / name;
+                auto to2 = to / name;
+                auto st = from2.lstat();
                 auto to_st = fs::symlink_status(to2);
-                if (fs::is_directory(st))
+                if (st.type == SourceAccessor::tDirectory)
                     copyDir(from2, to2);
-                else if (fs::is_regular_file(st)) {
-                    auto contents = readFile(from2.string());
+                else if (st.type == SourceAccessor::tRegular) {
+                    auto contents = from2.readFile();
                     if (fs::exists(to_st)) {
                         auto contents2 = readFile(to2.string());
                         if (contents != contents2) {
-                            printError("refusing to overwrite existing file '%s'\n please merge it manually with '%s'", to2.string(), from2.string());
+                            printError("refusing to overwrite existing file '%s'\n please merge it manually with '%s'", to2.string(), from2);
                             conflictedFiles.push_back(to2);
                         } else {
                             notice("skipping identical file: %s", from2);
@@ -930,18 +925,18 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand
                     } else
                         writeFile(to2, contents);
                 }
-                else if (fs::is_symlink(st)) {
-                    auto target = fs::read_symlink(from2);
+                else if (st.type == SourceAccessor::tSymlink) {
+                    auto target = from2.readLink();
                     if (fs::exists(to_st)) {
                         if (fs::read_symlink(to2) != target) {
-                            printError("refusing to overwrite existing file '%s'\n please merge it manually with '%s'", to2.string(), from2.string());
+                            printError("refusing to overwrite existing file '%s'\n please merge it manually with '%s'", to2.string(), from2);
                             conflictedFiles.push_back(to2);
                         } else {
                             notice("skipping identical file: %s", from2);
                         }
                         continue;
                     } else
-                          fs::create_symlink(target, to2);
+                        fs::create_symlink(target, to2);
                 }
                 else
                     throw Error("file '%s' has unsupported type", from2);
@@ -957,14 +952,14 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand
             for (auto & s : changedFiles) args.emplace_back(s.string());
             runProgram("git", true, args);
         }
-        auto welcomeText = cursor->maybeGetAttr("welcomeText");
-        if (welcomeText) {
+
+        if (auto welcomeText = cursor->maybeGetAttr("welcomeText")) {
             notice("\n");
             notice(renderMarkdownToTerminal(welcomeText->getString()));
         }
 
         if (!conflictedFiles.empty())
-            throw Error("Encountered %d conflicts - see above", conflictedFiles.size());
+            throw Error("encountered %d conflicts - see above", conflictedFiles.size());
     }
 };
 

From f1b4f14055077e660b7aa5b4859ce6965b62b886 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 19 Nov 2024 17:30:38 +0100
Subject: [PATCH 24/73] Trivial changes from lazy-trees

---
 src/libexpr/primops.cc            | 7 ++++---
 src/libfetchers/fetch-to-store.cc | 2 ++
 src/libfetchers/meson.build       | 6 +++---
 3 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index aea623435..42136b467 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1603,7 +1603,8 @@ static RegisterPrimOp primop_placeholder({
  *************************************************************/
 
 
-/* Convert the argument to a path.  !!! obsolete? */
+/* Convert the argument to a path and then to a string (confusing,
+   eh?).  !!! obsolete? */
 static void prim_toPath(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
     NixStringContext context;
@@ -2614,13 +2615,13 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value
             expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), HashAlgorithm::SHA256);
         else
             state.error<EvalError>(
-                "unsupported argument '%1%' to 'addPath'",
+                "unsupported argument '%1%' to 'builtins.path'",
                 state.symbols[attr.name]
             ).atPos(attr.pos).debugThrow();
     }
     if (!path)
         state.error<EvalError>(
-            "missing required 'path' attribute in the first argument to builtins.path"
+            "missing required 'path' attribute in the first argument to 'builtins.path'"
         ).atPos(pos).debugThrow();
     if (name.empty())
         name = path->baseName();
diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc
index 65aa72a6c..fe347a59d 100644
--- a/src/libfetchers/fetch-to-store.cc
+++ b/src/libfetchers/fetch-to-store.cc
@@ -44,6 +44,8 @@ StorePath fetchToStore(
         : store.addToStore(
             name, path, method, HashAlgorithm::SHA256, {}, filter2, repair);
 
+    debug(mode == FetchMode::DryRun ? "hashed '%s'" : "copied '%s' to '%s'", path, store.printStorePath(storePath));
+
     if (cacheKey && mode == FetchMode::Copy)
         fetchers::getCache()->upsert(*cacheKey, store, {}, storePath);
 
diff --git a/src/libfetchers/meson.build b/src/libfetchers/meson.build
index d4f202796..ff638578d 100644
--- a/src/libfetchers/meson.build
+++ b/src/libfetchers/meson.build
@@ -52,15 +52,15 @@ sources = files(
   'fetch-to-store.cc',
   'fetchers.cc',
   'filtering-source-accessor.cc',
-  'git.cc',
   'git-utils.cc',
+  'git.cc',
   'github.cc',
   'indirect.cc',
   'mercurial.cc',
   'mounted-source-accessor.cc',
   'path.cc',
-  'store-path-accessor.cc',
   'registry.cc',
+  'store-path-accessor.cc',
   'tarball.cc',
 )
 
@@ -71,10 +71,10 @@ headers = files(
   'cache.hh',
   'fetch-settings.hh',
   'fetch-to-store.hh',
+  'fetchers.hh',
   'filtering-source-accessor.hh',
   'git-utils.hh',
   'mounted-source-accessor.hh',
-  'fetchers.hh',
   'registry.hh',
   'store-path-accessor.hh',
   'tarball.hh',

From a58e38dab7f6a9b9d217f6163fe8b565f19a6405 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Tue, 19 Nov 2024 17:30:58 +0100
Subject: [PATCH 25/73] Make EvalState::getBuiltin safe for missing attr

---
 src/libexpr/eval.cc | 6 +++++-
 src/libexpr/eval.hh | 5 +++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 0283a14d6..dd14f485e 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -525,7 +525,11 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
 
 Value & EvalState::getBuiltin(const std::string & name)
 {
-    return *baseEnv.values[0]->attrs()->find(symbols.create(name))->value;
+    auto it = baseEnv.values[0]->attrs()->get(symbols.create(name));
+    if (it)
+        return *it->value;
+    else
+        throw EvalError("builtin '%1%' not found", name);
 }
 
 
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 742f1cafe..01c725930 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -623,6 +623,11 @@ private:
 
 public:
 
+    /**
+     * Retrieve a specific builtin, equivalent to evaluating `builtins.${name}`.
+     * @param name The attribute name of the builtin to retrieve.
+     * @throws EvalError if the builtin does not exist.
+     */
     Value & getBuiltin(const std::string & name);
 
     struct Doc

From af07f33d37410de8c97bb64f2d77214c1f4640b4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 19 Nov 2024 18:03:31 +0100
Subject: [PATCH 26/73] resolveLookupPathPath(): Return a SourcePath instead of
 a string

Cherry-picked from lazy-trees.
---
 src/libcmd/common-eval-args.cc |  6 +++---
 src/libexpr/eval-settings.hh   |  8 +++-----
 src/libexpr/eval.cc            | 23 +++++++++++------------
 src/libexpr/eval.hh            |  6 +++---
 tests/functional/restricted.sh |  2 +-
 5 files changed, 21 insertions(+), 24 deletions(-)

diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc
index ccbf957d9..de967e3fe 100644
--- a/src/libcmd/common-eval-args.cc
+++ b/src/libcmd/common-eval-args.cc
@@ -29,13 +29,13 @@ EvalSettings evalSettings {
     {
         {
             "flake",
-            [](ref<Store> store, std::string_view rest) {
+            [](EvalState & state, std::string_view rest) {
                 experimentalFeatureSettings.require(Xp::Flakes);
                 // FIXME `parseFlakeRef` should take a `std::string_view`.
                 auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false);
                 debug("fetching flake search path element '%s''", rest);
-                auto storePath = flakeRef.resolve(store).fetchTree(store).first;
-                return store->toRealPath(storePath);
+                auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first;
+                return state.rootPath(state.store->toRealPath(storePath));
             },
         },
     },
diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh
index 115e3ee50..a8fcce539 100644
--- a/src/libexpr/eval-settings.hh
+++ b/src/libexpr/eval-settings.hh
@@ -3,10 +3,11 @@
 
 #include "config.hh"
 #include "ref.hh"
+#include "source-path.hh"
 
 namespace nix {
 
-class Store;
+class EvalState;
 
 struct EvalSettings : Config
 {
@@ -18,11 +19,8 @@ struct EvalSettings : Config
      *
      * The return value is (a) whether the entry was valid, and, if so,
      * what does it map to.
-     *
-     * @todo Return (`std::optional` of) `SourceAccssor` or something
-     * more structured instead of mere `std::string`?
      */
-    using LookupPathHook = std::optional<std::string>(ref<Store> store, std::string_view);
+    using LookupPathHook = std::optional<SourcePath>(EvalState & state, std::string_view);
 
     /**
      * Map from "scheme" to a `LookupPathHook`.
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 0283a14d6..83a1cf4e9 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -3025,8 +3025,8 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_
         if (!rOpt) continue;
         auto r = *rOpt;
 
-        Path res = suffix == "" ? r : concatStrings(r, "/", suffix);
-        if (pathExists(res)) return rootPath(CanonPath(canonPath(res)));
+        auto res = (r / CanonPath(suffix)).resolveSymlinks();
+        if (res.pathExists()) return res;
     }
 
     if (hasPrefix(path, "nix/"))
@@ -3041,13 +3041,13 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_
 }
 
 
-std::optional<std::string> EvalState::resolveLookupPathPath(const LookupPath::Path & value0, bool initAccessControl)
+std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Path & value0, bool initAccessControl)
 {
     auto & value = value0.s;
     auto i = lookupPathResolved.find(value);
     if (i != lookupPathResolved.end()) return i->second;
 
-    auto finish = [&](std::string res) {
+    auto finish = [&](SourcePath res) {
         debug("resolved search path element '%s' to '%s'", value, res);
         lookupPathResolved.emplace(value, res);
         return res;
@@ -3060,7 +3060,7 @@ std::optional<std::string> EvalState::resolveLookupPathPath(const LookupPath::Pa
                 fetchSettings,
                 EvalSettings::resolvePseudoUrl(value));
             auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy);
-            return finish(store->toRealPath(storePath));
+            return finish(rootPath(store->toRealPath(storePath)));
         } catch (Error & e) {
             logWarning({
                 .msg = HintFmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value)
@@ -3072,29 +3072,29 @@ std::optional<std::string> EvalState::resolveLookupPathPath(const LookupPath::Pa
         auto scheme = value.substr(0, colPos);
         auto rest = value.substr(colPos + 1);
         if (auto * hook = get(settings.lookupPathHooks, scheme)) {
-            auto res = (*hook)(store, rest);
+            auto res = (*hook)(*this, rest);
             if (res)
                 return finish(std::move(*res));
         }
     }
 
     {
-        auto path = absPath(value);
+        auto path = rootPath(value);
 
         /* Allow access to paths in the search path. */
         if (initAccessControl) {
-            allowPath(path);
-            if (store->isInStore(path)) {
+            allowPath(path.path.abs());
+            if (store->isInStore(path.path.abs())) {
                 try {
                     StorePathSet closure;
-                    store->computeFSClosure(store->toStorePath(path).first, closure);
+                    store->computeFSClosure(store->toStorePath(path.path.abs()).first, closure);
                     for (auto & p : closure)
                         allowPath(p);
                 } catch (InvalidPath &) { }
             }
         }
 
-        if (pathExists(path))
+        if (path.pathExists())
             return finish(std::move(path));
         else {
             logWarning({
@@ -3105,7 +3105,6 @@ std::optional<std::string> EvalState::resolveLookupPathPath(const LookupPath::Pa
 
     debug("failed to resolve search path element '%s'", value);
     return std::nullopt;
-
 }
 
 
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 742f1cafe..6c0bae451 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -347,7 +347,7 @@ private:
 
     LookupPath lookupPath;
 
-    std::map<std::string, std::optional<std::string>> lookupPathResolved;
+    std::map<std::string, std::optional<SourcePath>> lookupPathResolved;
 
     /**
      * Cache used by prim_match().
@@ -452,9 +452,9 @@ public:
      *
      * If the specified search path element is a URI, download it.
      *
-     * If it is not found, return `std::nullopt`
+     * If it is not found, return `std::nullopt`.
      */
-    std::optional<std::string> resolveLookupPathPath(
+    std::optional<SourcePath> resolveLookupPathPath(
         const LookupPath::Path & elem,
         bool initAccessControl = false);
 
diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh
index 00ee4ddc8..a92a9b8a3 100755
--- a/tests/functional/restricted.sh
+++ b/tests/functional/restricted.sh
@@ -23,7 +23,7 @@ nix-instantiate --restrict-eval ./simple.nix -I src1=./simple.nix -I src2=./conf
 (! nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix')
 nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I src=../..
 
-expectStderr 1 nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile <foo/simple.nix>' | grepQuiet "forbidden in restricted mode"
+expectStderr 1 nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile <foo/simple.nix>' | grepQuiet "was not found in the Nix search path"
 nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile <foo/simple.nix>' -I src=.
 
 p=$(nix eval --raw --expr "builtins.fetchurl file://${_NIX_TEST_SOURCE_DIR}/restricted.sh" --impure --restrict-eval --allowed-uris "file://${_NIX_TEST_SOURCE_DIR}")

From 8a36d2d8a77dc3f3214ac6f7fd67cbe15ec6c706 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Tue, 19 Nov 2024 18:23:05 +0100
Subject: [PATCH 27/73] Add EvalState::getBuiltins

---
 src/libexpr-test-support/tests/libexpr.hh |  6 ++++++
 src/libexpr-tests/eval.cc                 | 25 ++++++++++++++++++++++-
 src/libexpr/eval.cc                       |  8 +++++++-
 src/libexpr/eval.hh                       |  6 ++++++
 4 files changed, 43 insertions(+), 2 deletions(-)

diff --git a/src/libexpr-test-support/tests/libexpr.hh b/src/libexpr-test-support/tests/libexpr.hh
index 045607e87..095ea1d0e 100644
--- a/src/libexpr-test-support/tests/libexpr.hh
+++ b/src/libexpr-test-support/tests/libexpr.hh
@@ -40,6 +40,12 @@ namespace nix {
                 return v;
             }
 
+            Value * maybeThunk(std::string input, bool forceValue = true) {
+                Expr * e = state.parseExprFromString(input, state.rootPath(CanonPath::root));
+                assert(e);
+                return e->maybeThunk(state, state.baseEnv);
+            }
+
             Symbol createSymbol(const char * value) {
                 return state.symbols.create(value);
             }
diff --git a/src/libexpr-tests/eval.cc b/src/libexpr-tests/eval.cc
index 93d3f658f..61f6be0db 100644
--- a/src/libexpr-tests/eval.cc
+++ b/src/libexpr-tests/eval.cc
@@ -138,4 +138,27 @@ TEST(nix_isAllowedURI, non_scheme_colon) {
     ASSERT_FALSE(isAllowedURI("https://foo/bar:baz", allowed));
 }
 
-} // namespace nix
\ No newline at end of file
+class EvalStateTest : public LibExprTest {};
+
+TEST_F(EvalStateTest, getBuiltins_ok) {
+    auto evaled = maybeThunk("builtins");
+    auto & builtins = state.getBuiltins();
+    ASSERT_TRUE(builtins.type() == nAttrs);
+    ASSERT_EQ(evaled, &builtins);
+}
+
+TEST_F(EvalStateTest, getBuiltin_ok) {
+    auto & builtin = state.getBuiltin("toString");
+    ASSERT_TRUE(builtin.type() == nFunction);
+    // FIXME
+    // auto evaled = maybeThunk("builtins.toString");
+    // ASSERT_EQ(evaled, &builtin);
+    auto & builtin2 = state.getBuiltin("true");
+    ASSERT_EQ(state.forceBool(builtin2, noPos, "in unit test"), true);
+}
+
+TEST_F(EvalStateTest, getBuiltin_fail) {
+    ASSERT_THROW(state.getBuiltin("nonexistent"), EvalError);
+}
+
+} // namespace nix
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index dd14f485e..b1b7a0fe6 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -523,13 +523,19 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
 }
 
 
+Value & EvalState::getBuiltins()
+{
+    return *baseEnv.values[0];
+}
+
+
 Value & EvalState::getBuiltin(const std::string & name)
 {
     auto it = baseEnv.values[0]->attrs()->get(symbols.create(name));
     if (it)
         return *it->value;
     else
-        throw EvalError("builtin '%1%' not found", name);
+        error<EvalError>("builtin '%1%' not found", name).debugThrow();
 }
 
 
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 01c725930..a14e88f0e 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -630,6 +630,12 @@ public:
      */
     Value & getBuiltin(const std::string & name);
 
+    /**
+     * Retrieve the `builtins` attrset, equivalent to evaluating the reference `builtins`.
+     * Always returns an attribute set value.
+     */
+    Value & getBuiltins();
+
     struct Doc
     {
         Pos pos;

From 5c258d7e255946c5d78ca8116966160e07455347 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Tue, 19 Nov 2024 18:32:09 +0100
Subject: [PATCH 28/73] refactor: Use EvalState::getBuiltins()

---
 src/libexpr/eval.cc    | 6 +++---
 src/libexpr/primops.cc | 2 +-
 src/nix/main.cc        | 3 ++-
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index b1b7a0fe6..63c2e8a71 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -448,7 +448,7 @@ void EvalState::addConstant(const std::string & name, Value * v, Constant info)
         /* Install value the base environment. */
         staticBaseEnv->vars.emplace_back(symbols.create(name), baseEnvDispl);
         baseEnv.values[baseEnvDispl++] = v;
-        baseEnv.values[0]->payload.attrs->push_back(Attr(symbols.create(name2), v));
+        getBuiltins().payload.attrs->push_back(Attr(symbols.create(name2), v));
     }
 }
 
@@ -516,7 +516,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
     else {
         staticBaseEnv->vars.emplace_back(envName, baseEnvDispl);
         baseEnv.values[baseEnvDispl++] = v;
-        baseEnv.values[0]->payload.attrs->push_back(Attr(symbols.create(primOp.name), v));
+        getBuiltins().payload.attrs->push_back(Attr(symbols.create(primOp.name), v));
     }
 
     return v;
@@ -531,7 +531,7 @@ Value & EvalState::getBuiltins()
 
 Value & EvalState::getBuiltin(const std::string & name)
 {
-    auto it = baseEnv.values[0]->attrs()->get(symbols.create(name));
+    auto it = getBuiltins().attrs()->get(symbols.create(name));
     if (it)
         return *it->value;
     else
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index aea623435..50709b18a 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -4937,7 +4937,7 @@ void EvalState::createBaseEnv()
 
     /* Now that we've added all primops, sort the `builtins' set,
        because attribute lookups expect it to be sorted. */
-    baseEnv.values[0]->payload.attrs->sort();
+    getBuiltins().payload.attrs->sort();
 
     staticBaseEnv->sort();
 
diff --git a/src/nix/main.cc b/src/nix/main.cc
index eff2d60a4..b0e26e093 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -435,7 +435,8 @@ void mainWrapped(int argc, char * * argv)
         evalSettings.pureEval = false;
         EvalState state({}, openStore("dummy://"), fetchSettings, evalSettings);
         auto builtinsJson = nlohmann::json::object();
-        for (auto & builtin : *state.baseEnv.values[0]->attrs()) {
+        for (auto & builtinPtr : state.getBuiltins().attrs()->lexicographicOrder(state.symbols)) {
+            auto & builtin = *builtinPtr;
             auto b = nlohmann::json::object();
             if (!builtin.value->isPrimOp()) continue;
             auto primOp = builtin.value->primOp();

From e948c8e03304a899893fffc1238d0d6025ed2564 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Tue, 19 Nov 2024 18:59:08 +0100
Subject: [PATCH 29/73] Bump fetcher cache version

We're getting more reports in https://github.com/NixOS/nix/issues/10985

It appears that something hasn't gone right process-wise.
I find this mistake not to be worth investigating, but rather something
to pay attention to going forward.

Let's nip this in the bud.

Closes https://github.com/NixOS/nix/issues/10985
---
 src/libfetchers/cache.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc
index b0b6cb887..6c2241f3a 100644
--- a/src/libfetchers/cache.cc
+++ b/src/libfetchers/cache.cc
@@ -36,7 +36,7 @@ struct CacheImpl : Cache
     {
         auto state(_state.lock());
 
-        auto dbPath = getCacheDir() + "/fetcher-cache-v2.sqlite";
+        auto dbPath = getCacheDir() + "/fetcher-cache-v3.sqlite";
         createDirs(dirOf(dbPath));
 
         state->db = SQLite(dbPath);

From 4fca22b0dc6fe8054cb18e4e622b6dd2e3b8635c Mon Sep 17 00:00:00 2001
From: Gavin John <gavinnjohn@gmail.com>
Date: Tue, 19 Nov 2024 11:52:45 -0800
Subject: [PATCH 30/73] Update issue and pull request templates

---
 .github/ISSUE_TEMPLATE/bug_report.md          | 39 +++++++++++++------
 .github/ISSUE_TEMPLATE/feature_request.md     | 35 ++++++++++++-----
 .github/ISSUE_TEMPLATE/installer.md           | 17 ++++++--
 .../ISSUE_TEMPLATE/missing_documentation.md   |  2 +-
 .github/PULL_REQUEST_TEMPLATE.md              |  8 ++--
 5 files changed, 72 insertions(+), 29 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
index 984f9a9ea..7ea82dcbc 100644
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -1,36 +1,51 @@
 ---
 name: Bug report
-about: Create a report to help us improve
+about: Report unexpected or incorrect behaviour
 title: ''
 labels: bug
 assignees: ''
 
 ---
 
-**Describe the bug**
+## Describe the bug
 
-A clear and concise description of what the bug is.
+<!--
+  A clear and concise description of what the bug is.
 
-If you have a problem with a specific package or NixOS,
-you probably want to file an issue at https://github.com/NixOS/nixpkgs/issues. 
+  If you have a problem with a specific package or NixOS,
+  you probably want to file an issue at https://github.com/NixOS/nixpkgs/issues.
+-->
 
-**Steps To Reproduce**
+## Steps To Reproduce
 
 1. Go to '...'
 2. Click on '....'
 3. Scroll down to '....'
 4. See error
 
-**Expected behavior**
+## Expected behavior
 
-A clear and concise description of what you expected to happen.
+<!-- A clear and concise description of what you expected to happen. -->
 
-**`nix-env --version` output**
+## Metadata
 
-**Additional context**
+<!-- Please insert the output of running `nix-env --version` below this line -->
 
-Add any other context about the problem here.
+## Additional context
 
-**Priorities**
+<!-- Add any other context about the problem here. -->
+
+## Checklist
+
+<!-- make sure this issue is not redundant or obsolete -->
+
+- [ ] checked [latest Nix manual] \([source])
+- [ ] checked [open bug issues and pull requests] for possible duplicates
+
+[latest Nix manual]: https://nixos.org/manual/nix/unstable/
+[source]: https://github.com/NixOS/nix/tree/master/doc/manual/source
+[open bug issues and pull requests]: https://github.com/NixOS/nix/labels/bug
+
+---
 
 Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc).
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
index 42c658b52..c75a46951 100644
--- a/.github/ISSUE_TEMPLATE/feature_request.md
+++ b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -1,24 +1,39 @@
 ---
 name: Feature request
-about: Suggest an idea for this project
+about: Suggest a new feature
 title: ''
 labels: feature
 assignees: ''
 
 ---
 
-**Is your feature request related to a problem? Please describe.**
-A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+## Is your feature request related to a problem?
 
-**Describe the solution you'd like**
-A clear and concise description of what you want to happen.
+<!-- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] -->
 
-**Describe alternatives you've considered**
-A clear and concise description of any alternative solutions or features you've considered.
+## Proposed solution
 
-**Additional context**
-Add any other context or screenshots about the feature request here.
+<!-- A clear and concise description of what you want to happen. -->
 
-**Priorities**
+## Alternative solutions
+
+<!-- A clear and concise description of any alternative solutions or features you've considered. -->
+
+## Additional context
+
+<!-- Add any other context or screenshots about the feature request here. -->
+
+## Checklist
+
+<!-- make sure this issue is not redundant or obsolete -->
+
+- [ ] checked [latest Nix manual] \([source])
+- [ ] checked [open feature issues and pull requests] for possible duplicates
+
+[latest Nix manual]: https://nixos.org/manual/nix/unstable/
+[source]: https://github.com/NixOS/nix/tree/master/doc/manual/source
+[open feature issues and pull requests]: https://github.com/NixOS/nix/labels/feature
+
+---
 
 Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc).
diff --git a/.github/ISSUE_TEMPLATE/installer.md b/.github/ISSUE_TEMPLATE/installer.md
index 3768a49c9..ed5e1ce87 100644
--- a/.github/ISSUE_TEMPLATE/installer.md
+++ b/.github/ISSUE_TEMPLATE/installer.md
@@ -23,14 +23,25 @@ assignees: ''
 
 <details><summary>Output</summary>
 
-```log
+<!-- paste console output inside the below code block -->
 
-<!-- paste console output here and remove this comment -->
+```log
 
 ```
 
 </details>
 
-## Priorities
+## Checklist
+
+<!-- make sure this issue is not redundant or obsolete -->
+
+- [ ] checked [latest Nix manual] \([source])
+- [ ] checked [open installer issues and pull requests] for possible duplicates
+
+[latest Nix manual]: https://nixos.org/manual/nix/unstable/
+[source]: https://github.com/NixOS/nix/tree/master/doc/manual/source
+[open installer issues and pull requests]: https://github.com/NixOS/nix/labels/installer
+
+---
 
 Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc).
diff --git a/.github/ISSUE_TEMPLATE/missing_documentation.md b/.github/ISSUE_TEMPLATE/missing_documentation.md
index cf663e28d..6c334b722 100644
--- a/.github/ISSUE_TEMPLATE/missing_documentation.md
+++ b/.github/ISSUE_TEMPLATE/missing_documentation.md
@@ -26,6 +26,6 @@ assignees: ''
 [source]: https://github.com/NixOS/nix/tree/master/doc/manual/source
 [open documentation issues and pull requests]: https://github.com/NixOS/nix/labels/documentation
 
-## Priorities
+---
 
 Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc).
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 69da87db7..c6843d86f 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -17,10 +17,12 @@ so you understand the process and the expectations.
 
 -->
 
-# Motivation
+## Motivation
+
 <!-- Briefly explain what the change is about and why it is desirable. -->
 
-# Context
+## Context
+
 <!-- Provide context. Reference open issues if available. -->
 
 <!-- Non-trivial change: Briefly outline the implementation strategy. -->
@@ -29,7 +31,7 @@ so you understand the process and the expectations.
 
 <!-- Large change: Provide instructions to reviewers how to read the diff. -->
 
-# Priorities and Process
+---
 
 Add :+1: to [pull requests you find important](https://github.com/NixOS/nix/pulls?q=is%3Aopen+sort%3Areactions-%2B1-desc).
 

From df9ccdf31b24439c0aac4c2fad6a04529aaf6f50 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 19 Nov 2024 17:35:32 +0100
Subject: [PATCH 31/73] BasicDerivation: Add applyRewrites() method

This is the first part of rewriteDerivation() factored out into its
own method. It's not used anywhere else at the moment, but it's useful
on lazy-trees for rewriting virtual paths.
---
 src/libstore/derivations.cc | 26 ++++++++++++++------------
 src/libstore/derivations.hh |  6 ++++++
 2 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 9b6f67852..1f37b0c38 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -1017,29 +1017,31 @@ std::string hashPlaceholder(const OutputNameView outputName)
     return "/" + hashString(HashAlgorithm::SHA256, concatStrings("nix-output:", outputName)).to_string(HashFormat::Nix32, false);
 }
 
-
-
-
-static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites)
+void BasicDerivation::applyRewrites(const StringMap & rewrites)
 {
-    debug("Rewriting the derivation");
+    if (rewrites.empty()) return;
 
-    for (auto & rewrite : rewrites) {
+    debug("rewriting the derivation");
+
+    for (auto & rewrite : rewrites)
         debug("rewriting %s as %s", rewrite.first, rewrite.second);
-    }
 
-    drv.builder = rewriteStrings(drv.builder, rewrites);
-    for (auto & arg : drv.args) {
+    builder = rewriteStrings(builder, rewrites);
+    for (auto & arg : args)
         arg = rewriteStrings(arg, rewrites);
-    }
 
     StringPairs newEnv;
-    for (auto & envVar : drv.env) {
+    for (auto & envVar : env) {
         auto envName = rewriteStrings(envVar.first, rewrites);
         auto envValue = rewriteStrings(envVar.second, rewrites);
         newEnv.emplace(envName, envValue);
     }
-    drv.env = newEnv;
+    env = std::move(newEnv);
+}
+
+static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites)
+{
+    drv.applyRewrites(rewrites);
 
     auto hashModulo = hashDerivationModulo(store, Derivation(drv), true);
     for (auto & [outputName, output] : drv.outputs) {
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 40740d545..765b66ade 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -325,6 +325,12 @@ struct BasicDerivation
 
     static std::string_view nameFromPath(const StorePath & storePath);
 
+    /**
+     * Apply string rewrites to the `env`, `args` and `builder`
+     * fields.
+     */
+    void applyRewrites(const StringMap & rewrites);
+
     bool operator == (const BasicDerivation &) const = default;
     // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet.
     //auto operator <=> (const BasicDerivation &) const = default;

From ced8d311a593fcf9c3823e4e118474ac132d8e60 Mon Sep 17 00:00:00 2001
From: Picnoir <picnoir@alternativebit.fr>
Date: Wed, 20 Nov 2024 13:33:00 +0100
Subject: [PATCH 32/73] gc: resume GC after a pathinuse error

First the motivation: I recently faced a bug that I assume is coming
from the topoSortPaths function where the GC was trying to delete a
path having some alive referrers. I resolved this by manually deleting
the faulty path referrers using nix-store --query --referrers. I sadly
did not manage to reproduce this bug.

This bug alone is not a big deal. However, this bug is
triggering a cascading failure: invalidatePathChecked is throwing a
PathInUse exception. This exception is not catched and fails the whole GC
run. From there, the machine (a builder machine) was unable to GC its
Nix store, which led to an almost full disk with no way to
automatically delete the dead Nix paths.

Instead, I think we should log the error for the specific store path
we're trying to delete, specifying we can't delete this path because
it still has referrers. Once we're done with logging that, the GC run
should continue to delete the dead store paths it can delete.
---
 src/libstore/gc.cc | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 73195794a..45dfe4ad8 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -4,6 +4,7 @@
 #include "finally.hh"
 #include "unix-domain-socket.hh"
 #include "signals.hh"
+#include "posix-fs-canonicalise.hh"
 
 #if !defined(__linux__)
 // For shelling out to lsof
@@ -763,13 +764,18 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
                 }
             }
         }
-
         for (auto & path : topoSortPaths(visited)) {
             if (!dead.insert(path).second) continue;
             if (shouldDelete) {
-                invalidatePathChecked(path);
-                deleteFromStore(path.to_string());
-                referrersCache.erase(path);
+                try {
+                    invalidatePathChecked(path);
+                    deleteFromStore(path.to_string());
+                    referrersCache.erase(path);
+                } catch (PathInUse &e) {
+                    // If we end up here, it's likely a new occurence
+                    // of https://github.com/NixOS/nix/issues/11923
+                    printError("BUG: %s", e.what());
+                }
             }
         }
     };

From 1800853b2a450b8f80514d2f4acb8ab394a22705 Mon Sep 17 00:00:00 2001
From: Sergei Zimmerman <145775305+xokdvium@users.noreply.github.com>
Date: Thu, 14 Nov 2024 11:03:58 +0300
Subject: [PATCH 33/73] fix(libexpr/eval-inline): get rid of references to
 nullptr env

When diagnosing infinite recursion references to nullptr `Env` can be formed.
This happens only with `ExprBlackHole` is evaluated, which always leads to
`InfiniteRecursionError`.

UBSAN log for one such case:

```
../src/libexpr/eval-inline.hh:94:31: runtime error: reference binding to null pointer of type 'Env'
SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior ../src/libexpr/eval-inline.hh:94:31 in
```
---
 src/libexpr/eval-inline.hh | 6 +++++-
 src/libexpr/eval.cc        | 7 +++++--
 src/libexpr/nixexpr.hh     | 1 +
 3 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh
index d5ce238b2..631c0f396 100644
--- a/src/libexpr/eval-inline.hh
+++ b/src/libexpr/eval-inline.hh
@@ -87,11 +87,15 @@ void EvalState::forceValue(Value & v, const PosIdx pos)
 {
     if (v.isThunk()) {
         Env * env = v.payload.thunk.env;
+        assert(env || v.isBlackhole());
         Expr * expr = v.payload.thunk.expr;
         try {
             v.mkBlackhole();
             //checkInterrupt();
-            expr->eval(*this, *env, v);
+            if (env) [[likely]]
+                expr->eval(*this, *env, v);
+            else
+                ExprBlackHole::throwInfiniteRecursionError(*this, v);
         } catch (...) {
             v.mkThunk(env, expr);
             tryFixupBlackHolePos(v, pos);
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 2fe2d5249..05f58957e 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -2052,9 +2052,12 @@ void ExprPos::eval(EvalState & state, Env & env, Value & v)
     state.mkPos(v, pos);
 }
 
-
-void ExprBlackHole::eval(EvalState & state, Env & env, Value & v)
+void ExprBlackHole::eval(EvalState & state, [[maybe_unused]] Env & env, Value & v)
 {
+    throwInfiniteRecursionError(state, v);
+}
+
+[[gnu::noinline]] [[noreturn]] void ExprBlackHole::throwInfiniteRecursionError(EvalState & state, Value &v) {
     state.error<InfiniteRecursionError>("infinite recursion encountered")
         .atPos(v.determinePos(noPos))
         .debugThrow();
diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh
index 948839bd9..2950ff1fd 100644
--- a/src/libexpr/nixexpr.hh
+++ b/src/libexpr/nixexpr.hh
@@ -468,6 +468,7 @@ struct ExprBlackHole : Expr
     void show(const SymbolTable & symbols, std::ostream & str) const override {}
     void eval(EvalState & state, Env & env, Value & v) override;
     void bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env) override {}
+    [[noreturn]] static void throwInfiniteRecursionError(EvalState & state, Value & v);
 };
 
 extern ExprBlackHole eBlackHole;

From ad7ad017ea5a7f7835148a03e81ef63a2689e8c3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 20 Nov 2024 16:35:47 +0100
Subject: [PATCH 34/73] EvalState::callPathFilter(): Remove unnecessary pathArg
 argument

---
 src/libexpr/eval.hh    | 1 -
 src/libexpr/primops.cc | 5 ++---
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 107334ffe..3ac3c8a8a 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -819,7 +819,6 @@ public:
     bool callPathFilter(
         Value * filterFun,
         const SourcePath & path,
-        std::string_view pathArg,
         PosIdx pos);
 
     DocComment getDocCommentForPos(PosIdx pos);
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 503632328..53bfce6c5 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -2438,7 +2438,6 @@ static RegisterPrimOp primop_toFile({
 bool EvalState::callPathFilter(
     Value * filterFun,
     const SourcePath & path,
-    std::string_view pathArg,
     PosIdx pos)
 {
     auto st = path.lstat();
@@ -2446,7 +2445,7 @@ bool EvalState::callPathFilter(
     /* Call the filter function.  The first argument is the path, the
        second is a string indicating the type of the file. */
     Value arg1;
-    arg1.mkString(pathArg);
+    arg1.mkString(path.path.abs());
 
     // assert that type is not "unknown"
     Value * args []{&arg1, fileTypeToString(*this, st.type)};
@@ -2489,7 +2488,7 @@ static void addPath(
         if (filterFun)
             filter = std::make_unique<PathFilter>([&](const Path & p) {
                 auto p2 = CanonPath(p);
-                return state.callPathFilter(filterFun, {path.accessor, p2}, p2.abs(), pos);
+                return state.callPathFilter(filterFun, {path.accessor, p2}, pos);
             });
 
         std::optional<StorePath> expectedStorePath;

From 5533b0c735bea0e7277b75ba1024bfbc2521c635 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 20 Nov 2024 18:08:31 +0100
Subject: [PATCH 35/73] Move shebang flake tests into a separate test

---
 tests/functional/flakes/flakes.sh   | 102 +-----------------------
 tests/functional/flakes/meson.build |   1 +
 tests/functional/flakes/shebang.sh  | 117 ++++++++++++++++++++++++++++
 3 files changed, 119 insertions(+), 101 deletions(-)
 create mode 100644 tests/functional/flakes/shebang.sh

diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh
index 5e901c5d1..2b503df02 100755
--- a/tests/functional/flakes/flakes.sh
+++ b/tests/functional/flakes/flakes.sh
@@ -74,100 +74,9 @@ cat > "$nonFlakeDir/README.md" <<EOF
 FNORD
 EOF
 
-cat > "$nonFlakeDir/shebang.sh" <<EOF
-#! $(type -P env) nix
-#! nix --offline shell
-#! nix flake1#fooScript
-#! nix --no-write-lock-file --command bash
-set -ex
-foo
-echo "\$@"
-EOF
-chmod +x "$nonFlakeDir/shebang.sh"
-
-git -C "$nonFlakeDir" add README.md shebang.sh
+git -C "$nonFlakeDir" add README.md
 git -C "$nonFlakeDir" commit -m 'Initial'
 
-# this also tests a fairly trivial double backtick quoted string, ``--command``
-cat > $nonFlakeDir/shebang-comments.sh <<EOF
-#! $(type -P env) nix
-# some comments
-# some comments
-# some comments
-#! nix --offline shell
-#! nix flake1#fooScript
-#! nix --no-write-lock-file ``--command`` bash
-foo
-EOF
-chmod +x $nonFlakeDir/shebang-comments.sh
-
-cat > $nonFlakeDir/shebang-different-comments.sh <<EOF
-#! $(type -P env) nix
-# some comments
-// some comments
-/* some comments
-* some comments
-\ some comments
-% some comments
-@ some comments
--- some comments
-(* some comments
-#! nix --offline shell
-#! nix flake1#fooScript
-#! nix --no-write-lock-file --command cat
-foo
-EOF
-chmod +x $nonFlakeDir/shebang-different-comments.sh
-
-cat > $nonFlakeDir/shebang-reject.sh <<EOF
-#! $(type -P env) nix
-# some comments
-# some comments
-# some comments
-#! nix --offline shell *
-#! nix flake1#fooScript
-#! nix --no-write-lock-file --command bash
-foo
-EOF
-chmod +x $nonFlakeDir/shebang-reject.sh
-
-cat > $nonFlakeDir/shebang-inline-expr.sh <<EOF
-#! $(type -P env) nix
-EOF
-cat >> $nonFlakeDir/shebang-inline-expr.sh <<"EOF"
-#! nix --offline shell
-#! nix --impure --expr ``
-#! nix let flake = (builtins.getFlake (toString ../flake1)).packages;
-#! nix     fooScript = flake.${builtins.currentSystem}.fooScript;
-#! nix     /* just a comment !@#$%^&*()__+ # */
-#! nix  in fooScript
-#! nix ``
-#! nix --no-write-lock-file --command bash
-set -ex
-foo
-echo "$@"
-EOF
-chmod +x $nonFlakeDir/shebang-inline-expr.sh
-
-cat > $nonFlakeDir/fooScript.nix <<"EOF"
-let flake = (builtins.getFlake (toString ../flake1)).packages;
-    fooScript = flake.${builtins.currentSystem}.fooScript;
- in fooScript
-EOF
-
-cat > $nonFlakeDir/shebang-file.sh <<EOF
-#! $(type -P env) nix
-EOF
-cat >> $nonFlakeDir/shebang-file.sh <<"EOF"
-#! nix --offline shell
-#! nix --impure --file ./fooScript.nix
-#! nix --no-write-lock-file --command bash
-set -ex
-foo
-echo "$@"
-EOF
-chmod +x $nonFlakeDir/shebang-file.sh
-
 # Construct a custom registry, additionally test the --registry flag
 nix registry add --registry "$registry" flake1 "git+file://$flake1Dir"
 nix registry add --registry "$registry" flake2 "git+file://$percentEncodedFlake2Dir"
@@ -644,15 +553,6 @@ nix flake metadata "$flake2Dir" --reference-lock-file $TEST_ROOT/flake2-overridd
 # reference-lock-file can only be used if allow-dirty is set.
 expectStderr 1 nix flake metadata "$flake2Dir" --no-allow-dirty --reference-lock-file $TEST_ROOT/flake2-overridden.lock
 
-# Test shebang
-[[ $($nonFlakeDir/shebang.sh) = "foo" ]]
-[[ $($nonFlakeDir/shebang.sh "bar") = "foo"$'\n'"bar" ]]
-[[ $($nonFlakeDir/shebang-comments.sh ) = "foo" ]]
-[[ "$($nonFlakeDir/shebang-different-comments.sh)" = "$(cat $nonFlakeDir/shebang-different-comments.sh)" ]]
-[[ $($nonFlakeDir/shebang-inline-expr.sh baz) = "foo"$'\n'"baz" ]]
-[[ $($nonFlakeDir/shebang-file.sh baz) = "foo"$'\n'"baz" ]]
-expect 1 $nonFlakeDir/shebang-reject.sh 2>&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?'
-
 # Test that the --commit-lock-file-summary flag and its alias work
 cat > "$lockfileSummaryFlake/flake.nix" <<EOF
 {
diff --git a/tests/functional/flakes/meson.build b/tests/functional/flakes/meson.build
index c16e2834c..902eed130 100644
--- a/tests/functional/flakes/meson.build
+++ b/tests/functional/flakes/meson.build
@@ -24,6 +24,7 @@ suites += {
     'config.sh',
     'show.sh',
     'dubious-query.sh',
+    'shebang.sh',
   ],
   'workdir': meson.current_source_dir(),
 }
diff --git a/tests/functional/flakes/shebang.sh b/tests/functional/flakes/shebang.sh
new file mode 100644
index 000000000..ac00ca764
--- /dev/null
+++ b/tests/functional/flakes/shebang.sh
@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+
+source ./common.sh
+
+TODO_NixOS
+
+requireGit
+
+flake1Dir=$TEST_ROOT/flake1
+nonFlakeDir=$TEST_ROOT/nonFlake
+
+createGitRepo "$flake1Dir" ""
+createSimpleGitFlake "$flake1Dir"
+nix registry add --registry "$registry" flake1 "git+file://$flake1Dir"
+
+mkdir -p "$nonFlakeDir"
+
+cat > "$nonFlakeDir/shebang.sh" <<EOF
+#! $(type -P env) nix
+#! nix --offline shell
+#! nix flake1#fooScript
+#! nix --no-write-lock-file --command bash
+set -ex
+foo
+echo "\$@"
+EOF
+chmod +x "$nonFlakeDir/shebang.sh"
+
+#git -C "$nonFlakeDir" add shebang.sh
+
+# this also tests a fairly trivial double backtick quoted string, ``--command``
+cat > $nonFlakeDir/shebang-comments.sh <<EOF
+#! $(type -P env) nix
+# some comments
+# some comments
+# some comments
+#! nix --offline shell
+#! nix flake1#fooScript
+#! nix --no-write-lock-file ``--command`` bash
+foo
+EOF
+chmod +x $nonFlakeDir/shebang-comments.sh
+
+cat > $nonFlakeDir/shebang-different-comments.sh <<EOF
+#! $(type -P env) nix
+# some comments
+// some comments
+/* some comments
+* some comments
+\ some comments
+% some comments
+@ some comments
+-- some comments
+(* some comments
+#! nix --offline shell
+#! nix flake1#fooScript
+#! nix --no-write-lock-file --command cat
+foo
+EOF
+chmod +x $nonFlakeDir/shebang-different-comments.sh
+
+cat > $nonFlakeDir/shebang-reject.sh <<EOF
+#! $(type -P env) nix
+# some comments
+# some comments
+# some comments
+#! nix --offline shell *
+#! nix flake1#fooScript
+#! nix --no-write-lock-file --command bash
+foo
+EOF
+chmod +x $nonFlakeDir/shebang-reject.sh
+
+cat > $nonFlakeDir/shebang-inline-expr.sh <<EOF
+#! $(type -P env) nix
+EOF
+cat >> $nonFlakeDir/shebang-inline-expr.sh <<"EOF"
+#! nix --offline shell
+#! nix --impure --expr ``
+#! nix let flake = (builtins.getFlake (toString ../flake1)).packages;
+#! nix     fooScript = flake.${builtins.currentSystem}.fooScript;
+#! nix     /* just a comment !@#$%^&*()__+ # */
+#! nix  in fooScript
+#! nix ``
+#! nix --no-write-lock-file --command bash
+set -ex
+foo
+echo "$@"
+EOF
+chmod +x $nonFlakeDir/shebang-inline-expr.sh
+
+cat > $nonFlakeDir/fooScript.nix <<"EOF"
+let flake = (builtins.getFlake (toString ../flake1)).packages;
+    fooScript = flake.${builtins.currentSystem}.fooScript;
+ in fooScript
+EOF
+
+cat > $nonFlakeDir/shebang-file.sh <<EOF
+#! $(type -P env) nix
+EOF
+cat >> $nonFlakeDir/shebang-file.sh <<"EOF"
+#! nix --offline shell
+#! nix --impure --file ./fooScript.nix
+#! nix --no-write-lock-file --command bash
+set -ex
+foo
+echo "$@"
+EOF
+chmod +x $nonFlakeDir/shebang-file.sh
+
+[[ $($nonFlakeDir/shebang.sh) = "foo" ]]
+[[ $($nonFlakeDir/shebang.sh "bar") = "foo"$'\n'"bar" ]]
+[[ $($nonFlakeDir/shebang-comments.sh ) = "foo" ]]
+[[ "$($nonFlakeDir/shebang-different-comments.sh)" = "$(cat $nonFlakeDir/shebang-different-comments.sh)" ]]
+[[ $($nonFlakeDir/shebang-inline-expr.sh baz) = "foo"$'\n'"baz" ]]
+[[ $($nonFlakeDir/shebang-file.sh baz) = "foo"$'\n'"baz" ]]
+expect 1 $nonFlakeDir/shebang-reject.sh 2>&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?'

From fd2df5f02f59f5f739c1e1931f8ff8c11a7720e0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 20 Nov 2024 18:23:20 +0100
Subject: [PATCH 36/73] Rename nonFlakeDir -> scriptDir

---
 tests/functional/flakes/shebang.sh | 50 +++++++++++++++---------------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/tests/functional/flakes/shebang.sh b/tests/functional/flakes/shebang.sh
index ac00ca764..23db19baa 100644
--- a/tests/functional/flakes/shebang.sh
+++ b/tests/functional/flakes/shebang.sh
@@ -7,15 +7,15 @@ TODO_NixOS
 requireGit
 
 flake1Dir=$TEST_ROOT/flake1
-nonFlakeDir=$TEST_ROOT/nonFlake
+scriptDir=$TEST_ROOT/nonFlake
 
 createGitRepo "$flake1Dir" ""
 createSimpleGitFlake "$flake1Dir"
 nix registry add --registry "$registry" flake1 "git+file://$flake1Dir"
 
-mkdir -p "$nonFlakeDir"
+mkdir -p "$scriptDir"
 
-cat > "$nonFlakeDir/shebang.sh" <<EOF
+cat > "$scriptDir/shebang.sh" <<EOF
 #! $(type -P env) nix
 #! nix --offline shell
 #! nix flake1#fooScript
@@ -24,12 +24,12 @@ set -ex
 foo
 echo "\$@"
 EOF
-chmod +x "$nonFlakeDir/shebang.sh"
+chmod +x "$scriptDir/shebang.sh"
 
-#git -C "$nonFlakeDir" add shebang.sh
+#git -C "$scriptDir" add shebang.sh
 
 # this also tests a fairly trivial double backtick quoted string, ``--command``
-cat > $nonFlakeDir/shebang-comments.sh <<EOF
+cat > $scriptDir/shebang-comments.sh <<EOF
 #! $(type -P env) nix
 # some comments
 # some comments
@@ -39,9 +39,9 @@ cat > $nonFlakeDir/shebang-comments.sh <<EOF
 #! nix --no-write-lock-file ``--command`` bash
 foo
 EOF
-chmod +x $nonFlakeDir/shebang-comments.sh
+chmod +x $scriptDir/shebang-comments.sh
 
-cat > $nonFlakeDir/shebang-different-comments.sh <<EOF
+cat > $scriptDir/shebang-different-comments.sh <<EOF
 #! $(type -P env) nix
 # some comments
 // some comments
@@ -57,9 +57,9 @@ cat > $nonFlakeDir/shebang-different-comments.sh <<EOF
 #! nix --no-write-lock-file --command cat
 foo
 EOF
-chmod +x $nonFlakeDir/shebang-different-comments.sh
+chmod +x $scriptDir/shebang-different-comments.sh
 
-cat > $nonFlakeDir/shebang-reject.sh <<EOF
+cat > $scriptDir/shebang-reject.sh <<EOF
 #! $(type -P env) nix
 # some comments
 # some comments
@@ -69,12 +69,12 @@ cat > $nonFlakeDir/shebang-reject.sh <<EOF
 #! nix --no-write-lock-file --command bash
 foo
 EOF
-chmod +x $nonFlakeDir/shebang-reject.sh
+chmod +x $scriptDir/shebang-reject.sh
 
-cat > $nonFlakeDir/shebang-inline-expr.sh <<EOF
+cat > $scriptDir/shebang-inline-expr.sh <<EOF
 #! $(type -P env) nix
 EOF
-cat >> $nonFlakeDir/shebang-inline-expr.sh <<"EOF"
+cat >> $scriptDir/shebang-inline-expr.sh <<"EOF"
 #! nix --offline shell
 #! nix --impure --expr ``
 #! nix let flake = (builtins.getFlake (toString ../flake1)).packages;
@@ -87,18 +87,18 @@ set -ex
 foo
 echo "$@"
 EOF
-chmod +x $nonFlakeDir/shebang-inline-expr.sh
+chmod +x $scriptDir/shebang-inline-expr.sh
 
-cat > $nonFlakeDir/fooScript.nix <<"EOF"
+cat > $scriptDir/fooScript.nix <<"EOF"
 let flake = (builtins.getFlake (toString ../flake1)).packages;
     fooScript = flake.${builtins.currentSystem}.fooScript;
  in fooScript
 EOF
 
-cat > $nonFlakeDir/shebang-file.sh <<EOF
+cat > $scriptDir/shebang-file.sh <<EOF
 #! $(type -P env) nix
 EOF
-cat >> $nonFlakeDir/shebang-file.sh <<"EOF"
+cat >> $scriptDir/shebang-file.sh <<"EOF"
 #! nix --offline shell
 #! nix --impure --file ./fooScript.nix
 #! nix --no-write-lock-file --command bash
@@ -106,12 +106,12 @@ set -ex
 foo
 echo "$@"
 EOF
-chmod +x $nonFlakeDir/shebang-file.sh
+chmod +x $scriptDir/shebang-file.sh
 
-[[ $($nonFlakeDir/shebang.sh) = "foo" ]]
-[[ $($nonFlakeDir/shebang.sh "bar") = "foo"$'\n'"bar" ]]
-[[ $($nonFlakeDir/shebang-comments.sh ) = "foo" ]]
-[[ "$($nonFlakeDir/shebang-different-comments.sh)" = "$(cat $nonFlakeDir/shebang-different-comments.sh)" ]]
-[[ $($nonFlakeDir/shebang-inline-expr.sh baz) = "foo"$'\n'"baz" ]]
-[[ $($nonFlakeDir/shebang-file.sh baz) = "foo"$'\n'"baz" ]]
-expect 1 $nonFlakeDir/shebang-reject.sh 2>&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?'
+[[ $($scriptDir/shebang.sh) = "foo" ]]
+[[ $($scriptDir/shebang.sh "bar") = "foo"$'\n'"bar" ]]
+[[ $($scriptDir/shebang-comments.sh ) = "foo" ]]
+[[ "$($scriptDir/shebang-different-comments.sh)" = "$(cat $scriptDir/shebang-different-comments.sh)" ]]
+[[ $($scriptDir/shebang-inline-expr.sh baz) = "foo"$'\n'"baz" ]]
+[[ $($scriptDir/shebang-file.sh baz) = "foo"$'\n'"baz" ]]
+expect 1 $scriptDir/shebang-reject.sh 2>&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?'

From e1cb905acae40b6bb55d30312ba9997c418eebf8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 20 Nov 2024 18:42:33 +0100
Subject: [PATCH 37/73] Move --commit-lock-file-summary tests into a separate
 file

---
 .../flakes/commit-lock-file-summary.sh        | 50 +++++++++++++++++++
 tests/functional/flakes/flakes.sh             | 40 +--------------
 tests/functional/flakes/meson.build           |  1 +
 3 files changed, 52 insertions(+), 39 deletions(-)
 create mode 100644 tests/functional/flakes/commit-lock-file-summary.sh

diff --git a/tests/functional/flakes/commit-lock-file-summary.sh b/tests/functional/flakes/commit-lock-file-summary.sh
new file mode 100644
index 000000000..21504eb21
--- /dev/null
+++ b/tests/functional/flakes/commit-lock-file-summary.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+source ./common.sh
+
+TODO_NixOS
+
+requireGit
+
+flake1Dir=$TEST_ROOT/flake1
+lockfileSummaryFlake=$TEST_ROOT/lockfileSummaryFlake
+
+createGitRepo "$flake1Dir" ""
+createSimpleGitFlake "$flake1Dir"
+nix registry add --registry "$registry" flake1 "git+file://$flake1Dir"
+
+createGitRepo "$lockfileSummaryFlake" "--initial-branch=main"
+
+# Test that the --commit-lock-file-summary flag and its alias work
+cat > "$lockfileSummaryFlake/flake.nix" <<EOF
+{
+  inputs = {
+    flake1.url = "git+file://$flake1Dir";
+  };
+
+  description = "lockfileSummaryFlake";
+
+  outputs = inputs: rec {
+    packages.$system.default = inputs.flake1.packages.$system.foo;
+  };
+}
+EOF
+
+git -C "$lockfileSummaryFlake" add flake.nix
+git -C "$lockfileSummaryFlake" commit -m 'Add lockfileSummaryFlake'
+
+testSummary="test summary 1"
+nix flake lock "$lockfileSummaryFlake" --commit-lock-file --commit-lock-file-summary "$testSummary"
+[[ -e "$lockfileSummaryFlake/flake.lock" ]]
+[[ -z $(git -C "$lockfileSummaryFlake" diff main || echo failed) ]]
+[[ "$(git -C "$lockfileSummaryFlake" log --format=%s -n 1)" = "$testSummary" ]]
+
+git -C "$lockfileSummaryFlake" rm :/:flake.lock
+git -C "$lockfileSummaryFlake" commit -m "remove flake.lock"
+testSummary="test summary 2"
+# NOTE(cole-h): We use `--option` here because Nix settings do not currently support flag-ifying the
+# alias of a setting: https://github.com/NixOS/nix/issues/10989
+nix flake lock "$lockfileSummaryFlake" --commit-lock-file --option commit-lockfile-summary "$testSummary"
+[[ -e "$lockfileSummaryFlake/flake.lock" ]]
+[[ -z $(git -C "$lockfileSummaryFlake" diff main || echo failed) ]]
+[[ "$(git -C "$lockfileSummaryFlake" log --format=%s -n 1)" = "$testSummary" ]]
diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh
index 2b503df02..6718e2370 100755
--- a/tests/functional/flakes/flakes.sh
+++ b/tests/functional/flakes/flakes.sh
@@ -19,17 +19,13 @@ flake7Dir=$TEST_ROOT/flake7
 nonFlakeDir=$TEST_ROOT/nonFlake
 badFlakeDir=$TEST_ROOT/badFlake
 flakeGitBare=$TEST_ROOT/flakeGitBare
-lockfileSummaryFlake=$TEST_ROOT/lockfileSummaryFlake
 
-for repo in "$flake1Dir" "$flake2Dir" "$flake3Dir" "$flake7Dir" "$nonFlakeDir" "$lockfileSummaryFlake"; do
+for repo in "$flake1Dir" "$flake2Dir" "$flake3Dir" "$flake7Dir" "$nonFlakeDir"; do
     # Give one repo a non-main initial branch.
     extraArgs=
     if [[ "$repo" == "$flake2Dir" ]]; then
       extraArgs="--initial-branch=main"
     fi
-    if [[ "$repo" == "$lockfileSummaryFlake" ]]; then
-      extraArgs="--initial-branch=main"
-    fi
 
     createGitRepo "$repo" "$extraArgs"
 done
@@ -552,37 +548,3 @@ nix flake metadata "$flake2Dir" --reference-lock-file $TEST_ROOT/flake2-overridd
 
 # reference-lock-file can only be used if allow-dirty is set.
 expectStderr 1 nix flake metadata "$flake2Dir" --no-allow-dirty --reference-lock-file $TEST_ROOT/flake2-overridden.lock
-
-# Test that the --commit-lock-file-summary flag and its alias work
-cat > "$lockfileSummaryFlake/flake.nix" <<EOF
-{
-  inputs = {
-    flake1.url = "git+file://$flake1Dir";
-  };
-
-  description = "lockfileSummaryFlake";
-
-  outputs = inputs: rec {
-    packages.$system.default = inputs.flake1.packages.$system.foo;
-  };
-}
-EOF
-
-git -C "$lockfileSummaryFlake" add flake.nix
-git -C "$lockfileSummaryFlake" commit -m 'Add lockfileSummaryFlake'
-
-testSummary="test summary 1"
-nix flake lock "$lockfileSummaryFlake" --commit-lock-file --commit-lock-file-summary "$testSummary"
-[[ -e "$lockfileSummaryFlake/flake.lock" ]]
-[[ -z $(git -C "$lockfileSummaryFlake" diff main || echo failed) ]]
-[[ "$(git -C "$lockfileSummaryFlake" log --format=%s -n 1)" = "$testSummary" ]]
-
-git -C "$lockfileSummaryFlake" rm :/:flake.lock
-git -C "$lockfileSummaryFlake" commit -m "remove flake.lock"
-testSummary="test summary 2"
-# NOTE(cole-h): We use `--option` here because Nix settings do not currently support flag-ifying the
-# alias of a setting: https://github.com/NixOS/nix/issues/10989
-nix flake lock "$lockfileSummaryFlake" --commit-lock-file --option commit-lockfile-summary "$testSummary"
-[[ -e "$lockfileSummaryFlake/flake.lock" ]]
-[[ -z $(git -C "$lockfileSummaryFlake" diff main || echo failed) ]]
-[[ "$(git -C "$lockfileSummaryFlake" log --format=%s -n 1)" = "$testSummary" ]]
diff --git a/tests/functional/flakes/meson.build b/tests/functional/flakes/meson.build
index 902eed130..a646abc00 100644
--- a/tests/functional/flakes/meson.build
+++ b/tests/functional/flakes/meson.build
@@ -25,6 +25,7 @@ suites += {
     'show.sh',
     'dubious-query.sh',
     'shebang.sh',
+    'commit-lock-file-summary.sh',
   ],
   'workdir': meson.current_source_dir(),
 }

From 4cafea7c7b230f5b25162c988dcd75145515f0be Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 20 Nov 2024 18:51:23 +0100
Subject: [PATCH 38/73] Add a utility function for creating/registering a
 simple flake

---
 tests/functional/flakes/commit-lock-file-summary.sh | 8 +-------
 tests/functional/flakes/common.sh                   | 9 +++++++++
 tests/functional/flakes/dubious-query.sh            | 7 ++-----
 tests/functional/flakes/edit.sh                     | 7 +------
 tests/functional/flakes/shebang.sh                  | 8 +-------
 5 files changed, 14 insertions(+), 25 deletions(-)

diff --git a/tests/functional/flakes/commit-lock-file-summary.sh b/tests/functional/flakes/commit-lock-file-summary.sh
index 21504eb21..314d43ec3 100644
--- a/tests/functional/flakes/commit-lock-file-summary.sh
+++ b/tests/functional/flakes/commit-lock-file-summary.sh
@@ -4,15 +4,9 @@ source ./common.sh
 
 TODO_NixOS
 
-requireGit
+createFlake1
 
-flake1Dir=$TEST_ROOT/flake1
 lockfileSummaryFlake=$TEST_ROOT/lockfileSummaryFlake
-
-createGitRepo "$flake1Dir" ""
-createSimpleGitFlake "$flake1Dir"
-nix registry add --registry "$registry" flake1 "git+file://$flake1Dir"
-
 createGitRepo "$lockfileSummaryFlake" "--initial-branch=main"
 
 # Test that the --commit-lock-file-summary flag and its alias work
diff --git a/tests/functional/flakes/common.sh b/tests/functional/flakes/common.sh
index cc9b2e466..0a020a1d7 100644
--- a/tests/functional/flakes/common.sh
+++ b/tests/functional/flakes/common.sh
@@ -38,12 +38,21 @@ EOF
 }
 
 createSimpleGitFlake() {
+    requireGit
     local flakeDir="$1"
     writeSimpleFlake "$flakeDir"
     git -C "$flakeDir" add flake.nix simple.nix shell.nix simple.builder.sh config.nix
     git -C "$flakeDir" commit -m 'Initial'
 }
 
+# Create a simple Git flake and add it to the registry as "flake1".
+createFlake1() {
+    flake1Dir="$TEST_ROOT/flake1"
+    createGitRepo "$flake1Dir" ""
+    createSimpleGitFlake "$flake1Dir"
+    nix registry add --registry "$registry" flake1 "git+file://$flake1Dir"
+}
+
 writeDependentFlake() {
     local flakeDir="$1"
     cat > "$flakeDir/flake.nix" <<EOF
diff --git a/tests/functional/flakes/dubious-query.sh b/tests/functional/flakes/dubious-query.sh
index b481b5f81..c147034c3 100644
--- a/tests/functional/flakes/dubious-query.sh
+++ b/tests/functional/flakes/dubious-query.sh
@@ -2,11 +2,8 @@
 
 source ./common.sh
 
-requireGit
-
-repoDir="$TEST_ROOT/repo"
-createGitRepo "$repoDir"
-createSimpleGitFlake "$repoDir"
+createFlake1
+repoDir="$flake1Dir"
 
 # Check that a flakeref without a query is accepted correctly.
 expectStderr 0 nix --offline build --dry-run "git+file://$repoDir#foo"
diff --git a/tests/functional/flakes/edit.sh b/tests/functional/flakes/edit.sh
index 0fdf8b95a..fda5c29fd 100755
--- a/tests/functional/flakes/edit.sh
+++ b/tests/functional/flakes/edit.sh
@@ -2,12 +2,7 @@
 
 source ./common.sh
 
-requireGit
-
-flake1Dir=$TEST_ROOT/flake1
-
-createGitRepo "$flake1Dir"
-createSimpleGitFlake "$flake1Dir"
+createFlake1
 
 export EDITOR=cat
 nix edit "$flake1Dir#" | grepQuiet simple.builder.sh
diff --git a/tests/functional/flakes/shebang.sh b/tests/functional/flakes/shebang.sh
index 23db19baa..645b1fa38 100644
--- a/tests/functional/flakes/shebang.sh
+++ b/tests/functional/flakes/shebang.sh
@@ -4,15 +4,9 @@ source ./common.sh
 
 TODO_NixOS
 
-requireGit
+createFlake1
 
-flake1Dir=$TEST_ROOT/flake1
 scriptDir=$TEST_ROOT/nonFlake
-
-createGitRepo "$flake1Dir" ""
-createSimpleGitFlake "$flake1Dir"
-nix registry add --registry "$registry" flake1 "git+file://$flake1Dir"
-
 mkdir -p "$scriptDir"
 
 cat > "$scriptDir/shebang.sh" <<EOF

From 6832ee5d6ff9f6d3a691a1f504d2dc97e40da3ed Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 20 Nov 2024 19:51:04 +0100
Subject: [PATCH 39/73] Move non-flake input tests into a separate file

---
 tests/functional/flakes/common.sh           |  23 +++
 tests/functional/flakes/flakes.sh           | 166 ++------------------
 tests/functional/flakes/meson.build         |   1 +
 tests/functional/flakes/non-flake-inputs.sh | 126 +++++++++++++++
 4 files changed, 164 insertions(+), 152 deletions(-)
 create mode 100644 tests/functional/flakes/non-flake-inputs.sh

diff --git a/tests/functional/flakes/common.sh b/tests/functional/flakes/common.sh
index 0a020a1d7..b1c3988e3 100644
--- a/tests/functional/flakes/common.sh
+++ b/tests/functional/flakes/common.sh
@@ -53,6 +53,29 @@ createFlake1() {
     nix registry add --registry "$registry" flake1 "git+file://$flake1Dir"
 }
 
+createFlake2() {
+    flake2Dir="$TEST_ROOT/flake 2"
+    percentEncodedFlake2Dir="$TEST_ROOT/flake%202"
+
+    # Give one repo a non-main initial branch.
+    createGitRepo "$flake2Dir" "--initial-branch=main"
+
+    cat > "$flake2Dir/flake.nix" <<EOF
+{
+  description = "Fnord";
+
+  outputs = { self, flake1 }: rec {
+    packages.$system.bar = flake1.packages.$system.foo;
+  };
+}
+EOF
+
+    git -C "$flake2Dir" add flake.nix
+    git -C "$flake2Dir" commit -m 'Initial'
+
+    nix registry add --registry "$registry" flake2 "git+file://$percentEncodedFlake2Dir"
+}
+
 writeDependentFlake() {
     local flakeDir="$1"
     cat > "$flakeDir/flake.nix" <<EOF
diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh
index 6718e2370..de37ae1b7 100755
--- a/tests/functional/flakes/flakes.sh
+++ b/tests/functional/flakes/flakes.sh
@@ -9,42 +9,20 @@ requireGit
 clearStore
 rm -rf $TEST_HOME/.cache $TEST_HOME/.config
 
-flake1Dir=$TEST_ROOT/flake1
-flake2Dir=$TEST_ROOT/flake\ 2
-percentEncodedFlake2Dir=$TEST_ROOT/flake%202
+createFlake1
+createFlake2
+
 flake3Dir=$TEST_ROOT/flake%20
 percentEncodedFlake3Dir=$TEST_ROOT/flake%2520
 flake5Dir=$TEST_ROOT/flake5
 flake7Dir=$TEST_ROOT/flake7
-nonFlakeDir=$TEST_ROOT/nonFlake
 badFlakeDir=$TEST_ROOT/badFlake
 flakeGitBare=$TEST_ROOT/flakeGitBare
 
-for repo in "$flake1Dir" "$flake2Dir" "$flake3Dir" "$flake7Dir" "$nonFlakeDir"; do
-    # Give one repo a non-main initial branch.
-    extraArgs=
-    if [[ "$repo" == "$flake2Dir" ]]; then
-      extraArgs="--initial-branch=main"
-    fi
-
-    createGitRepo "$repo" "$extraArgs"
+for repo in "$flake3Dir" "$flake7Dir"; do
+    createGitRepo "$repo" ""
 done
 
-createSimpleGitFlake "$flake1Dir"
-
-cat > "$flake2Dir/flake.nix" <<EOF
-{
-  description = "Fnord";
-
-  outputs = { self, flake1 }: rec {
-    packages.$system.bar = flake1.packages.$system.foo;
-  };
-}
-EOF
-
-git -C "$flake2Dir" add flake.nix
-git -C "$flake2Dir" commit -m 'Initial'
-
 cat > "$flake3Dir/flake.nix" <<EOF
 {
   description = "Fnord";
@@ -66,22 +44,13 @@ EOF
 git -C "$flake3Dir" add flake.nix default.nix
 git -C "$flake3Dir" commit -m 'Initial'
 
-cat > "$nonFlakeDir/README.md" <<EOF
-FNORD
-EOF
-
-git -C "$nonFlakeDir" add README.md
-git -C "$nonFlakeDir" commit -m 'Initial'
-
 # Construct a custom registry, additionally test the --registry flag
 nix registry add --registry "$registry" flake1 "git+file://$flake1Dir"
-nix registry add --registry "$registry" flake2 "git+file://$percentEncodedFlake2Dir"
 nix registry add --registry "$registry" flake3 "git+file://$percentEncodedFlake3Dir"
-nix registry add --registry "$registry" flake4 flake3
 nix registry add --registry "$registry" nixpkgs flake1
 
 # Test 'nix registry list'.
-[[ $(nix registry list | wc -l) == 5 ]]
+[[ $(nix registry list | wc -l) == 4 ]]
 nix registry list | grep        '^global'
 nix registry list | grepInverse '^user' # nothing in user registry
 
@@ -251,77 +220,8 @@ _NIX_FORCE_HTTP=1 nix build -o "$TEST_ROOT/result" "git+file://$percentEncodedFl
 mv "$flake1Dir.tmp" "$flake1Dir"
 mv "$flake2Dir.tmp" "$flake2Dir"
 
-# Add nonFlakeInputs to flake3.
-rm "$flake3Dir/flake.nix"
-
-cat > "$flake3Dir/flake.nix" <<EOF
-{
-  inputs = {
-    flake1 = {};
-    flake2 = {};
-    nonFlake = {
-      url = git+file://$nonFlakeDir;
-      flake = false;
-    };
-    nonFlakeFile = {
-      url = path://$nonFlakeDir/README.md;
-      flake = false;
-    };
-    nonFlakeFile2 = {
-      url = "$nonFlakeDir/README.md";
-      flake = false;
-    };
-  };
-
-  description = "Fnord";
-
-  outputs = inputs: rec {
-    packages.$system.xyzzy = inputs.flake2.packages.$system.bar;
-    packages.$system.sth = inputs.flake1.packages.$system.foo;
-    packages.$system.fnord =
-      with import ./config.nix;
-      mkDerivation {
-        inherit system;
-        name = "fnord";
-        dummy = builtins.readFile (builtins.path { name = "source"; path = ./.; filter = path: type: baseNameOf path == "config.nix"; } + "/config.nix");
-        dummy2 = builtins.readFile (builtins.path { name = "source"; path = inputs.flake1; filter = path: type: baseNameOf path == "simple.nix"; } + "/simple.nix");
-        buildCommand = ''
-          cat \${inputs.nonFlake}/README.md > \$out
-          [[ \$(cat \${inputs.nonFlake}/README.md) = \$(cat \${inputs.nonFlakeFile}) ]]
-          [[ \${inputs.nonFlakeFile} = \${inputs.nonFlakeFile2} ]]
-        '';
-      };
-  };
-}
-EOF
-
-cp "${config_nix}" "$flake3Dir"
-
-git -C "$flake3Dir" add flake.nix config.nix
-git -C "$flake3Dir" commit -m 'Add nonFlakeInputs'
-
-# Check whether `nix build` works with a lockfile which is missing a
-# nonFlakeInputs.
-nix build -o "$TEST_ROOT/result" "$flake3Dir#sth" --commit-lock-file
-
-nix build -o "$TEST_ROOT/result" flake3#fnord
-[[ $(cat $TEST_ROOT/result) = FNORD ]]
-
-# Check whether flake input fetching is lazy: flake3#sth does not
-# depend on flake2, so this shouldn't fail.
-rm -rf "$TEST_HOME/.cache"
-clearStore
-mv "$flake2Dir" "$flake2Dir.tmp"
-mv "$nonFlakeDir" "$nonFlakeDir.tmp"
-nix build -o "$TEST_ROOT/result" flake3#sth
-(! nix build -o "$TEST_ROOT/result" flake3#xyzzy)
-(! nix build -o "$TEST_ROOT/result" flake3#fnord)
-mv "$flake2Dir.tmp" "$flake2Dir"
-mv "$nonFlakeDir.tmp" "$nonFlakeDir"
-nix build -o "$TEST_ROOT/result" flake3#xyzzy flake3#fnord
-
 # Test doing multiple `lookupFlake`s
-nix build -o "$TEST_ROOT/result" flake4#xyzzy
+nix build -o "$TEST_ROOT/result" flake3#xyzzy
 
 # Test 'nix flake update' and --override-flake.
 nix flake lock "$flake3Dir"
@@ -330,53 +230,15 @@ nix flake lock "$flake3Dir"
 nix flake update --flake "$flake3Dir" --override-flake flake2 nixpkgs
 [[ ! -z $(git -C "$flake3Dir" diff master || echo failed) ]]
 
-# Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore
-git -C "$flake3Dir" checkout -b removeXyzzy
-rm "$flake3Dir/flake.nix"
-
-cat > "$flake3Dir/flake.nix" <<EOF
-{
-  inputs = {
-    nonFlake = {
-      url = "$nonFlakeDir";
-      flake = false;
-    };
-  };
-
-  description = "Fnord";
-
-  outputs = { self, flake1, flake2, nonFlake }: rec {
-    packages.$system.sth = flake1.packages.$system.foo;
-    packages.$system.fnord =
-      with import ./config.nix;
-      mkDerivation {
-        inherit system;
-        name = "fnord";
-        buildCommand = ''
-          cat \${nonFlake}/README.md > \$out
-        '';
-      };
-  };
-}
-EOF
-nix flake lock "$flake3Dir"
-git -C "$flake3Dir" add flake.nix flake.lock
-git -C "$flake3Dir" commit -m 'Remove packages.xyzzy'
-git -C "$flake3Dir" checkout master
-
-# Test whether fuzzy-matching works for registry entries.
-(! nix build -o "$TEST_ROOT/result" flake4/removeXyzzy#xyzzy)
-nix build -o "$TEST_ROOT/result" flake4/removeXyzzy#sth
-
 # Testing the nix CLI
 nix registry add flake1 flake3
-[[ $(nix registry list | wc -l) == 6 ]]
-nix registry pin flake1
-[[ $(nix registry list | wc -l) == 6 ]]
-nix registry pin flake1 flake3
-[[ $(nix registry list | wc -l) == 6 ]]
-nix registry remove flake1
 [[ $(nix registry list | wc -l) == 5 ]]
+nix registry pin flake1
+[[ $(nix registry list | wc -l) == 5 ]]
+nix registry pin flake1 flake3
+[[ $(nix registry list | wc -l) == 5 ]]
+nix registry remove flake1
+[[ $(nix registry list | wc -l) == 4 ]]
 
 # Test 'nix registry list' with a disabled global registry.
 nix registry add user-flake1 git+file://$flake1Dir
@@ -386,7 +248,7 @@ nix --flake-registry "" registry list | grepQuietInverse '^global' # nothing in
 nix --flake-registry "" registry list | grepQuiet '^user'
 nix registry remove user-flake1
 nix registry remove user-flake2
-[[ $(nix registry list | wc -l) == 5 ]]
+[[ $(nix registry list | wc -l) == 4 ]]
 
 # Test 'nix flake clone'.
 rm -rf $TEST_ROOT/flake1-v2
diff --git a/tests/functional/flakes/meson.build b/tests/functional/flakes/meson.build
index a646abc00..00060e3c9 100644
--- a/tests/functional/flakes/meson.build
+++ b/tests/functional/flakes/meson.build
@@ -26,6 +26,7 @@ suites += {
     'dubious-query.sh',
     'shebang.sh',
     'commit-lock-file-summary.sh',
+    'non-flake-inputs.sh',
   ],
   'workdir': meson.current_source_dir(),
 }
diff --git a/tests/functional/flakes/non-flake-inputs.sh b/tests/functional/flakes/non-flake-inputs.sh
new file mode 100644
index 000000000..66dc74e0c
--- /dev/null
+++ b/tests/functional/flakes/non-flake-inputs.sh
@@ -0,0 +1,126 @@
+#!/usr/bin/env bash
+
+source ./common.sh
+
+createFlake1
+createFlake2
+
+nonFlakeDir=$TEST_ROOT/nonFlake
+createGitRepo "$nonFlakeDir" ""
+
+cat > "$nonFlakeDir/README.md" <<EOF
+FNORD
+EOF
+
+git -C "$nonFlakeDir" add README.md
+git -C "$nonFlakeDir" commit -m 'Initial'
+
+flake3Dir=$TEST_ROOT/flake3
+createGitRepo "$flake3Dir" ""
+
+cat > "$flake3Dir/flake.nix" <<EOF
+{
+  inputs = {
+    flake1 = {};
+    flake2 = {};
+    nonFlake = {
+      url = git+file://$nonFlakeDir;
+      flake = false;
+    };
+    nonFlakeFile = {
+      url = path://$nonFlakeDir/README.md;
+      flake = false;
+    };
+    nonFlakeFile2 = {
+      url = "$nonFlakeDir/README.md";
+      flake = false;
+    };
+  };
+
+  description = "Fnord";
+
+  outputs = inputs: rec {
+    packages.$system.xyzzy = inputs.flake2.packages.$system.bar;
+    packages.$system.sth = inputs.flake1.packages.$system.foo;
+    packages.$system.fnord =
+      with import ./config.nix;
+      mkDerivation {
+        inherit system;
+        name = "fnord";
+        dummy = builtins.readFile (builtins.path { name = "source"; path = ./.; filter = path: type: baseNameOf path == "config.nix"; } + "/config.nix");
+        dummy2 = builtins.readFile (builtins.path { name = "source"; path = inputs.flake1; filter = path: type: baseNameOf path == "simple.nix"; } + "/simple.nix");
+        buildCommand = ''
+          cat \${inputs.nonFlake}/README.md > \$out
+          [[ \$(cat \${inputs.nonFlake}/README.md) = \$(cat \${inputs.nonFlakeFile}) ]]
+          [[ \${inputs.nonFlakeFile} = \${inputs.nonFlakeFile2} ]]
+        '';
+      };
+  };
+}
+EOF
+
+cp "${config_nix}" "$flake3Dir"
+
+git -C "$flake3Dir" add flake.nix config.nix
+git -C "$flake3Dir" commit -m 'Add nonFlakeInputs'
+
+# Check whether `nix build` works with a lockfile which is missing a
+# nonFlakeInputs.
+nix build -o "$TEST_ROOT/result" "$flake3Dir#sth" --commit-lock-file
+
+nix registry add --registry "$registry" flake3 "git+file://$flake3Dir"
+
+nix build -o "$TEST_ROOT/result" flake3#fnord
+[[ $(cat $TEST_ROOT/result) = FNORD ]]
+
+# Check whether flake input fetching is lazy: flake3#sth does not
+# depend on flake2, so this shouldn't fail.
+rm -rf "$TEST_HOME/.cache"
+clearStore
+mv "$flake2Dir" "$flake2Dir.tmp"
+mv "$nonFlakeDir" "$nonFlakeDir.tmp"
+nix build -o "$TEST_ROOT/result" flake3#sth
+(! nix build -o "$TEST_ROOT/result" flake3#xyzzy)
+(! nix build -o "$TEST_ROOT/result" flake3#fnord)
+mv "$flake2Dir.tmp" "$flake2Dir"
+mv "$nonFlakeDir.tmp" "$nonFlakeDir"
+nix build -o "$TEST_ROOT/result" flake3#xyzzy flake3#fnord
+
+# Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore
+git -C "$flake3Dir" checkout -b removeXyzzy
+rm "$flake3Dir/flake.nix"
+
+cat > "$flake3Dir/flake.nix" <<EOF
+{
+  inputs = {
+    nonFlake = {
+      url = "$nonFlakeDir";
+      flake = false;
+    };
+  };
+
+  description = "Fnord";
+
+  outputs = { self, flake1, flake2, nonFlake }: rec {
+    packages.$system.sth = flake1.packages.$system.foo;
+    packages.$system.fnord =
+      with import ./config.nix;
+      mkDerivation {
+        inherit system;
+        name = "fnord";
+        buildCommand = ''
+          cat \${nonFlake}/README.md > \$out
+        '';
+      };
+  };
+}
+EOF
+nix flake lock "$flake3Dir"
+git -C "$flake3Dir" add flake.nix flake.lock
+git -C "$flake3Dir" commit -m 'Remove packages.xyzzy'
+git -C "$flake3Dir" checkout master
+
+# Test whether fuzzy-matching works for registry entries.
+nix registry add --registry "$registry" flake4 flake3
+(! nix build -o "$TEST_ROOT/result" flake4/removeXyzzy#xyzzy)
+nix build -o "$TEST_ROOT/result" flake4/removeXyzzy#sth

From db0525692dcb94e4f274e033091ad1ce54a30cf3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 20 Nov 2024 21:07:22 +0100
Subject: [PATCH 40/73] Formatting

---
 src/libcmd/command.cc | 36 +++++++++++++++++++-----------------
 src/libcmd/command.hh |  5 +----
 2 files changed, 20 insertions(+), 21 deletions(-)

diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc
index 0a3a516c2..86d13fab7 100644
--- a/src/libcmd/command.cc
+++ b/src/libcmd/command.cc
@@ -369,28 +369,30 @@ void MixEnvironment::setEnviron()
     return;
 }
 
-void createOutLinks(
-    const std::filesystem::path & outLink,
-    const BuiltPaths & buildables,
-    LocalFSStore & store)
+void createOutLinks(const std::filesystem::path & outLink, const BuiltPaths & buildables, LocalFSStore & store)
 {
     for (const auto & [_i, buildable] : enumerate(buildables)) {
         auto i = _i;
-        std::visit(overloaded {
-            [&](const BuiltPath::Opaque & bo) {
-                auto symlink = outLink;
-                if (i) symlink += fmt("-%d", i);
-                store.addPermRoot(bo.path, absPath(symlink.string()));
-            },
-            [&](const BuiltPath::Built & bfd) {
-                for (auto & output : bfd.outputs) {
+        std::visit(
+            overloaded{
+                [&](const BuiltPath::Opaque & bo) {
                     auto symlink = outLink;
-                    if (i) symlink += fmt("-%d", i);
-                    if (output.first != "out") symlink += fmt("-%s", output.first);
-                    store.addPermRoot(output.second, absPath(symlink.string()));
-                }
+                    if (i)
+                        symlink += fmt("-%d", i);
+                    store.addPermRoot(bo.path, absPath(symlink.string()));
+                },
+                [&](const BuiltPath::Built & bfd) {
+                    for (auto & output : bfd.outputs) {
+                        auto symlink = outLink;
+                        if (i)
+                            symlink += fmt("-%d", i);
+                        if (output.first != "out")
+                            symlink += fmt("-%s", output.first);
+                        store.addPermRoot(output.second, absPath(symlink.string()));
+                    }
+                },
             },
-        }, buildable.raw());
+            buildable.raw());
     }
 }
 
diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh
index 67c208d28..23529848f 100644
--- a/src/libcmd/command.hh
+++ b/src/libcmd/command.hh
@@ -372,9 +372,6 @@ void printClosureDiff(
  * Create symlinks prefixed by `outLink` to the store paths in
  * `buildables`.
  */
-void createOutLinks(
-    const std::filesystem::path & outLink,
-    const BuiltPaths & buildables,
-    LocalFSStore & store);
+void createOutLinks(const std::filesystem::path & outLink, const BuiltPaths & buildables, LocalFSStore & store);
 
 }

From 2f24030bffa8f8f6769ac8ce0002d6c201312933 Mon Sep 17 00:00:00 2001
From: Gavin John <gavinnjohn@gmail.com>
Date: Wed, 20 Nov 2024 13:23:02 -0800
Subject: [PATCH 41/73] Move bug report list to comment and make it more
 nix-specific

---
 .github/ISSUE_TEMPLATE/bug_report.md | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
index 7ea82dcbc..a5005f8a0 100644
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -18,10 +18,13 @@ assignees: ''
 
 ## Steps To Reproduce
 
-1. Go to '...'
-2. Click on '....'
-3. Scroll down to '....'
-4. See error
+<!--
+  Example:
+
+  1. Clone this repository: ...
+  2. Run `nix-... ...`
+  3. Observe unexpected behaviour
+-->
 
 ## Expected behavior
 

From 671df02bf7d0cbf339d2626266260e0b7ccf5988 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 20 Nov 2024 20:42:13 +0100
Subject: [PATCH 42/73] shellcheck

---
 tests/functional/flakes/non-flake-inputs.sh |  2 +-
 tests/functional/flakes/shebang.sh          | 44 ++++++++++-----------
 2 files changed, 22 insertions(+), 24 deletions(-)

diff --git a/tests/functional/flakes/non-flake-inputs.sh b/tests/functional/flakes/non-flake-inputs.sh
index 66dc74e0c..c9c9803d3 100644
--- a/tests/functional/flakes/non-flake-inputs.sh
+++ b/tests/functional/flakes/non-flake-inputs.sh
@@ -71,7 +71,7 @@ nix build -o "$TEST_ROOT/result" "$flake3Dir#sth" --commit-lock-file
 nix registry add --registry "$registry" flake3 "git+file://$flake3Dir"
 
 nix build -o "$TEST_ROOT/result" flake3#fnord
-[[ $(cat $TEST_ROOT/result) = FNORD ]]
+[[ $(cat "$TEST_ROOT/result") = FNORD ]]
 
 # Check whether flake input fetching is lazy: flake3#sth does not
 # depend on flake2, so this shouldn't fail.
diff --git a/tests/functional/flakes/shebang.sh b/tests/functional/flakes/shebang.sh
index 645b1fa38..576a62e2c 100644
--- a/tests/functional/flakes/shebang.sh
+++ b/tests/functional/flakes/shebang.sh
@@ -6,7 +6,7 @@ TODO_NixOS
 
 createFlake1
 
-scriptDir=$TEST_ROOT/nonFlake
+scriptDir="$TEST_ROOT/nonFlake"
 mkdir -p "$scriptDir"
 
 cat > "$scriptDir/shebang.sh" <<EOF
@@ -20,10 +20,8 @@ echo "\$@"
 EOF
 chmod +x "$scriptDir/shebang.sh"
 
-#git -C "$scriptDir" add shebang.sh
-
 # this also tests a fairly trivial double backtick quoted string, ``--command``
-cat > $scriptDir/shebang-comments.sh <<EOF
+cat > "$scriptDir/shebang-comments.sh" <<EOF
 #! $(type -P env) nix
 # some comments
 # some comments
@@ -33,9 +31,9 @@ cat > $scriptDir/shebang-comments.sh <<EOF
 #! nix --no-write-lock-file ``--command`` bash
 foo
 EOF
-chmod +x $scriptDir/shebang-comments.sh
+chmod +x "$scriptDir/shebang-comments.sh"
 
-cat > $scriptDir/shebang-different-comments.sh <<EOF
+cat > "$scriptDir/shebang-different-comments.sh" <<EOF
 #! $(type -P env) nix
 # some comments
 // some comments
@@ -51,9 +49,9 @@ cat > $scriptDir/shebang-different-comments.sh <<EOF
 #! nix --no-write-lock-file --command cat
 foo
 EOF
-chmod +x $scriptDir/shebang-different-comments.sh
+chmod +x "$scriptDir/shebang-different-comments.sh"
 
-cat > $scriptDir/shebang-reject.sh <<EOF
+cat > "$scriptDir/shebang-reject.sh" <<EOF
 #! $(type -P env) nix
 # some comments
 # some comments
@@ -63,12 +61,12 @@ cat > $scriptDir/shebang-reject.sh <<EOF
 #! nix --no-write-lock-file --command bash
 foo
 EOF
-chmod +x $scriptDir/shebang-reject.sh
+chmod +x "$scriptDir/shebang-reject.sh"
 
-cat > $scriptDir/shebang-inline-expr.sh <<EOF
+cat > "$scriptDir/shebang-inline-expr.sh" <<EOF
 #! $(type -P env) nix
 EOF
-cat >> $scriptDir/shebang-inline-expr.sh <<"EOF"
+cat >> "$scriptDir/shebang-inline-expr.sh" <<"EOF"
 #! nix --offline shell
 #! nix --impure --expr ``
 #! nix let flake = (builtins.getFlake (toString ../flake1)).packages;
@@ -81,18 +79,18 @@ set -ex
 foo
 echo "$@"
 EOF
-chmod +x $scriptDir/shebang-inline-expr.sh
+chmod +x "$scriptDir/shebang-inline-expr.sh"
 
-cat > $scriptDir/fooScript.nix <<"EOF"
+cat > "$scriptDir/fooScript.nix" <<"EOF"
 let flake = (builtins.getFlake (toString ../flake1)).packages;
     fooScript = flake.${builtins.currentSystem}.fooScript;
  in fooScript
 EOF
 
-cat > $scriptDir/shebang-file.sh <<EOF
+cat > "$scriptDir/shebang-file.sh" <<EOF
 #! $(type -P env) nix
 EOF
-cat >> $scriptDir/shebang-file.sh <<"EOF"
+cat >> "$scriptDir/shebang-file.sh" <<"EOF"
 #! nix --offline shell
 #! nix --impure --file ./fooScript.nix
 #! nix --no-write-lock-file --command bash
@@ -100,12 +98,12 @@ set -ex
 foo
 echo "$@"
 EOF
-chmod +x $scriptDir/shebang-file.sh
+chmod +x "$scriptDir/shebang-file.sh"
 
-[[ $($scriptDir/shebang.sh) = "foo" ]]
-[[ $($scriptDir/shebang.sh "bar") = "foo"$'\n'"bar" ]]
-[[ $($scriptDir/shebang-comments.sh ) = "foo" ]]
-[[ "$($scriptDir/shebang-different-comments.sh)" = "$(cat $scriptDir/shebang-different-comments.sh)" ]]
-[[ $($scriptDir/shebang-inline-expr.sh baz) = "foo"$'\n'"baz" ]]
-[[ $($scriptDir/shebang-file.sh baz) = "foo"$'\n'"baz" ]]
-expect 1 $scriptDir/shebang-reject.sh 2>&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?'
+[[ $("$scriptDir/shebang.sh") = "foo" ]]
+[[ $("$scriptDir/shebang.sh" "bar") = "foo"$'\n'"bar" ]]
+[[ $("$scriptDir/shebang-comments.sh" ) = "foo" ]]
+[[ "$("$scriptDir/shebang-different-comments.sh")" = "$(cat "$scriptDir/shebang-different-comments.sh")" ]]
+[[ $("$scriptDir/shebang-inline-expr.sh" baz) = "foo"$'\n'"baz" ]]
+[[ $("$scriptDir/shebang-file.sh" baz) = "foo"$'\n'"baz" ]]
+expect 1 "$scriptDir/shebang-reject.sh" 2>&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?'

From e122acef97edd4a1190dba63e8f91b2b363a7df2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 20 Nov 2024 20:55:45 +0100
Subject: [PATCH 43/73] Fix VM test

---
 tests/functional/flakes/non-flake-inputs.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/functional/flakes/non-flake-inputs.sh b/tests/functional/flakes/non-flake-inputs.sh
index c9c9803d3..f5e12cd01 100644
--- a/tests/functional/flakes/non-flake-inputs.sh
+++ b/tests/functional/flakes/non-flake-inputs.sh
@@ -2,6 +2,8 @@
 
 source ./common.sh
 
+TODO_NixOS
+
 createFlake1
 createFlake2
 

From 4a18c783853858b7bf897ac0a69baa3be7248237 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 20 Nov 2024 21:26:20 +0100
Subject: [PATCH 44/73] flake_regressions: Pass -L to nix build

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index cb97fd211..9918875d9 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -220,4 +220,4 @@ jobs:
           path: flake-regressions/tests
       - uses: DeterminateSystems/nix-installer-action@main
       - uses: DeterminateSystems/magic-nix-cache-action@main
-      - run: nix build --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH MAX_FLAKES=25 flake-regressions/eval-all.sh
+      - run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH MAX_FLAKES=25 flake-regressions/eval-all.sh

From f4f4b698f6bbd19cec767686eb3f017567d9fef2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 21 Nov 2024 16:53:34 +0100
Subject: [PATCH 45/73] fetchTree: Don't crash if narHash is missing

Fixes

  nix: ../src/libexpr/primops/fetchTree.cc:37: void nix::emitTreeAttrs(EvalState&, const StorePath&, const fetchers::Input&, Value&, bool, bool): Assertion `narHash' failed.

on a lock file with an input that doesn't have a narHash. This can
happen when using a lock file created by the lazy-trees branch.

Cherry-picked from lazy-trees.
---
 src/libexpr/primops/fetchTree.cc | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index c207da8ad..47d1be1cc 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -33,9 +33,8 @@ void emitTreeAttrs(
 
     // FIXME: support arbitrary input attributes.
 
-    auto narHash = input.getNarHash();
-    assert(narHash);
-    attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true));
+    if (auto narHash = input.getNarHash())
+        attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true));
 
     if (input.getType() == "git")
         attrs.alloc("submodules").mkBool(

From 965ca18db85638d6a04f4ecaff16b48fd661fe33 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 21 Nov 2024 18:44:57 +0100
Subject: [PATCH 46/73] Merge build-utils-meson/{diagnostics,threads} into
 build-utils-meson/common

This reduces the amount of boilerplate. More importantly, it provides
a place to add compiler flags (such as -O3) without having to add it
to every subproject (and the risk of forgetting to include it).
---
 build-utils-meson/{diagnostics => common}/meson.build | 7 +++++++
 build-utils-meson/threads/meson.build                 | 6 ------
 src/libcmd/meson.build                                | 4 +---
 src/libexpr-c/meson.build                             | 4 +---
 src/libexpr-test-support/meson.build                  | 4 +---
 src/libexpr-tests/meson.build                         | 4 +---
 src/libexpr/meson.build                               | 4 +---
 src/libfetchers-tests/meson.build                     | 4 +---
 src/libfetchers/meson.build                           | 4 +---
 src/libflake-tests/meson.build                        | 4 +---
 src/libflake/meson.build                              | 4 +---
 src/libmain-c/meson.build                             | 4 +---
 src/libmain/meson.build                               | 4 +---
 src/libstore-c/meson.build                            | 4 +---
 src/libstore-test-support/meson.build                 | 4 +---
 src/libstore-tests/meson.build                        | 4 +---
 src/libstore/meson.build                              | 3 +--
 src/libutil-c/meson.build                             | 4 +---
 src/libutil-test-support/meson.build                  | 4 +---
 src/libutil-tests/meson.build                         | 4 +---
 src/libutil/meson.build                               | 3 +--
 src/nix/meson.build                                   | 4 +---
 22 files changed, 27 insertions(+), 64 deletions(-)
 rename build-utils-meson/{diagnostics => common}/meson.build (51%)
 delete mode 100644 build-utils-meson/threads/meson.build

diff --git a/build-utils-meson/diagnostics/meson.build b/build-utils-meson/common/meson.build
similarity index 51%
rename from build-utils-meson/diagnostics/meson.build
rename to build-utils-meson/common/meson.build
index 30eedfc13..67b6658f5 100644
--- a/build-utils-meson/diagnostics/meson.build
+++ b/build-utils-meson/common/meson.build
@@ -1,3 +1,10 @@
+# This is only conditional to work around
+# https://github.com/mesonbuild/meson/issues/13293. It should be
+# unconditional.
+if not (host_machine.system() == 'windows' and cxx.get_id() == 'gcc')
+  deps_private += dependency('threads')
+endif
+
 add_project_arguments(
   '-Wdeprecated-copy',
   '-Werror=suggest-override',
diff --git a/build-utils-meson/threads/meson.build b/build-utils-meson/threads/meson.build
deleted file mode 100644
index 294160de1..000000000
--- a/build-utils-meson/threads/meson.build
+++ /dev/null
@@ -1,6 +0,0 @@
-# This is only conditional to work around
-# https://github.com/mesonbuild/meson/issues/13293. It should be
-# unconditional.
-if not (host_machine.system() == 'windows' and cxx.get_id() == 'gcc')
-  deps_private += dependency('threads')
-endif
diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build
index c484cf998..1f27c1614 100644
--- a/src/libcmd/meson.build
+++ b/src/libcmd/meson.build
@@ -30,8 +30,6 @@ deps_public_maybe_subproject = [
 ]
 subdir('build-utils-meson/subprojects')
 
-subdir('build-utils-meson/threads')
-
 nlohmann_json = dependency('nlohmann_json', version : '>= 3.9')
 deps_public += nlohmann_json
 
@@ -72,7 +70,7 @@ add_project_arguments(
   language : 'cpp',
 )
 
-subdir('build-utils-meson/diagnostics')
+subdir('build-utils-meson/common')
 
 sources = files(
   'built-path.cc',
diff --git a/src/libexpr-c/meson.build b/src/libexpr-c/meson.build
index 4160f0d5a..5bcca29e0 100644
--- a/src/libexpr-c/meson.build
+++ b/src/libexpr-c/meson.build
@@ -29,8 +29,6 @@ deps_public_maybe_subproject = [
 ]
 subdir('build-utils-meson/subprojects')
 
-subdir('build-utils-meson/threads')
-
 # TODO rename, because it will conflict with downstream projects
 configdata.set_quoted('PACKAGE_VERSION', meson.project_version())
 
@@ -55,7 +53,7 @@ add_project_arguments(
   language : 'cpp',
 )
 
-subdir('build-utils-meson/diagnostics')
+subdir('build-utils-meson/common')
 
 sources = files(
   'nix_api_expr.cc',
diff --git a/src/libexpr-test-support/meson.build b/src/libexpr-test-support/meson.build
index b9e7f390d..bdfd435a8 100644
--- a/src/libexpr-test-support/meson.build
+++ b/src/libexpr-test-support/meson.build
@@ -27,8 +27,6 @@ deps_public_maybe_subproject = [
 ]
 subdir('build-utils-meson/subprojects')
 
-subdir('build-utils-meson/threads')
-
 rapidcheck = dependency('rapidcheck')
 deps_public += rapidcheck
 
@@ -41,7 +39,7 @@ add_project_arguments(
   language : 'cpp',
 )
 
-subdir('build-utils-meson/diagnostics')
+subdir('build-utils-meson/common')
 
 sources = files(
   'tests/value/context.cc',
diff --git a/src/libexpr-tests/meson.build b/src/libexpr-tests/meson.build
index 5a5c9f1d4..b50c18c9c 100644
--- a/src/libexpr-tests/meson.build
+++ b/src/libexpr-tests/meson.build
@@ -25,8 +25,6 @@ deps_public_maybe_subproject = [
 ]
 subdir('build-utils-meson/subprojects')
 
-subdir('build-utils-meson/threads')
-
 subdir('build-utils-meson/export-all-symbols')
 subdir('build-utils-meson/windows-version')
 
@@ -51,7 +49,7 @@ add_project_arguments(
   language : 'cpp',
 )
 
-subdir('build-utils-meson/diagnostics')
+subdir('build-utils-meson/common')
 
 sources = files(
   'derived-path.cc',
diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build
index 4d8a38b43..28318579e 100644
--- a/src/libexpr/meson.build
+++ b/src/libexpr/meson.build
@@ -27,8 +27,6 @@ deps_public_maybe_subproject = [
 ]
 subdir('build-utils-meson/subprojects')
 
-subdir('build-utils-meson/threads')
-
 boost = dependency(
   'boost',
   modules : ['container', 'context'],
@@ -79,7 +77,7 @@ add_project_arguments(
   language : 'cpp',
 )
 
-subdir('build-utils-meson/diagnostics')
+subdir('build-utils-meson/common')
 
 parser_tab = custom_target(
   input : 'parser.y',
diff --git a/src/libfetchers-tests/meson.build b/src/libfetchers-tests/meson.build
index d948dbad6..fdab6ba6c 100644
--- a/src/libfetchers-tests/meson.build
+++ b/src/libfetchers-tests/meson.build
@@ -24,8 +24,6 @@ deps_public_maybe_subproject = [
 ]
 subdir('build-utils-meson/subprojects')
 
-subdir('build-utils-meson/threads')
-
 subdir('build-utils-meson/export-all-symbols')
 subdir('build-utils-meson/windows-version')
 
@@ -44,7 +42,7 @@ add_project_arguments(
   language : 'cpp',
 )
 
-subdir('build-utils-meson/diagnostics')
+subdir('build-utils-meson/common')
 
 sources = files(
   'public-key.cc',
diff --git a/src/libfetchers/meson.build b/src/libfetchers/meson.build
index ff638578d..07a1178cc 100644
--- a/src/libfetchers/meson.build
+++ b/src/libfetchers/meson.build
@@ -26,8 +26,6 @@ deps_public_maybe_subproject = [
 ]
 subdir('build-utils-meson/subprojects')
 
-subdir('build-utils-meson/threads')
-
 nlohmann_json = dependency('nlohmann_json', version : '>= 3.9')
 deps_public += nlohmann_json
 
@@ -43,7 +41,7 @@ add_project_arguments(
   language : 'cpp',
 )
 
-subdir('build-utils-meson/diagnostics')
+subdir('build-utils-meson/common')
 
 sources = files(
   'attrs.cc',
diff --git a/src/libflake-tests/meson.build b/src/libflake-tests/meson.build
index 592a7493b..c0a9b8847 100644
--- a/src/libflake-tests/meson.build
+++ b/src/libflake-tests/meson.build
@@ -24,8 +24,6 @@ deps_public_maybe_subproject = [
 ]
 subdir('build-utils-meson/subprojects')
 
-subdir('build-utils-meson/threads')
-
 subdir('build-utils-meson/export-all-symbols')
 subdir('build-utils-meson/windows-version')
 
@@ -44,7 +42,7 @@ add_project_arguments(
   language : 'cpp',
 )
 
-subdir('build-utils-meson/diagnostics')
+subdir('build-utils-meson/common')
 
 sources = files(
   'flakeref.cc',
diff --git a/src/libflake/meson.build b/src/libflake/meson.build
index d2bb179df..2c1a70a18 100644
--- a/src/libflake/meson.build
+++ b/src/libflake/meson.build
@@ -26,8 +26,6 @@ deps_public_maybe_subproject = [
 ]
 subdir('build-utils-meson/subprojects')
 
-subdir('build-utils-meson/threads')
-
 nlohmann_json = dependency('nlohmann_json', version : '>= 3.9')
 deps_public += nlohmann_json
 
@@ -41,7 +39,7 @@ add_project_arguments(
   language : 'cpp',
 )
 
-subdir('build-utils-meson/diagnostics')
+subdir('build-utils-meson/common')
 
 sources = files(
   'flake/config.cc',
diff --git a/src/libmain-c/meson.build b/src/libmain-c/meson.build
index 0ec0e3f6d..3cb1e4baa 100644
--- a/src/libmain-c/meson.build
+++ b/src/libmain-c/meson.build
@@ -29,8 +29,6 @@ deps_public_maybe_subproject = [
 ]
 subdir('build-utils-meson/subprojects')
 
-subdir('build-utils-meson/threads')
-
 # TODO rename, because it will conflict with downstream projects
 configdata.set_quoted('PACKAGE_VERSION', meson.project_version())
 
@@ -55,7 +53,7 @@ add_project_arguments(
   language : 'cpp',
 )
 
-subdir('build-utils-meson/diagnostics')
+subdir('build-utils-meson/common')
 
 sources = files(
   'nix_api_main.cc',
diff --git a/src/libmain/meson.build b/src/libmain/meson.build
index 7fcadf06d..6c6298e2b 100644
--- a/src/libmain/meson.build
+++ b/src/libmain/meson.build
@@ -26,8 +26,6 @@ deps_public_maybe_subproject = [
 ]
 subdir('build-utils-meson/subprojects')
 
-subdir('build-utils-meson/threads')
-
 pubsetbuf_test = '''
 #include <iostream>
 
@@ -60,7 +58,7 @@ add_project_arguments(
   language : 'cpp',
 )
 
-subdir('build-utils-meson/diagnostics')
+subdir('build-utils-meson/common')
 
 sources = files(
   'common-args.cc',
diff --git a/src/libstore-c/meson.build b/src/libstore-c/meson.build
index d4f86eeff..44b5fe11d 100644
--- a/src/libstore-c/meson.build
+++ b/src/libstore-c/meson.build
@@ -27,8 +27,6 @@ deps_public_maybe_subproject = [
 ]
 subdir('build-utils-meson/subprojects')
 
-subdir('build-utils-meson/threads')
-
 # TODO rename, because it will conflict with downstream projects
 configdata.set_quoted('PACKAGE_VERSION', meson.project_version())
 
@@ -51,7 +49,7 @@ add_project_arguments(
   language : 'cpp',
 )
 
-subdir('build-utils-meson/diagnostics')
+subdir('build-utils-meson/common')
 
 sources = files(
   'nix_api_store.cc',
diff --git a/src/libstore-test-support/meson.build b/src/libstore-test-support/meson.build
index 98ec9882e..f8308e7bb 100644
--- a/src/libstore-test-support/meson.build
+++ b/src/libstore-test-support/meson.build
@@ -25,8 +25,6 @@ deps_public_maybe_subproject = [
 ]
 subdir('build-utils-meson/subprojects')
 
-subdir('build-utils-meson/threads')
-
 rapidcheck = dependency('rapidcheck')
 deps_public += rapidcheck
 
@@ -38,7 +36,7 @@ add_project_arguments(
   language : 'cpp',
 )
 
-subdir('build-utils-meson/diagnostics')
+subdir('build-utils-meson/common')
 
 sources = files(
   'tests/derived-path.cc',
diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build
index f4f67d73a..fc9152f2f 100644
--- a/src/libstore-tests/meson.build
+++ b/src/libstore-tests/meson.build
@@ -25,8 +25,6 @@ deps_public_maybe_subproject = [
 ]
 subdir('build-utils-meson/subprojects')
 
-subdir('build-utils-meson/threads')
-
 subdir('build-utils-meson/export-all-symbols')
 subdir('build-utils-meson/windows-version')
 
@@ -52,7 +50,7 @@ add_project_arguments(
   language : 'cpp',
 )
 
-subdir('build-utils-meson/diagnostics')
+subdir('build-utils-meson/common')
 
 sources = files(
   'common-protocol.cc',
diff --git a/src/libstore/meson.build b/src/libstore/meson.build
index 101879e90..f836b8d4f 100644
--- a/src/libstore/meson.build
+++ b/src/libstore/meson.build
@@ -82,7 +82,6 @@ if host_machine.system() == 'windows'
 endif
 
 subdir('build-utils-meson/libatomic')
-subdir('build-utils-meson/threads')
 
 boost = dependency(
   'boost',
@@ -180,7 +179,7 @@ add_project_arguments(
   language : 'cpp',
 )
 
-subdir('build-utils-meson/diagnostics')
+subdir('build-utils-meson/common')
 
 sources = files(
   'binary-cache-store.cc',
diff --git a/src/libutil-c/meson.build b/src/libutil-c/meson.build
index 3d5a0b9c2..d44453676 100644
--- a/src/libutil-c/meson.build
+++ b/src/libutil-c/meson.build
@@ -25,8 +25,6 @@ deps_public_maybe_subproject = [
 ]
 subdir('build-utils-meson/subprojects')
 
-subdir('build-utils-meson/threads')
-
 # TODO rename, because it will conflict with downstream projects
 configdata.set_quoted('PACKAGE_VERSION', meson.project_version())
 
@@ -47,7 +45,7 @@ add_project_arguments(
   language : 'cpp',
 )
 
-subdir('build-utils-meson/diagnostics')
+subdir('build-utils-meson/common')
 
 sources = files(
   'nix_api_util.cc',
diff --git a/src/libutil-test-support/meson.build b/src/libutil-test-support/meson.build
index c5e1ba80b..fa1df7320 100644
--- a/src/libutil-test-support/meson.build
+++ b/src/libutil-test-support/meson.build
@@ -23,8 +23,6 @@ deps_public_maybe_subproject = [
 ]
 subdir('build-utils-meson/subprojects')
 
-subdir('build-utils-meson/threads')
-
 rapidcheck = dependency('rapidcheck')
 deps_public += rapidcheck
 
@@ -35,7 +33,7 @@ add_project_arguments(
   language : 'cpp',
 )
 
-subdir('build-utils-meson/diagnostics')
+subdir('build-utils-meson/common')
 
 sources = files(
   'tests/hash.cc',
diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build
index 5c3b5e5a3..f59350774 100644
--- a/src/libutil-tests/meson.build
+++ b/src/libutil-tests/meson.build
@@ -25,8 +25,6 @@ deps_public_maybe_subproject = [
 ]
 subdir('build-utils-meson/subprojects')
 
-subdir('build-utils-meson/threads')
-
 subdir('build-utils-meson/export-all-symbols')
 subdir('build-utils-meson/windows-version')
 
@@ -44,7 +42,7 @@ add_project_arguments(
   language : 'cpp',
 )
 
-subdir('build-utils-meson/diagnostics')
+subdir('build-utils-meson/common')
 
 sources = files(
   'args.cc',
diff --git a/src/libutil/meson.build b/src/libutil/meson.build
index a6dc86394..11b4ea592 100644
--- a/src/libutil/meson.build
+++ b/src/libutil/meson.build
@@ -54,7 +54,6 @@ endforeach
 configdata.set('HAVE_DECL_AT_SYMLINK_NOFOLLOW', cxx.has_header_symbol('fcntl.h', 'AT_SYMLINK_NOFOLLOW').to_int())
 
 subdir('build-utils-meson/libatomic')
-subdir('build-utils-meson/threads')
 
 if host_machine.system() == 'windows'
   socket = cxx.find_library('ws2_32')
@@ -121,7 +120,7 @@ add_project_arguments(
   language : 'cpp',
 )
 
-subdir('build-utils-meson/diagnostics')
+subdir('build-utils-meson/common')
 
 sources = files(
   'archive.cc',
diff --git a/src/nix/meson.build b/src/nix/meson.build
index 60ee48035..5c70c8216 100644
--- a/src/nix/meson.build
+++ b/src/nix/meson.build
@@ -32,8 +32,6 @@ deps_public_maybe_subproject = [
 ]
 subdir('build-utils-meson/subprojects')
 
-subdir('build-utils-meson/threads')
-
 subdir('build-utils-meson/export-all-symbols')
 subdir('build-utils-meson/windows-version')
 
@@ -65,7 +63,7 @@ add_project_arguments(
   language : 'cpp',
 )
 
-subdir('build-utils-meson/diagnostics')
+subdir('build-utils-meson/common')
 subdir('build-utils-meson/generate-header')
 
 nix_sources = [config_h] + files(

From ed120a61abc35b2a0a782dfce3935013cc108d50 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 21 Nov 2024 19:16:27 +0100
Subject: [PATCH 47/73] Use -O3 again

This was lost in the switch to the new build system. -O3 provides
around a 10% performance gain compared to -O2, see
e.g. nix-env.qaAggressive.time in
https://hydra.nixos.org/job/nix/master/metrics.nixpkgs#tabs-charts.
---
 build-utils-meson/common/meson.build | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/build-utils-meson/common/meson.build b/build-utils-meson/common/meson.build
index 67b6658f5..f0322183e 100644
--- a/build-utils-meson/common/meson.build
+++ b/build-utils-meson/common/meson.build
@@ -16,3 +16,7 @@ add_project_arguments(
   '-Wno-deprecated-declarations',
   language : 'cpp',
 )
+
+if get_option('buildtype') not in ['debug']
+  add_project_arguments('-O3', language : 'cpp')
+endif

From ba074465ba249da4df0752d0dc53529d0ab1a60d Mon Sep 17 00:00:00 2001
From: Vladimir Panteleev <CyberShadow@users.noreply.github.com>
Date: Thu, 21 Nov 2024 20:08:13 +0000
Subject: [PATCH 48/73] doc: Clarify that nix-shell still uses shell from host
 environment (#8809)

* doc: Clarify that nix-shell still uses shell from host environment

* doc: Fix NIX_BUILD_SHELL description

* doc: Add anchor and link to NIX_BUILD_SHELL

* doc: Add example of default shell trickiness

Co-authored-by: Valentin Gagarin <valentin@gagarin.work>
---
 doc/manual/source/command-ref/nix-shell.md | 31 ++++++++++++++++++----
 1 file changed, 26 insertions(+), 5 deletions(-)

diff --git a/doc/manual/source/command-ref/nix-shell.md b/doc/manual/source/command-ref/nix-shell.md
index 69a711bd5..e95db9bea 100644
--- a/doc/manual/source/command-ref/nix-shell.md
+++ b/doc/manual/source/command-ref/nix-shell.md
@@ -88,7 +88,9 @@ All options not listed here are passed to `nix-store
   cleared before the interactive shell is started, so you get an
   environment that more closely corresponds to the “real” Nix build. A
   few variables, in particular `HOME`, `USER` and `DISPLAY`, are
-  retained.
+  retained.  Note that the shell used to run commands is obtained from
+  [`NIX_BUILD_SHELL`](#env-NIX_BUILD_SHELL) / `<nixpkgs>` from
+  `NIX_PATH`, and therefore not affected by `--pure`.
 
 - `--packages` / `-p` *packages*…
 
@@ -112,11 +114,30 @@ All options not listed here are passed to `nix-store
 
 # Environment variables
 
-- `NIX_BUILD_SHELL`
+- <span id="env-NIX_BUILD_SHELL">[`NIX_BUILD_SHELL`](#env-NIX_BUILD_SHELL)</span>
 
-  Shell used to start the interactive environment. Defaults to the
-  `bash` found in `<nixpkgs>`, falling back to the `bash` found in
-  `PATH` if not found.
+  Shell used to start the interactive environment.
+  Defaults to the `bash` from `bashInteractive` found in `<nixpkgs>`, falling back to the `bash` found in `PATH` if not found.
+
+  > **Note**
+  >
+  > The shell obtained using this method may not necessarily be the same as any shells requested in *path*.
+
+  <!-- -->
+
+  > **Example
+  >
+  >  Despite `--pure`, this invocation will not result in a fully reproducible shell environment:
+  >
+  > ```nix
+  > #!/usr/bin/env -S nix-shell --pure
+  > let
+  >   pkgs = import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/854fdc68881791812eddd33b2fed94b954979a8e.tar.gz") {};
+  > in
+  > pkgs.mkShell {
+  >   buildInputs = pkgs.bashInteractive;
+  > }
+  > ```
 
 {{#include ./env-common.md}}
 

From ebb19cc1cdc2f04c291e2e8e34f434e5defa5bbd Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 22 Nov 2024 09:14:01 +0100
Subject: [PATCH 49/73] Drop std::make_pair
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Jörg Thalheim <Mic92@users.noreply.github.com>
---
 src/libflake/flake/flakeref.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc
index ed47ad737..cdcdcf87f 100644
--- a/src/libflake/flake/flakeref.cc
+++ b/src/libflake/flake/flakeref.cc
@@ -78,7 +78,7 @@ static std::pair<FlakeRef, std::string> fromParsedURL(
     std::string fragment;
     std::swap(fragment, parsedURL.fragment);
 
-    return std::make_pair(FlakeRef(fetchers::Input::fromURL(fetchSettings, parsedURL, isFlake), dir), fragment);
+    return {FlakeRef(fetchers::Input::fromURL(fetchSettings, parsedURL, isFlake), dir), fragment};
 }
 
 std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(

From 756758d968375c3f346412f8f01c82a9178689fb Mon Sep 17 00:00:00 2001
From: Sergei Zimmerman <145775305+xokdvium@users.noreply.github.com>
Date: Fri, 22 Nov 2024 18:05:52 +0300
Subject: [PATCH 50/73] chore: get rid of dead code and unused variables where
 appropriate

Looks like some cruft has been left over from previous refactorings.
This removes dead variables, which should not have side effects in their
constructors. In cases where the variable initialization has a purpose
[[maybe_unused]] is inserted to silence compiler warnings.
---
 src/libexpr/attr-path.cc                         | 1 -
 src/libexpr/primops.cc                           | 2 --
 src/libfetchers/git.cc                           | 2 --
 src/libflake/flake/flakeref.cc                   | 2 --
 src/libstore/build/derivation-goal.cc            | 2 +-
 src/libstore/export-import.cc                    | 5 -----
 src/libstore/unix/build/local-derivation-goal.cc | 2 +-
 src/libutil-tests/nix_api_util.cc                | 1 -
 src/libutil/serialise.cc                         | 3 +--
 src/libutil/terminal.cc                          | 2 +-
 src/nix/search.cc                                | 1 -
 src/nix/sigs.cc                                  | 1 -
 12 files changed, 4 insertions(+), 20 deletions(-)

diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc
index 2f67260c5..822ec7620 100644
--- a/src/libexpr/attr-path.cc
+++ b/src/libexpr/attr-path.cc
@@ -129,7 +129,6 @@ std::pair<SourcePath, uint32_t> findPackageFilename(EvalState & state, Value & v
     try {
         auto colon = fn.rfind(':');
         if (colon == std::string::npos) fail();
-        std::string filename(fn, 0, colon);
         auto lineno = std::stoi(std::string(fn, colon + 1, std::string::npos));
         return {SourcePath{path.accessor, CanonPath(fn.substr(0, colon))}, lineno};
     } catch (std::invalid_argument & e) {
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 53bfce6c5..5d2f75373 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -66,14 +66,12 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS
                 ensureValid(b.drvPath->getBaseStorePath());
             },
             [&](const NixStringContextElem::Opaque & o) {
-                auto ctxS = store->printStorePath(o.path);
                 ensureValid(o.path);
                 if (maybePathsOut)
                     maybePathsOut->emplace(o.path);
             },
             [&](const NixStringContextElem::DrvDeep & d) {
                 /* Treat same as Opaque */
-                auto ctxS = store->printStorePath(d.drvPath);
                 ensureValid(d.drvPath);
                 if (maybePathsOut)
                     maybePathsOut->emplace(d.drvPath);
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 99d91919e..a6883a2d3 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -514,8 +514,6 @@ struct GitInputScheme : InputScheme
 
         auto origRev = input.getRev();
 
-        std::string name = input.getName();
-
         auto originalRef = input.getRef();
         auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo);
         input.attrs.insert_or_assign("ref", ref);
diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc
index cdcdcf87f..9616fe0ea 100644
--- a/src/libflake/flake/flakeref.cc
+++ b/src/libflake/flake/flakeref.cc
@@ -257,8 +257,6 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
 {
     using namespace fetchers;
 
-    std::smatch match;
-
     if (auto res = parseFlakeIdRef(fetchSettings, url, isFlake)) {
         return *res;
     } else if (auto res = parseURLFlakeRef(fetchSettings, url, baseDir, isFlake)) {
diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 794be1568..bf1a25db1 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -1229,7 +1229,7 @@ HookReply DerivationGoal::tryBuildHook()
     hook->toHook.writeSide.close();
 
     /* Create the log file and pipe. */
-    Path logFile = openLogFile();
+    [[maybe_unused]] Path logFile = openLogFile();
 
     std::set<MuxablePipePollState::CommChannel> fds;
     fds.insert(hook->fromHook.readSide.get());
diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc
index cb36c0c1b..1c62cdfad 100644
--- a/src/libstore/export-import.cc
+++ b/src/libstore/export-import.cc
@@ -13,14 +13,9 @@ void Store::exportPaths(const StorePathSet & paths, Sink & sink)
     auto sorted = topoSortPaths(paths);
     std::reverse(sorted.begin(), sorted.end());
 
-    std::string doneLabel("paths exported");
-    //logger->incExpected(doneLabel, sorted.size());
-
     for (auto & path : sorted) {
-        //Activity act(*logger, lvlInfo, "exporting path '%s'", path);
         sink << 1;
         exportPath(path, sink);
-        //logger->incProgress(doneLabel);
     }
 
     sink << 0;
diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc
index dcfaadeef..06a2f85be 100644
--- a/src/libstore/unix/build/local-derivation-goal.cc
+++ b/src/libstore/unix/build/local-derivation-goal.cc
@@ -883,7 +883,7 @@ void LocalDerivationGoal::startBuilder()
         printMsg(lvlVomit, "setting builder env variable '%1%'='%2%'", i.first, i.second);
 
     /* Create the log file. */
-    Path logFile = openLogFile();
+    [[maybe_unused]] Path logFile = openLogFile();
 
     /* Create a pseudoterminal to get the output of the builder. */
     builderOut = posix_openpt(O_RDWR | O_NOCTTY);
diff --git a/src/libutil-tests/nix_api_util.cc b/src/libutil-tests/nix_api_util.cc
index b36f71042..7b77bd87f 100644
--- a/src/libutil-tests/nix_api_util.cc
+++ b/src/libutil-tests/nix_api_util.cc
@@ -136,7 +136,6 @@ TEST_F(nix_api_util_context, nix_err_name)
     // no error
     EXPECT_THROW(nix_err_name(NULL, ctx, OBSERVE_STRING(err_name)), nix::Error);
 
-    std::string err_msg_ref;
     try {
         throw nix::Error("testing error");
     } catch (...) {
diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index 168d2ed32..381e7ae38 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -90,7 +90,6 @@ void Source::operator () (std::string_view data)
 
 void Source::drainInto(Sink & sink)
 {
-    std::string s;
     std::array<char, 8192> buf;
     while (true) {
         size_t n;
@@ -427,7 +426,7 @@ Error readError(Source & source)
     auto type = readString(source);
     assert(type == "Error");
     auto level = (Verbosity) readInt(source);
-    auto name = readString(source); // removed
+    [[maybe_unused]] auto name = readString(source); // removed
     auto msg = readString(source);
     ErrorInfo info {
         .level = level,
diff --git a/src/libutil/terminal.cc b/src/libutil/terminal.cc
index db7a6fcd1..4c127ddb0 100644
--- a/src/libutil/terminal.cc
+++ b/src/libutil/terminal.cc
@@ -26,7 +26,7 @@ bool isTTY()
 
 std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int width)
 {
-    std::string t, e;
+    std::string t;
     size_t w = 0;
     auto i = s.begin();
 
diff --git a/src/nix/search.cc b/src/nix/search.cc
index c8d0b9e96..30b96c500 100644
--- a/src/nix/search.cc
+++ b/src/nix/search.cc
@@ -161,7 +161,6 @@ struct CmdSearch : InstallableValueCommand, MixJSON
                                 {"description", description},
                             };
                         } else {
-                            auto name2 = hiliteMatches(name.name, nameMatches, ANSI_GREEN, "\e[0;2m");
                             if (results > 1) logger->cout("");
                             logger->cout(
                                 "* %s%s",
diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc
index 2afe4b267..134d4f34a 100644
--- a/src/nix/sigs.cc
+++ b/src/nix/sigs.cc
@@ -41,7 +41,6 @@ struct CmdCopySigs : StorePathsCommand
 
         ThreadPool pool;
 
-        std::string doneLabel = "done";
         std::atomic<size_t> added{0};
 
         //logger->setExpected(doneLabel, storePaths.size());

From 09ddc34b62bf762cbe7e0a9adce4bdea9ff7fc6a Mon Sep 17 00:00:00 2001
From: Sergei Zimmerman <145775305+xokdvium@users.noreply.github.com>
Date: Sun, 24 Nov 2024 00:01:11 +0300
Subject: [PATCH 51/73] refactor(libfetchers/registry): use standard remove_if
 + erase

Get rid of this fixme. This does not appear to be used anywhere in
the nix codebase itself. Not sure why the comment mentioned C++20 erase
member function with predicate, but iterator-based algorithms are also fine.
---
 src/libfetchers/registry.cc | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc
index 7f7a09053..c761028ab 100644
--- a/src/libfetchers/registry.cc
+++ b/src/libfetchers/registry.cc
@@ -94,12 +94,9 @@ void Registry::add(
 
 void Registry::remove(const Input & input)
 {
-    // FIXME: use C++20 std::erase.
-    for (auto i = entries.begin(); i != entries.end(); )
-        if (i->from == input)
-            i = entries.erase(i);
-        else
-            ++i;
+    entries.erase(
+        std::remove_if(entries.begin(), entries.end(), [&](const Entry & entry) { return entry.from == input; }),
+        entries.end());
 }
 
 static Path getSystemRegistryPath()

From fbffd47fb715396f47be21ae0ed1a7f484852b4b Mon Sep 17 00:00:00 2001
From: Sergei Zimmerman <145775305+xokdvium@users.noreply.github.com>
Date: Sun, 24 Nov 2024 00:35:47 +0300
Subject: [PATCH 52/73] build(lib{expr,store,util}-test-support): depend on -c
 libraries

Since lib{expr,store,util}-test-support subprojects define nix_api_* helpers
for testing nix c bindings, they need to publicly depend on -c counterparts.
This makes their headers self-sufficient and does not rely on the -tests to add
necessary dependencies.
---
 src/libexpr-test-support/meson.build  | 1 +
 src/libexpr-test-support/package.nix  | 2 ++
 src/libstore-test-support/meson.build | 1 +
 src/libstore-test-support/package.nix | 2 ++
 src/libutil-test-support/meson.build  | 1 +
 src/libutil-test-support/package.nix  | 2 ++
 6 files changed, 9 insertions(+)

diff --git a/src/libexpr-test-support/meson.build b/src/libexpr-test-support/meson.build
index bdfd435a8..33d9e17a6 100644
--- a/src/libexpr-test-support/meson.build
+++ b/src/libexpr-test-support/meson.build
@@ -24,6 +24,7 @@ deps_public_maybe_subproject = [
   dependency('nix-store'),
   dependency('nix-store-test-support'),
   dependency('nix-expr'),
+  dependency('nix-expr-c'),
 ]
 subdir('build-utils-meson/subprojects')
 
diff --git a/src/libexpr-test-support/package.nix b/src/libexpr-test-support/package.nix
index bcf6118e0..7e92d145f 100644
--- a/src/libexpr-test-support/package.nix
+++ b/src/libexpr-test-support/package.nix
@@ -4,6 +4,7 @@
 
 , nix-store-test-support
 , nix-expr
+, nix-expr-c
 
 , rapidcheck
 
@@ -35,6 +36,7 @@ mkMesonLibrary (finalAttrs: {
   propagatedBuildInputs = [
     nix-store-test-support
     nix-expr
+    nix-expr-c
     rapidcheck
   ];
 
diff --git a/src/libstore-test-support/meson.build b/src/libstore-test-support/meson.build
index f8308e7bb..1f230914f 100644
--- a/src/libstore-test-support/meson.build
+++ b/src/libstore-test-support/meson.build
@@ -22,6 +22,7 @@ deps_public_maybe_subproject = [
   dependency('nix-util'),
   dependency('nix-util-test-support'),
   dependency('nix-store'),
+  dependency('nix-store-c'),
 ]
 subdir('build-utils-meson/subprojects')
 
diff --git a/src/libstore-test-support/package.nix b/src/libstore-test-support/package.nix
index 48f8b5e6b..2543049fe 100644
--- a/src/libstore-test-support/package.nix
+++ b/src/libstore-test-support/package.nix
@@ -4,6 +4,7 @@
 
 , nix-util-test-support
 , nix-store
+, nix-store-c
 
 , rapidcheck
 
@@ -35,6 +36,7 @@ mkMesonLibrary (finalAttrs: {
   propagatedBuildInputs = [
     nix-util-test-support
     nix-store
+    nix-store-c
     rapidcheck
   ];
 
diff --git a/src/libutil-test-support/meson.build b/src/libutil-test-support/meson.build
index fa1df7320..4afed01ca 100644
--- a/src/libutil-test-support/meson.build
+++ b/src/libutil-test-support/meson.build
@@ -20,6 +20,7 @@ deps_private_maybe_subproject = [
 ]
 deps_public_maybe_subproject = [
   dependency('nix-util'),
+  dependency('nix-util-c'),
 ]
 subdir('build-utils-meson/subprojects')
 
diff --git a/src/libutil-test-support/package.nix b/src/libutil-test-support/package.nix
index 2525e1602..c403e762c 100644
--- a/src/libutil-test-support/package.nix
+++ b/src/libutil-test-support/package.nix
@@ -3,6 +3,7 @@
 , mkMesonLibrary
 
 , nix-util
+, nix-util-c
 
 , rapidcheck
 
@@ -33,6 +34,7 @@ mkMesonLibrary (finalAttrs: {
 
   propagatedBuildInputs = [
     nix-util
+    nix-util-c
     rapidcheck
   ];
 

From 4145d18435f5e7073f0c99f813d45dd9058430dd Mon Sep 17 00:00:00 2001
From: Anatoli Babenia <anatoli@rainforce.org>
Date: Sat, 23 Nov 2024 08:36:51 +0300
Subject: [PATCH 53/73] Rename install-nix-from-closure.sh into
 install-nix-from-tarball.sh

Because it is only used as /install script from tarball.
---
 maintainers/flake-module.nix                                    | 2 +-
 scripts/binary-tarball.nix                                      | 2 +-
 ...{install-nix-from-closure.sh => install-nix-from-tarball.sh} | 0
 3 files changed, 2 insertions(+), 2 deletions(-)
 rename scripts/{install-nix-from-closure.sh => install-nix-from-tarball.sh} (100%)

diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix
index fdb031302..ba6cd2816 100644
--- a/maintainers/flake-module.nix
+++ b/maintainers/flake-module.nix
@@ -496,7 +496,7 @@
             ''^scripts/create-darwin-volume\.sh$''
             ''^scripts/install-darwin-multi-user\.sh$''
             ''^scripts/install-multi-user\.sh$''
-            ''^scripts/install-nix-from-closure\.sh$''
+            ''^scripts/install-nix-from-tarball\.sh$''
             ''^scripts/install-systemd-multi-user\.sh$''
             ''^src/nix/get-env\.sh$''
             ''^tests/functional/ca/build-dry\.sh$''
diff --git a/scripts/binary-tarball.nix b/scripts/binary-tarball.nix
index 104189b0c..671c8e96e 100644
--- a/scripts/binary-tarball.nix
+++ b/scripts/binary-tarball.nix
@@ -23,7 +23,7 @@ in
 runCommand "nix-binary-tarball-${version}" env ''
   cp ${installerClosureInfo}/registration $TMPDIR/reginfo
   cp ${./create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh
-  substitute ${./install-nix-from-closure.sh} $TMPDIR/install \
+  substitute ${./install-nix-from-tarball.sh} $TMPDIR/install \
     --subst-var-by nix ${nix} \
     --subst-var-by cacert ${cacert}
 
diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-tarball.sh
similarity index 100%
rename from scripts/install-nix-from-closure.sh
rename to scripts/install-nix-from-tarball.sh

From 82a23d9b6b96bf08e7c28008fcc346c0bdb671be Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 22 Nov 2024 16:24:20 +0100
Subject: [PATCH 54/73] libexpr-c: Add nix_eval_state_builder

---
 src/libexpr-c/nix_api_expr.cc         | 79 ++++++++++++++++++++++++---
 src/libexpr-c/nix_api_expr.h          | 65 +++++++++++++++++++++-
 src/libexpr-c/nix_api_expr_internal.h | 11 ++++
 src/libexpr-tests/nix_api_expr.cc     | 37 +++++++++++++
 4 files changed, 183 insertions(+), 9 deletions(-)

diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc
index 6144a7986..27a70afb6 100644
--- a/src/libexpr-c/nix_api_expr.cc
+++ b/src/libexpr-c/nix_api_expr.cc
@@ -6,6 +6,7 @@
 #include "eval-gc.hh"
 #include "globals.hh"
 #include "eval-settings.hh"
+#include "ref.hh"
 
 #include "nix_api_expr.h"
 #include "nix_api_expr_internal.h"
@@ -93,7 +94,46 @@ nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_val
     NIXC_CATCH_ERRS
 }
 
-EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c, Store * store)
+nix_eval_state_builder * nix_eval_state_builder_new(nix_c_context * context, Store * store)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        // Allocate ahead of time, because .settings needs self-reference
+        void * p = ::operator new(
+            sizeof(nix_eval_state_builder),
+            static_cast<std::align_val_t>(alignof(nix_eval_state_builder)));
+        auto * p2 = static_cast<nix_eval_state_builder *>(p);
+        new (p) nix_eval_state_builder{
+            .store = nix::ref<nix::Store>(store->ptr),
+            .settings = nix::EvalSettings{/* &bool */ p2->readOnlyMode},
+            .fetchSettings = nix::fetchers::Settings{},
+            .readOnlyMode = true,
+        };
+        return p2;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+void nix_eval_state_builder_free(nix_eval_state_builder * builder)
+{
+    delete builder;
+}
+
+nix_err nix_eval_state_builder_load(nix_c_context * context, nix_eval_state_builder * builder)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        // TODO: load in one go?
+        builder->settings.readOnlyMode = nix::settings.readOnlyMode;
+        loadConfFile(builder->settings);
+        loadConfFile(builder->fetchSettings);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_eval_state_builder_set_lookup_path(nix_c_context * context, nix_eval_state_builder * builder, const char ** lookupPath_c)
 {
     if (context)
         context->last_err_code = NIX_OK;
@@ -102,28 +142,51 @@ EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c
         if (lookupPath_c != nullptr)
             for (size_t i = 0; lookupPath_c[i] != nullptr; i++)
                 lookupPath.push_back(lookupPath_c[i]);
+        builder->lookupPath = nix::LookupPath::parse(lookupPath);
+    }
+    NIXC_CATCH_ERRS
+}
 
+EvalState * nix_eval_state_build(nix_c_context * context, nix_eval_state_builder * builder)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        // Allocate ahead of time, because .state init needs self-reference
         void * p = ::operator new(
             sizeof(EvalState),
             static_cast<std::align_val_t>(alignof(EvalState)));
         auto * p2 = static_cast<EvalState *>(p);
         new (p) EvalState {
-            .fetchSettings = nix::fetchers::Settings{},
-            .settings = nix::EvalSettings{
-                nix::settings.readOnlyMode,
-            },
+            .fetchSettings = std::move(builder->fetchSettings),
+            .settings = std::move(builder->settings),
             .state = nix::EvalState(
-                nix::LookupPath::parse(lookupPath),
-                store->ptr,
+                builder->lookupPath,
+                builder->store,
                 p2->fetchSettings,
                 p2->settings),
         };
-        loadConfFile(p2->settings);
         return p2;
     }
     NIXC_CATCH_ERRS_NULL
 }
 
+EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c, Store * store)
+{
+    auto builder = nix_eval_state_builder_new(context, store);
+    if (builder == nullptr)
+        return nullptr;
+
+    if (nix_eval_state_builder_load(context, builder) != NIX_OK)
+        return nullptr;
+
+    if (nix_eval_state_builder_set_lookup_path(context, builder, lookupPath_c)
+            != NIX_OK)
+        return nullptr;
+
+    return nix_eval_state_build(context, builder);
+}
+
 void nix_state_free(EvalState * state)
 {
     delete state;
diff --git a/src/libexpr-c/nix_api_expr.h b/src/libexpr-c/nix_api_expr.h
index e680f5ff1..f8d181452 100644
--- a/src/libexpr-c/nix_api_expr.h
+++ b/src/libexpr-c/nix_api_expr.h
@@ -30,6 +30,11 @@ extern "C" {
 // cffi start
 
 // Type definitions
+/**
+ * @brief Builder for EvalState
+ */
+typedef struct nix_eval_state_builder nix_eval_state_builder;
+
 /**
  * @brief Represents a state of the Nix language evaluator.
  *
@@ -174,12 +179,70 @@ nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value *
 nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_value * value);
 
 /**
- * @brief Create a new Nix language evaluator state.
+ * @brief Create a new nix_eval_state_builder
+ *
+ * The settings are initialized to their default value.
+ * Values can be sourced elsewhere with nix_eval_state_builder_load.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] store The Nix store to use.
+ * @return A new nix_eval_state_builder or NULL on failure.
+ */
+nix_eval_state_builder * nix_eval_state_builder_new(nix_c_context * context, Store * store);
+
+/**
+ * @brief Read settings from the ambient environment
+ *
+ * Settings are sourced from environment variables and configuration files,
+ * as documented in the Nix manual.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[out] builder The builder to modify.
+ * @return NIX_OK if successful, an error code otherwise.
+ */
+nix_err nix_eval_state_builder_load(nix_c_context * context, nix_eval_state_builder * builder);
+
+/**
+ * @brief Set the lookup path for `<...>` expressions
+ *
+ * @param[in] context Optional, stores error information
+ * @param[in] builder The builder to modify.
+ * @param[in] lookupPath Null-terminated array of strings corresponding to entries in NIX_PATH.
+ */
+nix_err nix_eval_state_builder_set_lookup_path(
+    nix_c_context * context, nix_eval_state_builder * builder, const char ** lookupPath);
+
+/**
+ * @brief Create a new Nix language evaluator state
+ *
+ * Remember to nix_eval_state_builder_free after building the state.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] builder The builder to use and free
+ * @return A new Nix state or NULL on failure.
+ * @see nix_eval_state_builder_new, nix_eval_state_builder_free
+ */
+EvalState * nix_eval_state_build(nix_c_context * context, nix_eval_state_builder * builder);
+
+/**
+ * @brief Free a nix_eval_state_builder
+ *
+ * Does not fail.
+ *
+ * @param[in] builder The builder to free.
+ */
+void nix_eval_state_builder_free(nix_eval_state_builder * builder);
+
+/**
+ * @brief Create a new Nix language evaluator state
+ *
+ * For more control, use nix_eval_state_builder
  *
  * @param[out] context Optional, stores error information
  * @param[in] lookupPath Null-terminated array of strings corresponding to entries in NIX_PATH.
  * @param[in] store The Nix store to use.
  * @return A new Nix state or NULL on failure.
+ * @see nix_state_builder_new
  */
 EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath, Store * store);
 
diff --git a/src/libexpr-c/nix_api_expr_internal.h b/src/libexpr-c/nix_api_expr_internal.h
index 12f24b6eb..f59664011 100644
--- a/src/libexpr-c/nix_api_expr_internal.h
+++ b/src/libexpr-c/nix_api_expr_internal.h
@@ -6,6 +6,17 @@
 #include "eval-settings.hh"
 #include "attr-set.hh"
 #include "nix_api_value.h"
+#include "search-path.hh"
+
+struct nix_eval_state_builder
+{
+    nix::ref<nix::Store> store;
+    nix::EvalSettings settings;
+    nix::fetchers::Settings fetchSettings;
+    nix::LookupPath lookupPath;
+    // TODO: make an EvalSettings setting own this instead?
+    bool readOnlyMode;
+};
 
 struct EvalState
 {
diff --git a/src/libexpr-tests/nix_api_expr.cc b/src/libexpr-tests/nix_api_expr.cc
index b37ac44b3..5ed78d2fc 100644
--- a/src/libexpr-tests/nix_api_expr.cc
+++ b/src/libexpr-tests/nix_api_expr.cc
@@ -7,12 +7,49 @@
 
 #include "tests/nix_api_expr.hh"
 #include "tests/string_callback.hh"
+#include "file-system.hh"
 
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
 
 namespace nixC {
 
+TEST_F(nix_api_store_test, nix_eval_state_lookup_path)
+{
+    auto tmpDir = nix::createTempDir();
+    auto delTmpDir = std::make_unique<nix::AutoDelete>(tmpDir, true);
+    auto nixpkgs = tmpDir + "/pkgs";
+    auto nixos = tmpDir + "/cfg";
+    std::filesystem::create_directories(nixpkgs);
+    std::filesystem::create_directories(nixos);
+
+    std::string nixpkgsEntry = "nixpkgs=" + nixpkgs;
+    std::string nixosEntry = "nixos-config=" + nixos;
+    const char * lookupPath[] = {nixpkgsEntry.c_str(), nixosEntry.c_str(), nullptr};
+
+    auto builder = nix_eval_state_builder_new(ctx, store);
+    assert_ctx_ok();
+
+    ASSERT_EQ(NIX_OK, nix_eval_state_builder_set_lookup_path(ctx, builder, lookupPath));
+    assert_ctx_ok();
+
+    auto state = nix_eval_state_build(ctx, builder);
+    assert_ctx_ok();
+
+    nix_eval_state_builder_free(builder);
+
+    Value * value = nix_alloc_value(ctx, state);
+    nix_expr_eval_from_string(ctx, state, "builtins.seq <nixos-config> <nixpkgs>", ".", value);
+    assert_ctx_ok();
+
+    ASSERT_EQ(nix_get_type(ctx, value), NIX_TYPE_PATH);
+    assert_ctx_ok();
+
+    auto pathStr = nix_get_path_string(ctx, value);
+    assert_ctx_ok();
+    ASSERT_EQ(0, strcmp(pathStr, nixpkgs.c_str()));
+}
+
 TEST_F(nix_api_expr_test, nix_expr_eval_from_string)
 {
     nix_expr_eval_from_string(nullptr, state, "builtins.nixVersion", ".", value);

From 1bd75178017102e098e7b9a6aa2ddf858b486b53 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 22 Nov 2024 16:27:17 +0100
Subject: [PATCH 55/73] Doc nix_get_path_string

---
 src/libexpr-c/nix_api_value.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr-c/nix_api_value.h b/src/libexpr-c/nix_api_value.h
index 8a0813ebe..711b0adbc 100644
--- a/src/libexpr-c/nix_api_value.h
+++ b/src/libexpr-c/nix_api_value.h
@@ -213,7 +213,7 @@ nix_get_string(nix_c_context * context, const nix_value * value, nix_get_string_
 /** @brief Get path as string
  * @param[out] context Optional, stores error information
  * @param[in] value Nix value to inspect
- * @return string
+ * @return string, if the type is NIX_TYPE_PATH
  * @return NULL in case of error.
  */
 const char * nix_get_path_string(nix_c_context * context, const nix_value * value);

From f06f611ff3f1e7beee51171b3ab13d4883e187f3 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 22 Nov 2024 17:26:26 +0100
Subject: [PATCH 56/73] refactor: Extract unsafe_new_with_self

---
 src/libexpr-c/nix_api_expr.cc | 69 +++++++++++++++++++++--------------
 1 file changed, 42 insertions(+), 27 deletions(-)

diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc
index 27a70afb6..a024248cd 100644
--- a/src/libexpr-c/nix_api_expr.cc
+++ b/src/libexpr-c/nix_api_expr.cc
@@ -19,6 +19,29 @@
 #  include <mutex>
 #endif
 
+/**
+ * @brief Allocate and initialize using self-reference
+ *
+ * This allows a brace initializer to reference the object being constructed.
+ *
+ * @warning Use with care, as the pointer points to an object that is not fully constructed yet.
+ *
+ * @tparam T Type to allocate
+ * @tparam F A function type for `init`, taking a T* and returning the initializer for T
+ * @param init Function that takes a T* and returns the initializer for T
+ * @return Pointer to allocated and initialized object
+ */
+template <typename T, typename F>
+static T * unsafe_new_with_self(F && init)
+{
+    // Allocate
+    void * p = ::operator new(
+        sizeof(T),
+        static_cast<std::align_val_t>(alignof(T)));
+    // Initialize with placement new
+    return new (p) T(init(static_cast<T *>(p)));
+}
+
 nix_err nix_libexpr_init(nix_c_context * context)
 {
     if (context)
@@ -99,18 +122,14 @@ nix_eval_state_builder * nix_eval_state_builder_new(nix_c_context * context, Sto
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        // Allocate ahead of time, because .settings needs self-reference
-        void * p = ::operator new(
-            sizeof(nix_eval_state_builder),
-            static_cast<std::align_val_t>(alignof(nix_eval_state_builder)));
-        auto * p2 = static_cast<nix_eval_state_builder *>(p);
-        new (p) nix_eval_state_builder{
-            .store = nix::ref<nix::Store>(store->ptr),
-            .settings = nix::EvalSettings{/* &bool */ p2->readOnlyMode},
-            .fetchSettings = nix::fetchers::Settings{},
-            .readOnlyMode = true,
-        };
-        return p2;
+        return unsafe_new_with_self<nix_eval_state_builder>([&](auto * self) {
+            return nix_eval_state_builder{
+                .store = nix::ref<nix::Store>(store->ptr),
+                .settings = nix::EvalSettings{/* &bool */ self->readOnlyMode},
+                .fetchSettings = nix::fetchers::Settings{},
+                .readOnlyMode = true,
+            };
+        });
     }
     NIXC_CATCH_ERRS_NULL
 }
@@ -152,21 +171,17 @@ EvalState * nix_eval_state_build(nix_c_context * context, nix_eval_state_builder
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        // Allocate ahead of time, because .state init needs self-reference
-        void * p = ::operator new(
-            sizeof(EvalState),
-            static_cast<std::align_val_t>(alignof(EvalState)));
-        auto * p2 = static_cast<EvalState *>(p);
-        new (p) EvalState {
-            .fetchSettings = std::move(builder->fetchSettings),
-            .settings = std::move(builder->settings),
-            .state = nix::EvalState(
-                builder->lookupPath,
-                builder->store,
-                p2->fetchSettings,
-                p2->settings),
-        };
-        return p2;
+        return unsafe_new_with_self<EvalState>([&](auto * self) {
+            return EvalState{
+                .fetchSettings = std::move(builder->fetchSettings),
+                .settings = std::move(builder->settings),
+                .state = nix::EvalState(
+                    builder->lookupPath,
+                    builder->store,
+                    self->fetchSettings,
+                    self->settings),
+            };
+        });
     }
     NIXC_CATCH_ERRS_NULL
 }

From 4eecf3c20ab454b4427363a276d757298f9220dc Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 22 Nov 2024 16:18:55 +0100
Subject: [PATCH 57/73] Add nix-flake-c, nix_flake_init_global,
 nix_flake_settings_new

---
 meson.build                              |  1 +
 packaging/components.nix                 |  1 +
 packaging/everything.nix                 |  4 +
 src/external-api-docs/doxygen.cfg.in     |  1 +
 src/external-api-docs/package.nix        |  1 +
 src/libflake-c/.version                  |  1 +
 src/libflake-c/build-utils-meson         |  1 +
 src/libflake-c/meson.build               | 93 ++++++++++++++++++++++++
 src/libflake-c/nix_api_flake.cc          | 32 ++++++++
 src/libflake-c/nix_api_flake.h           | 46 ++++++++++++
 src/libflake-c/nix_api_flake_internal.hh |  9 +++
 src/libflake-c/package.nix               | 60 +++++++++++++++
 src/libflake-tests/meson.build           |  3 +
 src/libflake-tests/nix_api_flake.cc      | 51 +++++++++++++
 src/libflake-tests/package.nix           |  3 +
 15 files changed, 307 insertions(+)
 create mode 120000 src/libflake-c/.version
 create mode 120000 src/libflake-c/build-utils-meson
 create mode 100644 src/libflake-c/meson.build
 create mode 100644 src/libflake-c/nix_api_flake.cc
 create mode 100644 src/libflake-c/nix_api_flake.h
 create mode 100644 src/libflake-c/nix_api_flake_internal.hh
 create mode 100644 src/libflake-c/package.nix
 create mode 100644 src/libflake-tests/nix_api_flake.cc

diff --git a/meson.build b/meson.build
index 8985b631e..49adf9832 100644
--- a/meson.build
+++ b/meson.build
@@ -34,6 +34,7 @@ endif
 subproject('libutil-c')
 subproject('libstore-c')
 subproject('libexpr-c')
+subproject('libflake-c')
 subproject('libmain-c')
 
 # Language Bindings
diff --git a/packaging/components.nix b/packaging/components.nix
index c29e04ae9..e1f661be8 100644
--- a/packaging/components.nix
+++ b/packaging/components.nix
@@ -44,6 +44,7 @@ in
   nix-expr-tests = callPackage ../src/libexpr-tests/package.nix { };
 
   nix-flake = callPackage ../src/libflake/package.nix { };
+  nix-flake-c = callPackage ../src/libflake-c/package.nix { };
   nix-flake-tests = callPackage ../src/libflake-tests/package.nix { };
 
   nix-main = callPackage ../src/libmain/package.nix { };
diff --git a/packaging/everything.nix b/packaging/everything.nix
index b09b9d2a9..0b04d2c6d 100644
--- a/packaging/everything.nix
+++ b/packaging/everything.nix
@@ -19,6 +19,7 @@
   nix-expr-tests,
 
   nix-flake,
+  nix-flake-c,
   nix-flake-tests,
 
   nix-main,
@@ -53,6 +54,7 @@ let
       nix-expr-c
       nix-fetchers
       nix-flake
+      nix-flake-c
       nix-main
       nix-main-c
       nix-store
@@ -86,6 +88,7 @@ let
         "nix-expr-c"
         "nix-fetchers"
         "nix-flake"
+        "nix-flake-c"
         "nix-main"
         "nix-main-c"
         "nix-store"
@@ -169,6 +172,7 @@ in
         nix-expr
         nix-expr-c
         nix-flake
+        nix-flake-c
         nix-main
         nix-main-c
       ;
diff --git a/src/external-api-docs/doxygen.cfg.in b/src/external-api-docs/doxygen.cfg.in
index 8e235dae5..3af2f5b81 100644
--- a/src/external-api-docs/doxygen.cfg.in
+++ b/src/external-api-docs/doxygen.cfg.in
@@ -40,6 +40,7 @@ GENERATE_LATEX         = NO
 INPUT                  = \
   @src@/src/libutil-c \
   @src@/src/libexpr-c \
+  @src@/src/libflake-c \
   @src@/src/libstore-c \
   @src@/src/external-api-docs/README.md
 
diff --git a/src/external-api-docs/package.nix b/src/external-api-docs/package.nix
index 0c592955a..57c5138cf 100644
--- a/src/external-api-docs/package.nix
+++ b/src/external-api-docs/package.nix
@@ -30,6 +30,7 @@ mkMesonDerivation (finalAttrs: {
       # Source is not compiled, but still must be available for Doxygen
       # to gather comments.
       (cpp ../libexpr-c)
+      (cpp ../libflake-c)
       (cpp ../libstore-c)
       (cpp ../libutil-c)
     ];
diff --git a/src/libflake-c/.version b/src/libflake-c/.version
new file mode 120000
index 000000000..b7badcd0c
--- /dev/null
+++ b/src/libflake-c/.version
@@ -0,0 +1 @@
+../../.version
\ No newline at end of file
diff --git a/src/libflake-c/build-utils-meson b/src/libflake-c/build-utils-meson
new file mode 120000
index 000000000..91937f183
--- /dev/null
+++ b/src/libflake-c/build-utils-meson
@@ -0,0 +1 @@
+../../build-utils-meson/
\ No newline at end of file
diff --git a/src/libflake-c/meson.build b/src/libflake-c/meson.build
new file mode 100644
index 000000000..00d9650e7
--- /dev/null
+++ b/src/libflake-c/meson.build
@@ -0,0 +1,93 @@
+project('nix-flake-c', 'cpp',
+  version : files('.version'),
+  default_options : [
+    'cpp_std=c++2a',
+    # TODO(Qyriad): increase the warning level
+    'warning_level=1',
+    'debug=true',
+    'optimization=2',
+    'errorlogs=true', # Please print logs for tests that fail
+  ],
+  meson_version : '>= 1.1',
+  license : 'LGPL-2.1-or-later',
+)
+
+cxx = meson.get_compiler('cpp')
+
+subdir('build-utils-meson/deps-lists')
+
+configdata = configuration_data()
+
+deps_private_maybe_subproject = [
+  dependency('nix-util'),
+  dependency('nix-store'),
+  dependency('nix-expr'),
+  dependency('nix-flake'),
+]
+deps_public_maybe_subproject = [
+  dependency('nix-util-c'),
+  dependency('nix-store-c'),
+  dependency('nix-expr-c'),
+]
+subdir('build-utils-meson/subprojects')
+
+# TODO rename, because it will conflict with downstream projects
+configdata.set_quoted('PACKAGE_VERSION', meson.project_version())
+
+config_h = configure_file(
+  configuration : configdata,
+  output : 'config-flake.h',
+)
+
+add_project_arguments(
+  # TODO(Qyriad): Yes this is how the autoconf+Make system did it.
+  # It would be nice for our headers to be idempotent instead.
+
+  # From C++ libraries, only for internals
+  '-include', 'config-util.hh',
+  '-include', 'config-store.hh',
+  '-include', 'config-expr.hh',
+  # not generated (yet?)
+  # '-include', 'config-flake.hh',
+
+  # From C libraries, for our public, installed headers too
+  '-include', 'config-util.h',
+  '-include', 'config-store.h',
+  '-include', 'config-expr.h',
+  '-include', 'config-flake.h',
+  language : 'cpp',
+)
+
+subdir('build-utils-meson/common')
+
+sources = files(
+  'nix_api_flake.cc',
+)
+
+include_dirs = [include_directories('.')]
+
+headers = [config_h] + files(
+  'nix_api_flake.h',
+)
+
+# TODO move this header to libexpr, maybe don't use it in tests?
+headers += files('nix_api_flake.h')
+
+subdir('build-utils-meson/export-all-symbols')
+subdir('build-utils-meson/windows-version')
+
+this_library = library(
+  'nixflakec',
+  sources,
+  dependencies : deps_public + deps_private + deps_other,
+  include_directories : include_dirs,
+  link_args: linker_export_flags,
+  prelink : true, # For C++ static initializers
+  install : true,
+)
+
+install_headers(headers, subdir : 'nix', preserve_path : true)
+
+libraries_private = []
+
+subdir('build-utils-meson/export')
diff --git a/src/libflake-c/nix_api_flake.cc b/src/libflake-c/nix_api_flake.cc
new file mode 100644
index 000000000..17cf6572d
--- /dev/null
+++ b/src/libflake-c/nix_api_flake.cc
@@ -0,0 +1,32 @@
+#include "nix_api_flake.h"
+#include "nix_api_flake_internal.hh"
+#include "nix_api_util_internal.h"
+
+#include "flake/flake.hh"
+
+nix_flake_settings * nix_flake_settings_new(nix_c_context * context)
+{
+    try {
+        auto settings = nix::make_ref<nix::flake::Settings>();
+        return new nix_flake_settings{settings};
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+void nix_flake_settings_free(nix_flake_settings * settings)
+{
+    delete settings;
+}
+
+nix_err nix_flake_init_global(nix_c_context * context, nix_flake_settings * settings)
+{
+    static std::shared_ptr<nix::flake::Settings> registeredSettings;
+    try {
+        if (registeredSettings)
+            throw nix::Error("nix_flake_init_global already initialized");
+
+        registeredSettings = settings->settings;
+        nix::flake::initLib(*registeredSettings);
+    }
+    NIXC_CATCH_ERRS
+}
diff --git a/src/libflake-c/nix_api_flake.h b/src/libflake-c/nix_api_flake.h
new file mode 100644
index 000000000..80051298d
--- /dev/null
+++ b/src/libflake-c/nix_api_flake.h
@@ -0,0 +1,46 @@
+#ifndef NIX_API_FLAKE_H
+#define NIX_API_FLAKE_H
+/** @defgroup libflake libflake
+ * @brief Bindings to the Nix Flakes library
+ *
+ * @{
+ */
+/** @file
+ * @brief Main entry for the libflake C bindings
+ */
+
+#include "nix_api_store.h"
+#include "nix_api_util.h"
+#include "nix_api_expr.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+// cffi start
+
+typedef struct nix_flake_settings nix_flake_settings;
+
+// Function prototypes
+/**
+ * Create a nix_flake_settings initialized with default values.
+ * @param[out] context Optional, stores error information
+ * @return A new nix_flake_settings or NULL on failure.
+ * @see nix_flake_settings_free
+ */
+nix_flake_settings * nix_flake_settings_new(nix_c_context * context);
+
+/**
+ * @brief Release the resources associated with a nix_flake_settings.
+ */
+void nix_flake_settings_free(nix_flake_settings * settings);
+
+/**
+ * @brief Register Flakes support process-wide.
+ */
+nix_err nix_flake_init_global(nix_c_context * context, nix_flake_settings * settings);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
+#endif
diff --git a/src/libflake-c/nix_api_flake_internal.hh b/src/libflake-c/nix_api_flake_internal.hh
new file mode 100644
index 000000000..4c154a342
--- /dev/null
+++ b/src/libflake-c/nix_api_flake_internal.hh
@@ -0,0 +1,9 @@
+#pragma once
+
+#include "ref.hh"
+#include "flake/settings.hh"
+
+struct nix_flake_settings
+{
+    nix::ref<nix::flake::Settings> settings;
+};
diff --git a/src/libflake-c/package.nix b/src/libflake-c/package.nix
new file mode 100644
index 000000000..a70cbf94e
--- /dev/null
+++ b/src/libflake-c/package.nix
@@ -0,0 +1,60 @@
+{ lib
+, stdenv
+, mkMesonLibrary
+
+, nix-store-c
+, nix-expr-c
+, nix-flake
+
+# Configuration Options
+
+, version
+}:
+
+let
+  inherit (lib) fileset;
+in
+
+mkMesonLibrary (finalAttrs: {
+  pname = "nix-flake-c";
+  inherit version;
+
+  workDir = ./.;
+  fileset = fileset.unions [
+    ../../build-utils-meson
+    ./build-utils-meson
+    ../../.version
+    ./.version
+    ./meson.build
+    # ./meson.options
+    (fileset.fileFilter (file: file.hasExt "cc") ./.)
+    (fileset.fileFilter (file: file.hasExt "hh") ./.)
+    (fileset.fileFilter (file: file.hasExt "h") ./.)
+  ];
+
+  propagatedBuildInputs = [
+    nix-expr-c
+    nix-store-c
+    nix-flake
+  ];
+
+  preConfigure =
+    # "Inline" .version so it's not a symlink, and includes the suffix.
+    # Do the meson utils, without modification.
+    ''
+      chmod u+w ./.version
+      echo ${version} > ../../.version
+    '';
+
+  mesonFlags = [
+  ];
+
+  env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) {
+    LDFLAGS = "-fuse-ld=gold";
+  };
+
+  meta = {
+    platforms = lib.platforms.unix ++ lib.platforms.windows;
+  };
+
+})
diff --git a/src/libflake-tests/meson.build b/src/libflake-tests/meson.build
index c0a9b8847..c494c414e 100644
--- a/src/libflake-tests/meson.build
+++ b/src/libflake-tests/meson.build
@@ -19,6 +19,7 @@ subdir('build-utils-meson/deps-lists')
 deps_private_maybe_subproject = [
   dependency('nix-expr-test-support'),
   dependency('nix-flake'),
+  dependency('nix-flake-c'),
 ]
 deps_public_maybe_subproject = [
 ]
@@ -46,6 +47,7 @@ subdir('build-utils-meson/common')
 
 sources = files(
   'flakeref.cc',
+  'nix_api_flake.cc',
   'url-name.cc',
 )
 
@@ -68,6 +70,7 @@ test(
   this_exe,
   env : {
     '_NIX_TEST_UNIT_DATA': meson.current_source_dir() / 'data',
+    'NIX_CONFIG': 'extra-experimental-features = flakes',
   },
   protocol : 'gtest',
 )
diff --git a/src/libflake-tests/nix_api_flake.cc b/src/libflake-tests/nix_api_flake.cc
new file mode 100644
index 000000000..21109d181
--- /dev/null
+++ b/src/libflake-tests/nix_api_flake.cc
@@ -0,0 +1,51 @@
+#include "nix_api_store.h"
+#include "nix_api_store_internal.h"
+#include "nix_api_util.h"
+#include "nix_api_util_internal.h"
+#include "nix_api_expr.h"
+#include "nix_api_value.h"
+#include "nix_api_flake.h"
+
+#include "tests/nix_api_expr.hh"
+#include "tests/string_callback.hh"
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+namespace nixC {
+
+TEST_F(nix_api_store_test, nix_api_init_global_getFlake_exists)
+{
+    nix_libstore_init(ctx);
+    assert_ctx_ok();
+    nix_libexpr_init(ctx);
+    assert_ctx_ok();
+
+    auto settings = nix_flake_settings_new(ctx);
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, settings);
+
+    nix_flake_init_global(ctx, settings);
+    assert_ctx_ok();
+
+    nix_eval_state_builder * builder = nix_eval_state_builder_new(ctx, store);
+    ASSERT_NE(nullptr, builder);
+    assert_ctx_ok();
+
+    auto state = nix_eval_state_build(ctx, builder);
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, state);
+
+    nix_eval_state_builder_free(builder);
+
+    auto value = nix_alloc_value(ctx, state);
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, value);
+
+    nix_err err = nix_expr_eval_from_string(ctx, state, "builtins.getFlake", ".", value);
+    assert_ctx_ok();
+    ASSERT_EQ(NIX_OK, err);
+    ASSERT_EQ(NIX_TYPE_FUNCTION, nix_get_type(ctx, value));
+}
+
+} // namespace nixC
diff --git a/src/libflake-tests/package.nix b/src/libflake-tests/package.nix
index 67e716979..b3a8ac466 100644
--- a/src/libflake-tests/package.nix
+++ b/src/libflake-tests/package.nix
@@ -4,6 +4,7 @@
 , mkMesonExecutable
 
 , nix-flake
+, nix-flake-c
 , nix-expr-test-support
 
 , rapidcheck
@@ -38,6 +39,7 @@ mkMesonExecutable (finalAttrs: {
 
   buildInputs = [
     nix-flake
+    nix-flake-c
     nix-expr-test-support
     rapidcheck
     gtest
@@ -67,6 +69,7 @@ mkMesonExecutable (finalAttrs: {
         mkdir -p "$HOME"
       '' + ''
         export _NIX_TEST_UNIT_DATA=${resolvePath ./data}
+        export NIX_CONFIG="extra-experimental-features = flakes"
         ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage}
         touch $out
       '');

From d004c524b84651c2eebb5bbb55f6a3a8324437e9 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 24 Nov 2024 13:52:40 +0100
Subject: [PATCH 58/73] test: Change FAIL to throw

[FAIL()] is a macro with `return`, making it unsuitable for helpers.

This uses std::runtime_error, because gtest does not seem to provide an
exception type of its own for this purpose. [AssertionException] is for
a different use case.

[FAIL()]: https://google.github.io/googletest/reference/assertions.html#FAIL
[AssertionException]: https://github.com/google/googletest/blob/35d0c365609296fa4730d62057c487e3cfa030ff/docs/reference/testing.md#assertionexception-assertionexception
---
 src/libutil-test-support/tests/gtest-with-params.hh | 2 +-
 src/libutil-test-support/tests/nix_api_util.hh      | 5 ++---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/libutil-test-support/tests/gtest-with-params.hh b/src/libutil-test-support/tests/gtest-with-params.hh
index d72aec4fd..a6e23ad89 100644
--- a/src/libutil-test-support/tests/gtest-with-params.hh
+++ b/src/libutil-test-support/tests/gtest-with-params.hh
@@ -40,7 +40,7 @@ void checkGTestWith(Testable && testable, MakeTestParams makeTestParams)
     } else {
         std::ostringstream ss;
         printResultMessage(result, ss);
-        FAIL() << ss.str() << std::endl;
+        throw std::runtime_error(ss.str());
     }
 }
 }
diff --git a/src/libutil-test-support/tests/nix_api_util.hh b/src/libutil-test-support/tests/nix_api_util.hh
index efd200116..006dc497c 100644
--- a/src/libutil-test-support/tests/nix_api_util.hh
+++ b/src/libutil-test-support/tests/nix_api_util.hh
@@ -26,14 +26,13 @@ protected:
 
     inline void assert_ctx_ok()
     {
-
         if (nix_err_code(ctx) == NIX_OK) {
             return;
         }
         unsigned int n;
         const char * p = nix_err_msg(nullptr, ctx, &n);
         std::string msg(p, n);
-        FAIL() << "nix_err_code(ctx) != NIX_OK, message: " << msg;
+        throw std::runtime_error(std::string("nix_err_code(ctx) != NIX_OK, message: ") + msg);
     }
 
     inline void assert_ctx_err()
@@ -41,7 +40,7 @@ protected:
         if (nix_err_code(ctx) != NIX_OK) {
             return;
         }
-        FAIL() << "Got NIX_OK, but expected an error!";
+        throw std::runtime_error("Got NIX_OK, but expected an error!");
     }
 };
 

From 6db6b269ed70788314209d35499812c90949057f Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 25 Nov 2024 01:16:02 +0100
Subject: [PATCH 59/73] .github/ci: Set max-jobs to 1, to reduce peak memory
 usage

---
 .github/workflows/ci.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 9918875d9..be96bb484 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -23,7 +23,9 @@ jobs:
     - uses: cachix/install-nix-action@v30
       with:
         # The sandbox would otherwise be disabled by default on Darwin
-        extra_nix_config: "sandbox = true"
+        extra_nix_config: |
+          sandbox = true
+          max-jobs = 1
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
     - uses: cachix/cachix-action@v15
       if: needs.check_secrets.outputs.cachix == 'true'

From 6502dc4d6af5baad369578ee0b4d2e1295d199a7 Mon Sep 17 00:00:00 2001
From: Robert Hensing <roberth@users.noreply.github.com>
Date: Mon, 25 Nov 2024 12:06:54 +0100
Subject: [PATCH 60/73] ci(Mergify): configuration update

Signed-off-by: Robert Hensing <null>
---
 .mergify.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/.mergify.yml b/.mergify.yml
index c297d3d5e..86623a138 100644
--- a/.mergify.yml
+++ b/.mergify.yml
@@ -90,3 +90,13 @@ pull_request_rules:
           - "2.24-maintenance"
         labels:
           - merge-queue
+
+  - name: backport patches to 2.25
+    conditions:
+      - label=backport 2.25-maintenance
+    actions:
+      backport:
+        branches:
+          - "2.25-maintenance"
+        labels:
+          - merge-queue

From 3fb7481e64fc22313211c6c4ea79ac314457f81b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Sun, 24 Nov 2024 11:17:17 +0100
Subject: [PATCH 61/73] source-accessor: fix case where normalization goes
 beyond root

fixes https://github.com/NixOS/nix/issues/11936
---
 src/libutil/source-accessor.cc | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc
index e797951c7..d3e304f74 100644
--- a/src/libutil/source-accessor.cc
+++ b/src/libutil/source-accessor.cc
@@ -84,9 +84,10 @@ CanonPath SourceAccessor::resolveSymlinks(
         todo.pop_front();
         if (c == "" || c == ".")
             ;
-        else if (c == "..")
-            res.pop();
-        else {
+        else if (c == "..") {
+            if (!res.isRoot())
+                res.pop();
+        } else {
             res.push(c);
             if (mode == SymlinkResolution::Full || !todo.empty()) {
                 if (auto st = maybeLstat(res); st && st->type == SourceAccessor::tSymlink) {

From 57fea81f8a6ab3b91b1003d27bd48effad30b25a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 25 Nov 2024 15:59:43 +0100
Subject: [PATCH 62/73] Work around gcc warning
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This shuts up a 300-line warning that includes

  /nix/store/fg7ass3a5m5pgl26qzfdniicbwbgzccy-gcc-13.2.0/include/c++/13.2.0/bits/stl_tree.h:182:25: warning: ‘*(std::_Rb_tree_header*)((char*)&<unnamed> + offsetof(nix::value_type, nix::DerivedPath::<unnamed>.std::variant<nix::DerivedPathOpaque, nix::DerivedPathBuilt>::<unnamed>.std::__detail::__variant::_Variant_base<nix::DerivedPathOpaque, nix::DerivedPathBuilt>::<unnamed>.std::__detail::__variant::_Move_assign_base<false, nix::DerivedPathOpaque, nix::DerivedPathBuilt>::<unnamed>.std::__detail::__variant::_Copy_assign_base<false, nix::DerivedPathOpaque, nix::DerivedPathBuilt>::<unnamed>.std::__detail::__variant::_Move_ctor_base<false, nix::DerivedPathOpaque, nix::DerivedPathBuilt>::<unnamed>.std::__detail::__variant::_Copy_ctor_base<false, nix::DerivedPathOpaque, nix::DerivedPathBuilt>::<unnamed>.std::__detail::__variant::_Variant_storage<false, nix::DerivedPathOpaque, nix::DerivedPathBuilt>::_M_u) + 24).std::_Rb_tree_header::_M_header.std::_Rb_tree_node_base::_M_parent’ may be used uninitialized [-Wmaybe-uninitialized]
    182 |       if (__x._M_header._M_parent != nullptr)
        |           ~~~~~~~~~~~~~~^~~~~~~~~
---
 src/nix/flake.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index ce2faacb0..925b66d3e 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -643,10 +643,11 @@ struct CmdFlakeCheck : FlakeCommand
                                             fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]),
                                             *attr2.value, attr2.pos);
                                         if (drvPath && attr_name == settings.thisSystem.get()) {
-                                            drvPaths.push_back(DerivedPath::Built {
+                                            auto path = DerivedPath::Built {
                                                 .drvPath = makeConstantStorePathRef(*drvPath),
                                                 .outputs = OutputsSpec::All { },
-                                            });
+                                            };
+                                            drvPaths.push_back(std::move(path));
                                         }
                                     }
                                 }

From fafaec5ac35d517a6e6217416336346b7353ea05 Mon Sep 17 00:00:00 2001
From: Sergei Zimmerman <145775305+xokdvium@users.noreply.github.com>
Date: Sun, 24 Nov 2024 01:05:08 +0300
Subject: [PATCH 63/73] fix(treewide): remove unnecessary copying in range for
 loops

This gets rid of unnecessary copies in range-based-for loops and
local variables, when they are used solely as `const &`.

Also added a fixme comment about a suspicious move out of const,
which might not be intended.
---
 src/libcmd/installables.cc                         | 2 +-
 src/libexpr/primops.cc                             | 2 +-
 src/libexpr/primops/context.cc                     | 2 ++
 src/libexpr/primops/fetchClosure.cc                | 2 +-
 src/libstore/build/drv-output-substitution-goal.cc | 2 +-
 src/libstore/build/substitution-goal.cc            | 2 +-
 src/libstore/keys.cc                               | 4 ++--
 src/libstore/local-overlay-store.cc                | 2 +-
 src/libstore/nar-info.cc                           | 2 +-
 src/libstore/s3-binary-cache-store.cc              | 2 +-
 src/libstore/store-api.cc                          | 4 ++--
 src/libutil-tests/hilite.cc                        | 3 +--
 src/libutil/args.cc                                | 2 +-
 src/libutil/url.cc                                 | 2 +-
 src/nix-store/nix-store.cc                         | 2 +-
 src/nix/develop.cc                                 | 2 +-
 src/nix/flake.cc                                   | 2 +-
 src/nix/hash.cc                                    | 4 ++--
 src/nix/run.cc                                     | 4 ++--
 src/nix/verify.cc                                  | 2 +-
 20 files changed, 25 insertions(+), 24 deletions(-)

diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index f2b27af7c..250cd1413 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -858,7 +858,7 @@ std::vector<FlakeRef> RawInstallablesCommand::getFlakeRefsForCompletion()
     applyDefaultInstallables(rawInstallables);
     std::vector<FlakeRef> res;
     res.reserve(rawInstallables.size());
-    for (auto i : rawInstallables)
+    for (const auto & i : rawInstallables)
         res.push_back(parseFlakeRefWithFragment(
             fetchSettings,
             expandTilde(i),
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 5d2f75373..7e13e945c 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -4383,7 +4383,7 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 
         for (auto i = begin; i != end; ++i) {
             assert(idx <= 2 * len + 1 - 3);
-            auto match = *i;
+            const auto & match = *i;
 
             // Add a string for non-matched characters.
             list[idx++] = mkString(state, match.prefix());
diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc
index 02683b173..ede7d97ba 100644
--- a/src/libexpr/primops/context.cc
+++ b/src/libexpr/primops/context.cc
@@ -132,6 +132,8 @@ static void prim_addDrvOutputDependencies(EvalState & state, const PosIdx pos, V
             },
             [&](const NixStringContextElem::DrvDeep & c) -> NixStringContextElem::DrvDeep {
                 /* Reuse original item because we want this to be idempotent. */
+                /* FIXME: Suspicious move out of const. This is actually a copy, so the comment
+                 above does not make much sense. */
                 return std::move(c);
             },
         }, context.begin()->raw) }),
diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc
index fc5bb3145..04b8d0595 100644
--- a/src/libexpr/primops/fetchClosure.cc
+++ b/src/libexpr/primops/fetchClosure.cc
@@ -40,7 +40,7 @@ static void runFetchClosureWithRewrite(EvalState & state, const PosIdx pos, Stor
             });
     }
 
-    auto toPath = *toPathMaybe;
+    const auto & toPath = *toPathMaybe;
 
     // check and return
 
diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc
index dedcad2b1..f069c0d94 100644
--- a/src/libstore/build/drv-output-substitution-goal.cc
+++ b/src/libstore/build/drv-output-substitution-goal.cc
@@ -32,7 +32,7 @@ Goal::Co DrvOutputSubstitutionGoal::init()
 
     bool substituterFailed = false;
 
-    for (auto sub : subs) {
+    for (const auto & sub : subs) {
         trace("trying next substituter");
 
         /* The callback of the curl download below can outlive `this` (if
diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc
index 315500719..983c86601 100644
--- a/src/libstore/build/substitution-goal.cc
+++ b/src/libstore/build/substitution-goal.cc
@@ -57,7 +57,7 @@ Goal::Co PathSubstitutionGoal::init()
 
     bool substituterFailed = false;
 
-    for (auto sub : subs) {
+    for (const auto & sub : subs) {
         trace("trying next substituter");
 
         cleanup();
diff --git a/src/libstore/keys.cc b/src/libstore/keys.cc
index 70478e7ad..668725fc7 100644
--- a/src/libstore/keys.cc
+++ b/src/libstore/keys.cc
@@ -10,12 +10,12 @@ PublicKeys getDefaultPublicKeys()
 
     // FIXME: filter duplicates
 
-    for (auto s : settings.trustedPublicKeys.get()) {
+    for (const auto & s : settings.trustedPublicKeys.get()) {
         PublicKey key(s);
         publicKeys.emplace(key.name, key);
     }
 
-    for (auto secretKeyFile : settings.secretKeyFiles.get()) {
+    for (const auto & secretKeyFile : settings.secretKeyFiles.get()) {
         try {
             SecretKey secretKey(readFile(secretKeyFile));
             publicKeys.emplace(secretKey.name, secretKey.toPublicKey());
diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc
index b86beba2c..56ff6bef3 100644
--- a/src/libstore/local-overlay-store.cc
+++ b/src/libstore/local-overlay-store.cc
@@ -156,7 +156,7 @@ void LocalOverlayStore::queryGCReferrers(const StorePath & path, StorePathSet &
 StorePathSet LocalOverlayStore::queryValidDerivers(const StorePath & path)
 {
     auto res = LocalStore::queryValidDerivers(path);
-    for (auto p : lowerStore->queryValidDerivers(path))
+    for (const auto & p : lowerStore->queryValidDerivers(path))
         res.insert(p);
     return res;
 }
diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc
index 8b2557060..27fcc2864 100644
--- a/src/libstore/nar-info.cc
+++ b/src/libstore/nar-info.cc
@@ -118,7 +118,7 @@ std::string NarInfo::to_string(const Store & store) const
     if (deriver)
         res += "Deriver: " + std::string(deriver->to_string()) + "\n";
 
-    for (auto sig : sigs)
+    for (const auto & sig : sigs)
         res += "Sig: " + sig + "\n";
 
     if (ca)
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index bcbf0b55e..bf351a56d 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -454,7 +454,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
             debug("got %d keys, next marker '%s'",
                 contents.size(), res.GetNextMarker());
 
-            for (auto object : contents) {
+            for (const auto & object : contents) {
                 auto & key = object.GetKey();
                 if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue;
                 paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-" + MissingName));
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 10577fa2a..78cc3b917 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -1332,7 +1332,7 @@ ref<Store> openStore(StoreReference && storeURI)
                 return std::make_shared<LocalStore>(params);
         },
         [&](const StoreReference::Specified & g) {
-            for (auto implem : *Implementations::registered)
+            for (const auto & implem : *Implementations::registered)
                 if (implem.uriSchemes.count(g.scheme))
                     return implem.create(g.scheme, g.authority, params);
 
@@ -1363,7 +1363,7 @@ std::list<ref<Store>> getDefaultSubstituters()
             }
         };
 
-        for (auto uri : settings.substituters.get())
+        for (const auto & uri : settings.substituters.get())
             addStore(uri);
 
         stores.sort([](ref<Store> & a, ref<Store> & b) {
diff --git a/src/libutil-tests/hilite.cc b/src/libutil-tests/hilite.cc
index 1ff5980d5..5ef581888 100644
--- a/src/libutil-tests/hilite.cc
+++ b/src/libutil-tests/hilite.cc
@@ -52,8 +52,7 @@ namespace nix {
             std::regex("pt"),
         };
         std::vector<std::smatch> matches;
-        for(auto regex : regexes)
-        {
+        for (const auto & regex : regexes) {
             for(auto it = std::sregex_iterator(str.begin(), str.end(), regex); it != std::sregex_iterator(); ++it) {
                 matches.push_back(*it);
             }
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 385b6cd34..05ecf724e 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -348,7 +348,7 @@ void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang)
 
     /* Now that all the other args are processed, run the deferred completions.
      */
-    for (auto d : deferredCompletions)
+    for (const auto & d : deferredCompletions)
         d.completer(*completions, d.n, d.prefix);
 }
 
diff --git a/src/libutil/url.cc b/src/libutil/url.cc
index 9ed49dcbe..63b9734ee 100644
--- a/src/libutil/url.cc
+++ b/src/libutil/url.cc
@@ -77,7 +77,7 @@ std::map<std::string, std::string> decodeQuery(const std::string & query)
 {
     std::map<std::string, std::string> result;
 
-    for (auto s : tokenizeString<Strings>(query, "&")) {
+    for (const auto & s : tokenizeString<Strings>(query, "&")) {
         auto e = s.find('=');
         if (e == std::string::npos) {
             warn("dubious URI query '%s' is missing equal sign '%s', ignoring", s, "=");
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index c823c930e..b731b25af 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -222,7 +222,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs)
 {
     auto method = FileIngestionMethod::Flat;
 
-    for (auto i : opFlags)
+    for (const auto & i : opFlags)
         if (i == "--recursive") method = FileIngestionMethod::NixArchive;
         else throw UsageError("unknown flag '%1%'", i);
 
diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 9a95bc695..1736add9a 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -611,7 +611,7 @@ struct CmdDevelop : Common, MixEnvironment
         else if (!command.empty()) {
             std::vector<std::string> args;
             args.reserve(command.size());
-            for (auto s : command)
+            for (const auto & s : command)
                 args.push_back(shellEscape(s));
             script += fmt("exec %s\n", concatStringsSep(" ", args));
         }
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index 925b66d3e..cbc3cdb65 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -94,7 +94,7 @@ public:
             .label="inputs",
             .optional=true,
             .handler={[&](std::vector<std::string> inputsToUpdate){
-                for (auto inputToUpdate : inputsToUpdate) {
+                for (const auto & inputToUpdate : inputsToUpdate) {
                     InputPath inputPath;
                     try {
                         inputPath = flake::parseInputPath(inputToUpdate);
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index 62266fda1..2f9b3fe7c 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -79,7 +79,7 @@ struct CmdHashBase : Command
 
     void run() override
     {
-        for (auto path : paths) {
+        for (const auto & path : paths) {
             auto makeSink = [&]() -> std::unique_ptr<AbstractHashSink> {
                 if (modulus)
                     return std::make_unique<HashModuloSink>(hashAlgo, *modulus);
@@ -182,7 +182,7 @@ struct CmdToBase : Command
     void run() override
     {
         warn("The old format conversion sub commands of `nix hash` were deprecated in favor of `nix hash convert`.");
-        for (auto s : args)
+        for (const auto & s : args)
             logger->cout(Hash::parseAny(s, hashAlgo).to_string(hashFormat, hashFormat == HashFormat::SRI));
     }
 };
diff --git a/src/nix/run.cc b/src/nix/run.cc
index c9857e13e..a9f9ef60f 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -180,9 +180,9 @@ void chrootHelper(int argc, char * * argv)
         if (mount(realStoreDir.c_str(), (tmpDir + storeDir).c_str(), "", MS_BIND, 0) == -1)
             throw SysError("mounting '%s' on '%s'", realStoreDir, storeDir);
 
-        for (auto entry : fs::directory_iterator{"/"}) {
+        for (const auto & entry : fs::directory_iterator{"/"}) {
             checkInterrupt();
-            auto src = entry.path();
+            const auto & src = entry.path();
             fs::path dst = tmpDir / entry.path().filename();
             if (pathExists(dst)) continue;
             auto st = entry.symlink_status();
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index 124a05bed..52585fe08 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -129,7 +129,7 @@ struct CmdVerify : StorePathsCommand
                         size_t validSigs = 0;
 
                         auto doSigs = [&](StringSet sigs) {
-                            for (auto sig : sigs) {
+                            for (const auto & sig : sigs) {
                                 if (!sigsSeen.insert(sig).second) continue;
                                 if (validSigs < ValidPathInfo::maxSigs && info->checkSignature(*store, publicKeys, sig))
                                     validSigs++;

From f9980b5715fc403383f8e99d5da8bb91538c996d Mon Sep 17 00:00:00 2001
From: Sergei Zimmerman <145775305+xokdvium@users.noreply.github.com>
Date: Tue, 26 Nov 2024 00:13:54 +0300
Subject: [PATCH 64/73] fix(libutil/config): declare virtual dtor for
 AbstractConfig

This prevents any potential cases of deletion through base pointer and its
non-virtual dtor, which might leak memory. Also gets rid of the warning:

/nix/store/fg7ass3a5m5pgl26qzfdniicbwbgzccy-gcc-13.2.0/include/c++/13.2.0/bits/stl_construct.h:88:2: warning: destructor called on non-final 'nix::flake::Settings' that has virtual functions but non-virtual destructor [-Wdelete-non-abstract-non-virtual-dtor]
   88 |         __location->~_Tp();
....
../src/libflake-c/nix_api_flake.cc:10:30: note: in instantiation of function template specialization 'nix::make_ref<nix::flake::Settings>' requested here
   10 |         auto settings = nix::make_ref<nix::flake::Settings>();
---
 src/libutil/config.hh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index c0c59ac68..e98e09bf7 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -115,6 +115,8 @@ public:
      * Re-applies all previously attempted changes to unknown settings
      */
     void reapplyUnknownSettings();
+
+    virtual ~AbstractConfig() = default;
 };
 
 /**

From 5b8728d393dd1c9bbbf6737500669853da7de1b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Tue, 26 Nov 2024 07:07:31 +0100
Subject: [PATCH 65/73] more readable errors if symlinks cannot be created

Before:

filesystem error: cannot create symlink: Permission denied [/nix/store/1s2p3a4rs172336hj2l8n20nz74hf71j-nix-eval-jobs-2.24.1.drv] [/1s2p3a4rs172336hj2l8n20nz74hf71j-nix-eval-jobs-2.24.1.drv.tmp-2772352-1316231068]

Now:

creating symlink '/wfxz2q489c811n08cdqj7ywxm3n4z6m5-nix-eval-jobs-2.24.1.drv.tmp-2971297-324653080' -> '/nix/store/wfxz2q489c811n08cdqj7ywxm3n4z6m5-nix-eval-jobs-2.24.1.drv': Permission denied
---
 src/libutil/file-system.cc | 16 +++++++++++++---
 src/libutil/file-system.hh |  2 --
 src/nix/flake.cc           |  2 +-
 3 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc
index 92996ea47..829700336 100644
--- a/src/libutil/file-system.cc
+++ b/src/libutil/file-system.cc
@@ -602,7 +602,11 @@ std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix)
 
 void createSymlink(const Path & target, const Path & link)
 {
-    fs::create_symlink(target, link);
+    try {
+        fs::create_symlink(target, link);
+    } catch (fs::filesystem_error & e) {
+        throw SysError("creating symlink '%1%' -> '%2%'", link, target);
+    }
 }
 
 void replaceSymlink(const fs::path & target, const fs::path & link)
@@ -615,10 +619,16 @@ void replaceSymlink(const fs::path & target, const fs::path & link)
             fs::create_symlink(target, tmp);
         } catch (fs::filesystem_error & e) {
             if (e.code() == std::errc::file_exists) continue;
-            throw;
+            throw SysError("creating symlink '%1%' -> '%2%'", tmp, target);
+        }
+
+        try {
+            fs::rename(tmp, link);
+        } catch (fs::filesystem_error & e) {
+            if (e.code() == std::errc::file_exists) continue;
+            throw SysError("renaming '%1%' to '%2%'", tmp, link);
         }
 
-        fs::rename(tmp, link);
 
         break;
     }
diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh
index 4c08cdf58..3c49181a0 100644
--- a/src/libutil/file-system.hh
+++ b/src/libutil/file-system.hh
@@ -250,8 +250,6 @@ void setWriteTime(const std::filesystem::path & path, const struct stat & st);
 /**
  * Create a symlink.
  *
- * In the process of being deprecated for
- * `std::filesystem::create_symlink`.
  */
 void createSymlink(const Path & target, const Path & link);
 
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index 925b66d3e..4bb5c329e 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -937,7 +937,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand
                         }
                         continue;
                     } else
-                        fs::create_symlink(target, to2);
+                        createSymlink(target, to2);
                 }
                 else
                     throw Error("file '%s' has unsupported type", from2);

From d67aa03414ad6e75b2ac2145406fcc936c0f1798 Mon Sep 17 00:00:00 2001
From: Sergei Trofimovich <slyich@gmail.com>
Date: Tue, 26 Nov 2024 18:35:18 +0000
Subject: [PATCH 66/73] src/perl/meson.build: fall back to 'bz2' library lookup

Upstream `bzip2` does not provide `pkg-config` files. As a result an
attempt to build `nix` on some distributions like Gentoo failos the
configure as:

    $ meson setup ..
    ...
    Executing subproject perl
    ...
    perl| Run-time dependency bzip2 found: NO (tried pkgconfig and cmake)
    ../src/perl/meson.build:68:12: ERROR: Dependency "bzip2" not found, tried pkgconfig and cmake

The change falls back to `bz2` library for such cases.
---
 src/perl/meson.build | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/perl/meson.build b/src/perl/meson.build
index dcb6a68a4..52d85fd60 100644
--- a/src/perl/meson.build
+++ b/src/perl/meson.build
@@ -65,7 +65,13 @@ yath = find_program('yath', required : false)
 
 # Required Libraries
 #-------------------------------------------------
-bzip2_dep = dependency('bzip2')
+bzip2_dep = dependency('bzip2', required: false)
+if not bzip2_dep.found()
+  bzip2_dep = cpp.find_library('bz2')
+  if not bzip2_dep.found()
+    error('No "bzip2" pkg-config or "bz2" library found')
+  endif
+endif
 curl_dep = dependency('libcurl')
 libsodium_dep = dependency('libsodium')
 

From 2679e55232af74b0325877b6a49ed83502711fc0 Mon Sep 17 00:00:00 2001
From: Sergei Trofimovich <slyich@gmail.com>
Date: Tue, 26 Nov 2024 23:08:10 +0000
Subject: [PATCH 67/73] tests/functional/meson.build: always look up `ls` as a
 `coreutils` proxy

Without the change `meson setup` fails on `Gentoo or Debian as those
don't use multicall binary:

    $ meson setup ..
    ...
    Executing subproject nix-functional-tests
    ...
    ../src/nix-functional-tests/meson.build:24:14: ERROR: Program 'coreutils' not found or not executable

The change always uses `ls` to look `coreutils` up.

Closes: https://github.com/NixOS/nix/issues/11975
---
 tests/functional/meson.build | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/tests/functional/meson.build b/tests/functional/meson.build
index 0d46f9ce2..933595cd5 100644
--- a/tests/functional/meson.build
+++ b/tests/functional/meson.build
@@ -17,12 +17,10 @@ fs = import('fs')
 nix = find_program('nix')
 bash = find_program('bash', native : true)
 busybox = find_program('busybox', native : true, required : false)
-if host_machine.system() == 'windows'
-  # Because of the state of symlinks on Windows, coreutils.exe doesn't usually exist, but things like ls.exe will
-  coreutils = find_program('ls', native : true)
-else
-  coreutils = find_program('coreutils', native : true)
-endif
+# Look up `coreutils` package by searching for `ls` binary.
+# Previously we looked up `coreutils` on `linux`, but that is not
+# guaranteed to exist either.
+coreutils = find_program('ls', native : true)
 dot = find_program('dot', native : true, required : false)
 
 nix_bin_dir = fs.parent(nix.full_path())

From 21ddd2022e40d4727684cadf7aca44b0b4ec622c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Wed, 27 Nov 2024 07:39:30 +0100
Subject: [PATCH 68/73] mergify: drop installer test

---
 .mergify.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.mergify.yml b/.mergify.yml
index c297d3d5e..ac1bee111 100644
--- a/.mergify.yml
+++ b/.mergify.yml
@@ -2,9 +2,6 @@ queue_rules:
   - name: default
     # all required tests need to go here
     merge_conditions:
-      - check-success=installer
-      - check-success=installer_test (macos-latest)
-      - check-success=installer_test (ubuntu-latest)
       - check-success=tests (macos-latest)
       - check-success=tests (ubuntu-latest)
       - check-success=vm_tests

From a5c7709f97ceb567ffd1903aa1cd921ca70d7c7b Mon Sep 17 00:00:00 2001
From: h0nIg <h0nIg@users.noreply.github.com>
Date: Wed, 27 Nov 2024 13:24:46 +0100
Subject: [PATCH 69/73] docker: Fix command "nix profile install", Don't
 require --impure

---
 docker.nix | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docker.nix b/docker.nix
index 19479e57c..e2e9da728 100644
--- a/docker.nix
+++ b/docker.nix
@@ -258,14 +258,14 @@ let
       mkdir -p $out/nix/var/nix/profiles/per-user/${uname}
 
       ln -s ${profile} $out/nix/var/nix/profiles/default-1-link
-      ln -s $out/nix/var/nix/profiles/default-1-link $out/nix/var/nix/profiles/default
+      ln -s /nix/var/nix/profiles/default-1-link $out/nix/var/nix/profiles/default
       ln -s /nix/var/nix/profiles/default $out${userHome}/.nix-profile
 
       ln -s ${channel} $out/nix/var/nix/profiles/per-user/${uname}/channels-1-link
-      ln -s $out/nix/var/nix/profiles/per-user/${uname}/channels-1-link $out/nix/var/nix/profiles/per-user/${uname}/channels
+      ln -s /nix/var/nix/profiles/per-user/${uname}/channels-1-link $out/nix/var/nix/profiles/per-user/${uname}/channels
 
       mkdir -p $out${userHome}/.nix-defexpr
-      ln -s $out/nix/var/nix/profiles/per-user/${uname}/channels $out${userHome}/.nix-defexpr/channels
+      ln -s /nix/var/nix/profiles/per-user/${uname}/channels $out${userHome}/.nix-defexpr/channels
       echo "${channelURL} ${channelName}" > $out${userHome}/.nix-channels
 
       mkdir -p $out/bin $out/usr/bin

From 37fd80588fff0ee5e9e27c9d6a1dbc7d2f740b6d Mon Sep 17 00:00:00 2001
From: Anatoli Babenia <anatoli@rainforce.org>
Date: Wed, 27 Nov 2024 17:11:36 +0300
Subject: [PATCH 70/73] shellcheck: simplify install-nix-from-tarball.sh

---
 maintainers/flake-module.nix        |  1 -
 scripts/install-nix-from-tarball.sh | 15 +++++----------
 2 files changed, 5 insertions(+), 11 deletions(-)

diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix
index ba6cd2816..1d4e85c8c 100644
--- a/maintainers/flake-module.nix
+++ b/maintainers/flake-module.nix
@@ -496,7 +496,6 @@
             ''^scripts/create-darwin-volume\.sh$''
             ''^scripts/install-darwin-multi-user\.sh$''
             ''^scripts/install-multi-user\.sh$''
-            ''^scripts/install-nix-from-tarball\.sh$''
             ''^scripts/install-systemd-multi-user\.sh$''
             ''^src/nix/get-env\.sh$''
             ''^tests/functional/ca/build-dry\.sh$''
diff --git a/scripts/install-nix-from-tarball.sh b/scripts/install-nix-from-tarball.sh
index 794622530..007fe85ee 100644
--- a/scripts/install-nix-from-tarball.sh
+++ b/scripts/install-nix-from-tarball.sh
@@ -48,15 +48,14 @@ case "$(uname -s)" in
         INSTALL_MODE=no-daemon;;
 esac
 
-# space-separated string
-ACTIONS=
+ACTION=
 
 # handle the command line flags
 while [ $# -gt 0 ]; do
     case $1 in
         --daemon)
             INSTALL_MODE=daemon
-            ACTIONS="${ACTIONS}install "
+            ACTION=install
             ;;
         --no-daemon)
             if [ "$(uname -s)" = "Darwin" ]; then
@@ -65,18 +64,14 @@ while [ $# -gt 0 ]; do
             fi
             INSTALL_MODE=no-daemon
             # intentional tail space
-            ACTIONS="${ACTIONS}install "
+            ACTION=install
             ;;
-        # --uninstall)
-        #     # intentional tail space
-        #     ACTIONS="${ACTIONS}uninstall "
-        #     ;;
         --yes)
             export NIX_INSTALLER_YES=1;;
         --no-channel-add)
             export NIX_INSTALLER_NO_CHANNEL_ADD=1;;
         --daemon-user-count)
-            export NIX_USER_COUNT=$2
+            export NIX_USER_COUNT="$2"
             shift;;
         --no-modify-profile)
             NIX_INSTALLER_NO_MODIFY_PROFILE=1;;
@@ -128,7 +123,7 @@ done
 
 if [ "$INSTALL_MODE" = "daemon" ]; then
     printf '\e[1;31mSwitching to the Multi-user Installer\e[0m\n'
-    exec "$self/install-multi-user" $ACTIONS # let ACTIONS split
+    exec "$self/install-multi-user" $ACTION
     exit 0
 fi
 

From 8034589d7eb299c126f169ff0780b5242936acdd Mon Sep 17 00:00:00 2001
From: Ryan Hendrickson <ryan.hendrickson@alum.mit.edu>
Date: Wed, 14 Aug 2024 00:05:06 -0400
Subject: [PATCH 71/73] parser-state: fix attribute merging

---
 src/libexpr-tests/trivial.cc                  |  51 +++++++++
 src/libexpr/parser-state.hh                   | 108 +++++++++++-------
 ...fail-attrset-merge-drops-later-rec.err.exp |   5 +
 ...val-fail-attrset-merge-drops-later-rec.nix |   1 +
 ...val-okay-regrettable-rec-attrset-merge.exp |   1 +
 ...val-okay-regrettable-rec-attrset-merge.nix |   3 +
 .../parse-fail-mixed-nested-attrs1.err.exp    |   8 +-
 .../parse-fail-mixed-nested-attrs2.err.exp    |   8 +-
 8 files changed, 133 insertions(+), 52 deletions(-)
 create mode 100644 tests/functional/lang/eval-fail-attrset-merge-drops-later-rec.err.exp
 create mode 100644 tests/functional/lang/eval-fail-attrset-merge-drops-later-rec.nix
 create mode 100644 tests/functional/lang/eval-okay-regrettable-rec-attrset-merge.exp
 create mode 100644 tests/functional/lang/eval-okay-regrettable-rec-attrset-merge.nix

diff --git a/src/libexpr-tests/trivial.cc b/src/libexpr-tests/trivial.cc
index e455a571b..d77b4d53b 100644
--- a/src/libexpr-tests/trivial.cc
+++ b/src/libexpr-tests/trivial.cc
@@ -177,6 +177,57 @@ namespace nix {
             )
     );
 
+// The following macros ultimately define 48 tests (16 variations on three
+// templates). Each template tests an expression that can be written in 2^4
+// different ways, by making four choices about whether to write a particular
+// attribute path segment as `x.y = ...;` (collapsed) or `x = { y = ...; };`
+// (expanded).
+//
+// The nestedAttrsetMergeXXXX tests check that the expression
+// `{ a.b.c = 1; a.b.d = 2; }` has the same value regardless of how it is
+// expanded. (That exact expression is exercised in test
+// nestedAttrsetMerge0000, because it is fully collapsed. The test
+// nestedAttrsetMerge1001 would instead examine
+// `{ a = { b.c = 1; }; a.b = { d = 2; }; }`.)
+//
+// The nestedAttrsetMergeDupXXXX tests check that the expression
+// `{ a.b.c = 1; a.b.c = 2; }` throws a duplicate attribute error, again
+// regardless of how it is expanded.
+//
+// The nestedAttrsetMergeLetXXXX tests check that the expression
+// `let a.b.c = 1; a.b.d = 2; in a` has the same value regardless of how it is
+// expanded.
+#define X_EXPAND_IF0(k, v) k "." v
+#define X_EXPAND_IF1(k, v) k " = { " v " };"
+#define X4(w, x, y, z) \
+    TEST_F(TrivialExpressionTest, nestedAttrsetMerge##w##x##y##z) { \
+        auto v = eval("{ a.b = { c = 1; d = 2; }; } == { " \
+                X_EXPAND_IF##w("a", X_EXPAND_IF##x("b", "c = 1;")) " " \
+                X_EXPAND_IF##y("a", X_EXPAND_IF##z("b", "d = 2;")) " }"); \
+        ASSERT_THAT(v, IsTrue()); \
+    }; \
+    TEST_F(TrivialExpressionTest, nestedAttrsetMergeDup##w##x##y##z) { \
+        ASSERT_THROW(eval("{ " \
+                X_EXPAND_IF##w("a", X_EXPAND_IF##x("b", "c = 1;")) " " \
+                X_EXPAND_IF##y("a", X_EXPAND_IF##z("b", "c = 2;")) " }"), Error); \
+    }; \
+    TEST_F(TrivialExpressionTest, nestedAttrsetMergeLet##w##x##y##z) { \
+        auto v = eval("{ b = { c = 1; d = 2; }; } == (let " \
+                X_EXPAND_IF##w("a", X_EXPAND_IF##x("b", "c = 1;")) " " \
+                X_EXPAND_IF##y("a", X_EXPAND_IF##z("b", "d = 2;")) " in a)"); \
+        ASSERT_THAT(v, IsTrue()); \
+    };
+#define X3(...) X4(__VA_ARGS__, 0) X4(__VA_ARGS__, 1)
+#define X2(...) X3(__VA_ARGS__, 0) X3(__VA_ARGS__, 1)
+#define X1(...) X2(__VA_ARGS__, 0) X2(__VA_ARGS__, 1)
+    X1(0) X1(1)
+#undef X_EXPAND_IF0
+#undef X_EXPAND_IF1
+#undef X1
+#undef X2
+#undef X3
+#undef X4
+
     TEST_F(TrivialExpressionTest, functor) {
         auto v = eval("{ __functor = self: arg: self.v + arg; v = 10; } 5");
         ASSERT_THAT(v, IsIntEq(15));
diff --git a/src/libexpr/parser-state.hh b/src/libexpr/parser-state.hh
index 8ad0d9ad7..21a880e8e 100644
--- a/src/libexpr/parser-state.hh
+++ b/src/libexpr/parser-state.hh
@@ -88,6 +88,7 @@ struct ParserState
     void dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos);
     void dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos);
     void addAttr(ExprAttrs * attrs, AttrPath && attrPath, const ParserLocation & loc, Expr * e, const ParserLocation & exprLoc);
+    void addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symbol, ExprAttrs::AttrDef && def);
     Formals * validateFormals(Formals * formals, PosIdx pos = noPos, Symbol arg = {});
     Expr * stripIndentation(const PosIdx pos,
         std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>> && es);
@@ -120,64 +121,29 @@ inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, const
     // Checking attrPath validity.
     // ===========================
     for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) {
+        ExprAttrs * nested;
         if (i->symbol) {
             ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol);
             if (j != attrs->attrs.end()) {
-                if (j->second.kind != ExprAttrs::AttrDef::Kind::Inherited) {
-                    ExprAttrs * attrs2 = dynamic_cast<ExprAttrs *>(j->second.e);
-                    if (!attrs2) dupAttr(attrPath, pos, j->second.pos);
-                    attrs = attrs2;
-                } else
+                nested = dynamic_cast<ExprAttrs *>(j->second.e);
+                if (!nested) {
+                    attrPath.erase(i + 1, attrPath.end());
                     dupAttr(attrPath, pos, j->second.pos);
+                }
             } else {
-                ExprAttrs * nested = new ExprAttrs;
+                nested = new ExprAttrs;
                 attrs->attrs[i->symbol] = ExprAttrs::AttrDef(nested, pos);
-                attrs = nested;
             }
         } else {
-            ExprAttrs *nested = new ExprAttrs;
+            nested = new ExprAttrs;
             attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, nested, pos));
-            attrs = nested;
         }
+        attrs = nested;
     }
     // Expr insertion.
     // ==========================
     if (i->symbol) {
-        ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol);
-        if (j != attrs->attrs.end()) {
-            // This attr path is already defined. However, if both
-            // e and the expr pointed by the attr path are two attribute sets,
-            // we want to merge them.
-            // Otherwise, throw an error.
-            auto ae = dynamic_cast<ExprAttrs *>(e);
-            auto jAttrs = dynamic_cast<ExprAttrs *>(j->second.e);
-            if (jAttrs && ae) {
-                if (ae->inheritFromExprs && !jAttrs->inheritFromExprs)
-                    jAttrs->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
-                for (auto & ad : ae->attrs) {
-                    auto j2 = jAttrs->attrs.find(ad.first);
-                    if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error.
-                        dupAttr(ad.first, j2->second.pos, ad.second.pos);
-                    jAttrs->attrs.emplace(ad.first, ad.second);
-                    if (ad.second.kind == ExprAttrs::AttrDef::Kind::InheritedFrom) {
-                        auto & sel = dynamic_cast<ExprSelect &>(*ad.second.e);
-                        auto & from = dynamic_cast<ExprInheritFrom &>(*sel.e);
-                        from.displ += jAttrs->inheritFromExprs->size();
-                    }
-                }
-                jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(), ae->dynamicAttrs.begin(), ae->dynamicAttrs.end());
-                if (ae->inheritFromExprs) {
-                    jAttrs->inheritFromExprs->insert(jAttrs->inheritFromExprs->end(),
-                        ae->inheritFromExprs->begin(), ae->inheritFromExprs->end());
-                }
-            } else {
-                dupAttr(attrPath, pos, j->second.pos);
-            }
-        } else {
-            // This attr path is not defined. Let's create it.
-            attrs->attrs.emplace(i->symbol, ExprAttrs::AttrDef(e, pos));
-            e->setName(i->symbol);
-        }
+        addAttr(attrs, attrPath, i->symbol, ExprAttrs::AttrDef(e, pos));
     } else {
         attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos));
     }
@@ -189,6 +155,60 @@ inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, const
     }
 }
 
+/**
+ * Precondition: attrPath is used for error messages and should already contain
+ * symbol as its last element.
+ */
+inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symbol, ExprAttrs::AttrDef && def)
+{
+    ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(symbol);
+    if (j != attrs->attrs.end()) {
+        // This attr path is already defined. However, if both
+        // e and the expr pointed by the attr path are two attribute sets,
+        // we want to merge them.
+        // Otherwise, throw an error.
+        auto ae = dynamic_cast<ExprAttrs *>(def.e);
+        auto jAttrs = dynamic_cast<ExprAttrs *>(j->second.e);
+
+        // N.B. In a world in which we are less bound by our past mistakes, we
+        // would also test that jAttrs and ae are not recursive. The effect of
+        // not doing so is that any `rec` marker on ae is discarded, and any
+        // `rec` marker on jAttrs will apply to the attributes in ae.
+        // See https://github.com/NixOS/nix/issues/9020.
+        if (jAttrs && ae) {
+            if (ae->inheritFromExprs && !jAttrs->inheritFromExprs)
+                jAttrs->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
+            for (auto & ad : ae->attrs) {
+                if (ad.second.kind == ExprAttrs::AttrDef::Kind::InheritedFrom) {
+                    auto & sel = dynamic_cast<ExprSelect &>(*ad.second.e);
+                    auto & from = dynamic_cast<ExprInheritFrom &>(*sel.e);
+                    from.displ += jAttrs->inheritFromExprs->size();
+                }
+                attrPath.emplace_back(AttrName(ad.first));
+                addAttr(jAttrs, attrPath, ad.first, std::move(ad.second));
+                attrPath.pop_back();
+            }
+            ae->attrs.clear();
+            jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(),
+                std::make_move_iterator(ae->dynamicAttrs.begin()),
+                std::make_move_iterator(ae->dynamicAttrs.end()));
+            ae->dynamicAttrs.clear();
+            if (ae->inheritFromExprs) {
+                jAttrs->inheritFromExprs->insert(jAttrs->inheritFromExprs->end(),
+                    std::make_move_iterator(ae->inheritFromExprs->begin()),
+                    std::make_move_iterator(ae->inheritFromExprs->end()));
+                ae->inheritFromExprs = nullptr;
+            }
+        } else {
+            dupAttr(attrPath, def.pos, j->second.pos);
+        }
+    } else {
+        // This attr path is not defined. Let's create it.
+        attrs->attrs.emplace(symbol, def);
+        def.e->setName(symbol);
+    }
+}
+
 inline Formals * ParserState::validateFormals(Formals * formals, PosIdx pos, Symbol arg)
 {
     std::sort(formals->formals.begin(), formals->formals.end(),
diff --git a/tests/functional/lang/eval-fail-attrset-merge-drops-later-rec.err.exp b/tests/functional/lang/eval-fail-attrset-merge-drops-later-rec.err.exp
new file mode 100644
index 000000000..d1cdc7b76
--- /dev/null
+++ b/tests/functional/lang/eval-fail-attrset-merge-drops-later-rec.err.exp
@@ -0,0 +1,5 @@
+error: undefined variable 'd'
+       at /pwd/lang/eval-fail-attrset-merge-drops-later-rec.nix:1:26:
+            1| { a.b = 1; a = rec { c = d + 2; d = 3; }; }.c
+             |                          ^
+            2|
diff --git a/tests/functional/lang/eval-fail-attrset-merge-drops-later-rec.nix b/tests/functional/lang/eval-fail-attrset-merge-drops-later-rec.nix
new file mode 100644
index 000000000..fdb314b91
--- /dev/null
+++ b/tests/functional/lang/eval-fail-attrset-merge-drops-later-rec.nix
@@ -0,0 +1 @@
+{ a.b = 1; a = rec { c = d + 2; d = 3; }; }.c
diff --git a/tests/functional/lang/eval-okay-regrettable-rec-attrset-merge.exp b/tests/functional/lang/eval-okay-regrettable-rec-attrset-merge.exp
new file mode 100644
index 000000000..1e8b31496
--- /dev/null
+++ b/tests/functional/lang/eval-okay-regrettable-rec-attrset-merge.exp
@@ -0,0 +1 @@
+6
diff --git a/tests/functional/lang/eval-okay-regrettable-rec-attrset-merge.nix b/tests/functional/lang/eval-okay-regrettable-rec-attrset-merge.nix
new file mode 100644
index 000000000..8df6a2ad8
--- /dev/null
+++ b/tests/functional/lang/eval-okay-regrettable-rec-attrset-merge.nix
@@ -0,0 +1,3 @@
+# This is for backwards compatibility, not because we like it.
+# See https://github.com/NixOS/nix/issues/9020.
+{ a = rec { b = c + 1; d = 2; }; a.c = d + 3; }.a.b
diff --git a/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp b/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp
index a4472156b..49a07323f 100644
--- a/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp
+++ b/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp
@@ -1,6 +1,6 @@
-error: attribute 'z' already defined at «stdin»:3:16
-       at «stdin»:2:3:
-            1| {
+error: attribute 'x.z' already defined at «stdin»:2:3
+       at «stdin»:3:16:
             2|   x.z = 3;
-             |   ^
             3|   x = { y = 3; z = 3; };
+             |                ^
+            4| }
diff --git a/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp b/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp
index ead1f0dbd..36fab2fe6 100644
--- a/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp
+++ b/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp
@@ -1,6 +1,6 @@
-error: attribute 'y' already defined at «stdin»:3:9
-       at «stdin»:2:3:
-            1| {
+error: attribute 'x.y.y' already defined at «stdin»:2:3
+       at «stdin»:3:9:
             2|   x.y.y = 3;
-             |   ^
             3|   x = { y.y= 3; z = 3; };
+             |         ^
+            4| }

From e5e09006f97700f35c68411df6fe4f8a9d7dc807 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 28 Nov 2024 15:25:51 +0100
Subject: [PATCH 72/73] Work around gcc warning

Same as 57fea81f8a6ab3b91b1003d27bd48effad30b25a.
---
 src/nix-env/nix-env.cc | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index ba2baccee..e9eb52708 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -481,12 +481,13 @@ static void printMissing(EvalState & state, PackageInfos & elems)
 {
     std::vector<DerivedPath> targets;
     for (auto & i : elems)
-        if (auto drvPath = i.queryDrvPath())
-            targets.emplace_back(DerivedPath::Built{
+        if (auto drvPath = i.queryDrvPath()) {
+            auto path = DerivedPath::Built{
                 .drvPath = makeConstantStorePathRef(*drvPath),
                 .outputs = OutputsSpec::All { },
-            });
-        else
+            };
+            targets.emplace_back(std::move(path));
+        } else
             targets.emplace_back(DerivedPath::Opaque{
                 .path = i.queryOutPath(),
             });

From 747cf4e50f43b510ff054ec14bdef87634231237 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Wed, 6 Nov 2024 22:49:04 +0100
Subject: [PATCH 73/73] fix: Add splicing to fix the manual in cross

We *could* use a "native" manual instead - ie reusing a native
`nixpkgsFor.${buildPlatform}`, but this works, and also
works for possible cases where we have a custom or patched build tool.
---
 flake.nix | 36 +++++++++++++++++++++++++++---------
 1 file changed, 27 insertions(+), 9 deletions(-)

diff --git a/flake.nix b/flake.nix
index 06025e3b7..794736af4 100644
--- a/flake.nix
+++ b/flake.nix
@@ -124,18 +124,36 @@
           # without "polluting" the top level "`pkgs`" attrset.
           # This also has the benefit of providing us with a distinct set of packages
           # we can iterate over.
-          nixComponents = lib.makeScope final.nixDependencies.newScope (import ./packaging/components.nix {
-            inherit (final) lib;
-            inherit officialRelease;
-            src = self;
-          });
+          nixComponents =
+            lib.makeScopeWithSplicing'
+              {
+                inherit (final) splicePackages;
+                inherit (final.nixDependencies) newScope;
+              }
+              {
+                otherSplices = final.generateSplicesForMkScope "nixComponents";
+                f = import ./packaging/components.nix {
+                  inherit (final) lib;
+                  inherit officialRelease;
+                  src = self;
+                };
+              };
 
           # The dependencies are in their own scope, so that they don't have to be
           # in Nixpkgs top level `pkgs` or `nixComponents`.
-          nixDependencies = lib.makeScope final.newScope (import ./packaging/dependencies.nix {
-            inherit inputs stdenv;
-            pkgs = final;
-          });
+          nixDependencies =
+            lib.makeScopeWithSplicing'
+              {
+                inherit (final) splicePackages;
+                inherit (final) newScope; # layered directly on pkgs, unlike nixComponents above
+              }
+              {
+                otherSplices = final.generateSplicesForMkScope "nixDependencies";
+                f = import ./packaging/dependencies.nix {
+                  inherit inputs stdenv;
+                  pkgs = final;
+                };
+              };
 
           nix = final.nixComponents.nix-cli;