From 82ddb130984c7bdc45cdffc14e81bed720089200 Mon Sep 17 00:00:00 2001
From: Graham Bennett <bennettg@pdtpartners.com>
Date: Fri, 29 Apr 2022 15:15:25 -0400
Subject: [PATCH 001/294] Unlock output paths when a derivation is already
 built

Without this change, nix build processes will not drop the locks for derivation goals
which have already been built by another process when the current process gets
round to building them. This means the locks are held until the process
terminates.

If there are other nix build processes in a similar state, they will also try to
acquire the same locks when they try to build the same derivation, and so will
wait until the lock holder terminates (which might be a very long time if it has
a lot to build). In some pathological cases, those processes might be holding
their own locks on other derivations due to the same issue, and this can lead to
deadlock.

Resolves #6468
---
 src/libstore/build/derivation-goal.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 6472ecd99..befbfd10e 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -1456,6 +1456,7 @@ void DerivationGoal::done(
     SingleDrvOutputs builtOutputs,
     std::optional<Error> ex)
 {
+    outputLocks.unlock();
     buildResult.status = status;
     if (ex)
         buildResult.errorMsg = fmt("%s", normaltxt(ex->info().msg));

From a2f0ba6a6dbb79efbb83ebe92b287f79b3f3af91 Mon Sep 17 00:00:00 2001
From: Vladimir Kryachko <vkryachko@google.com>
Date: Mon, 16 Oct 2023 16:51:49 -0400
Subject: [PATCH 002/294] Fix transitive input locking.

Fixes reproducibility issue described in #9143

Fixes #9143
---
 src/libexpr/flake/flake.cc | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index a6212c12f..5d9d60655 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -351,10 +351,13 @@ LockedFlake lockFlake(
         debug("old lock file: %s", oldLockFile);
 
         std::map<InputPath, FlakeInput> overrides;
+        std::set<InputPath> explicitCliOverrides;
         std::set<InputPath> overridesUsed, updatesUsed;
 
-        for (auto & i : lockFlags.inputOverrides)
+        for (auto & i : lockFlags.inputOverrides) {
             overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second });
+            explicitCliOverrides.insert(i.first);
+        }
 
         LockFile newLockFile;
 
@@ -425,6 +428,7 @@ LockedFlake lockFlake(
                        ancestors? */
                     auto i = overrides.find(inputPath);
                     bool hasOverride = i != overrides.end();
+                    bool hasCliOverride = explicitCliOverrides.find(inputPath) != explicitCliOverrides.end();
                     if (hasOverride) {
                         overridesUsed.insert(inputPath);
                         // Respect the “flakeness” of the input even if we
@@ -460,7 +464,7 @@ LockedFlake lockFlake(
 
                     if (oldLock
                         && oldLock->originalRef == *input.ref
-                        && !hasOverride)
+                        && !hasCliOverride)
                     {
                         debug("keeping existing input '%s'", inputPathS);
 
@@ -541,7 +545,7 @@ LockedFlake lockFlake(
                             nuked the next time we update the lock
                             file. That is, overrides are sticky unless you
                             use --no-write-lock-file. */
-                        auto ref = input2.ref ? *input2.ref : *input.ref;
+                        auto ref = (input2.ref && explicitCliOverrides.contains(inputPath)) ? *input2.ref : *input.ref;
 
                         if (input.isFlake) {
                             Path localPath = parentPath;

From 311e2ad024441950cb1300e56c9745259deebdda Mon Sep 17 00:00:00 2001
From: Vladimir Kryachko <vkryachko@google.com>
Date: Wed, 18 Oct 2023 10:37:06 -0400
Subject: [PATCH 003/294] Address review comments

---
 src/libexpr/flake/flake.cc              |  2 +-
 tests/functional/flakes/follow-paths.sh | 76 +++++++++++++++++++++++++
 2 files changed, 77 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index 5d9d60655..2c7e12ec9 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -428,7 +428,7 @@ LockedFlake lockFlake(
                        ancestors? */
                     auto i = overrides.find(inputPath);
                     bool hasOverride = i != overrides.end();
-                    bool hasCliOverride = explicitCliOverrides.find(inputPath) != explicitCliOverrides.end();
+                    bool hasCliOverride = explicitCliOverrides.contains(inputPath);
                     if (hasOverride) {
                         overridesUsed.insert(inputPath);
                         // Respect the “flakeness” of the input even if we
diff --git a/tests/functional/flakes/follow-paths.sh b/tests/functional/flakes/follow-paths.sh
index 8573b5511..7f4e8bf5d 100644
--- a/tests/functional/flakes/follow-paths.sh
+++ b/tests/functional/flakes/follow-paths.sh
@@ -260,3 +260,79 @@ EOF
 
 checkRes=$(nix flake lock "$flakeFollowCycle" 2>&1 && fail "nix flake lock should have failed." || true)
 echo $checkRes | grep -F "error: follow cycle detected: [baz -> foo -> bar -> baz]"
+
+
+# Test transitive input url locking
+# This tests the following lockfile issue: https://github.com/NixOS/nix/issues/9143
+#
+# We construct the following graph, where p->q means p has input q.
+#
+# A -> B -> C
+#
+# And override B/C to flake D, first in A's flake.nix and then with --override-input.
+#
+# A -> B -> D
+flakeFollowsCustomUrlA="$TEST_ROOT/follows/custom-url/flakeA"
+flakeFollowsCustomUrlB="$TEST_ROOT/follows/custom-url/flakeA/flakeB"
+flakeFollowsCustomUrlC="$TEST_ROOT/follows/custom-url/flakeA/flakeB/flakeC"
+flakeFollowsCustomUrlD="$TEST_ROOT/follows/custom-url/flakeA/flakeB/flakeD"
+
+
+createGitRepo "$flakeFollowsCustomUrlA"
+mkdir -p "$flakeFollowsCustomUrlB"
+mkdir -p "$flakeFollowsCustomUrlC"
+mkdir -p "$flakeFollowsCustomUrlD"
+
+cat > "$flakeFollowsCustomUrlD/flake.nix" <<EOF
+{
+    description = "Flake D";
+    inputs = {};
+    outputs = { ... }: {};
+}
+EOF
+
+cat > "$flakeFollowsCustomUrlC/flake.nix" <<EOF
+{
+    description = "Flake C";
+    inputs = {};
+    outputs = { ... }: {};
+}
+EOF
+
+cat > "$flakeFollowsCustomUrlB/flake.nix" <<EOF
+{
+    description = "Flake B";
+    inputs = {
+        C = {
+            url = "path:./flakeC";
+        };
+    };
+    outputs = { ... }: {};
+}
+EOF
+
+cat > "$flakeFollowsCustomUrlA/flake.nix" <<EOF
+{
+    description = "Flake A";
+    inputs = {
+        B = {
+            url = "path:./flakeB";
+            inputs.C.url = "path:./flakeB/flakeD";
+        };
+    };
+    outputs = { ... }: {};
+}
+EOF
+
+git -C "$flakeFollowsCustomUrlA" add flake.nix flakeB/flake.nix \
+  flakeB/flakeC/flake.nix flakeB/flakeD/flake.nix
+
+# lock "original" entry should contain overridden url
+json=$(nix flake metadata "$flakeFollowsCustomUrlA" --json)
+[[ $(echo "$json" | jq -r .locks.nodes.C.original.path) = './flakeB/flakeD' ]]
+rm "$flakeFollowsCustomUrlA"/flake.lock
+
+# if override-input is specified, lock "original" entry should contain original url
+json=$(nix flake metadata "$flakeFollowsCustomUrlA" --override-input B/C "path:./flakeB/flakeD" --json)
+echo "$json" | jq .locks.nodes.C.original
+[[ $(echo "$json" | jq -r .locks.nodes.C.original.path) = './flakeC' ]]

From 7f71fc7540502295202b075f82e89fcf993e7e3a Mon Sep 17 00:00:00 2001
From: Pol Dellaiera <pol.dellaiera@protonmail.com>
Date: Mon, 23 Oct 2023 23:46:06 +0200
Subject: [PATCH 004/294] fix: make sure `tar` reproducibility flags are set

---
 tests/functional/tarball.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/functional/tarball.sh b/tests/functional/tarball.sh
index 6e621a28c..e59ee400e 100644
--- a/tests/functional/tarball.sh
+++ b/tests/functional/tarball.sh
@@ -18,7 +18,7 @@ test_tarball() {
     local compressor="$2"
 
     tarball=$TEST_ROOT/tarball.tar$ext
-    (cd $TEST_ROOT && tar cf - tarball) | $compressor > $tarball
+    (cd $TEST_ROOT && GNUTAR_REPRODUCIBLE= tar --mtime=$tarroot/default.nix --owner=0 --group=0 --numeric-owner --sort=name -c -f - tarball) | $compressor > $tarball
 
     nix-env -f file://$tarball -qa --out-path | grepQuiet dependencies
 

From 1d28d613b1d0447b60de2d2044aeac3e1d543aa6 Mon Sep 17 00:00:00 2001
From: Bouke van der Bijl <i@bou.ke>
Date: Wed, 25 Oct 2023 11:39:18 +0200
Subject: [PATCH 005/294] config: add included files into parsedContents before
 applying

Fixes #8719
---
 src/libutil/config.cc    | 16 ++++++++++++----
 tests/functional/init.sh |  3 ++-
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 8e06273ee..17380b6d8 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -88,10 +88,9 @@ void Config::getSettings(std::map<std::string, SettingInfo> & res, bool overridd
             res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description});
 }
 
-void AbstractConfig::applyConfig(const std::string & contents, const std::string & path) {
-    unsigned int pos = 0;
 
-    std::vector<std::pair<std::string, std::string>> parsedContents;
+static void applyConfigInner(const std::string & contents, const std::string & path, std::vector<std::pair<std::string, std::string>> & parsedContents) {
+    unsigned int pos = 0;
 
     while (pos < contents.size()) {
         std::string line;
@@ -123,7 +122,10 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string
                 throw UsageError("illegal configuration line '%1%' in '%2%'", line, path);
             auto p = absPath(tokens[1], dirOf(path));
             if (pathExists(p)) {
-                applyConfigFile(p);
+                try {
+                    std::string includedContents = readFile(path);
+                    applyConfigInner(includedContents, p, parsedContents);
+                } catch (SysError &) { }
             } else if (!ignoreMissing) {
                 throw Error("file '%1%' included from '%2%' not found", p, path);
             }
@@ -143,6 +145,12 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string
             concatStringsSep(" ", Strings(i, tokens.end())),
         });
     };
+}
+
+void AbstractConfig::applyConfig(const std::string & contents, const std::string & path) {
+    std::vector<std::pair<std::string, std::string>> parsedContents;
+
+    applyConfigInner(contents, path, parsedContents);
 
     // First apply experimental-feature related settings
     for (auto & [name, value] : parsedContents)
diff --git a/tests/functional/init.sh b/tests/functional/init.sh
index c420e8c9f..d697b1a30 100755
--- a/tests/functional/init.sh
+++ b/tests/functional/init.sh
@@ -20,7 +20,7 @@ cat > "$NIX_CONF_DIR"/nix.conf <<EOF
 build-users-group =
 keep-derivations = false
 sandbox = false
-experimental-features = nix-command flakes
+experimental-features = nix-command
 gc-reserved-space = 0
 substituters =
 flake-registry = $TEST_ROOT/registry.json
@@ -31,6 +31,7 @@ EOF
 
 cat > "$NIX_CONF_DIR"/nix.conf.extra <<EOF
 fsync-metadata = false
+extra-experimental-features = flakes
 !include nix.conf.extra.not-there
 EOF
 

From 78278f2b3f1c1c55aec7c147d2e266b805863985 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Wed, 25 Oct 2023 12:00:56 +0200
Subject: [PATCH 006/294] add notes on comments in code samples

---
 doc/manual/src/contributing/documentation.md | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/doc/manual/src/contributing/documentation.md b/doc/manual/src/contributing/documentation.md
index f73ab2149..190d367db 100644
--- a/doc/manual/src/contributing/documentation.md
+++ b/doc/manual/src/contributing/documentation.md
@@ -73,6 +73,17 @@ It should therefore aim to be correct, consistent, complete, and easy to navigat
 
   Non-trivial examples may need additional explanation, especially if they use concepts from outside the given context.
 
+- Always explain code examples in the text.
+
+  Use comments in code samples very sparingly, for instance to highlight a particular aspect.
+  Readers tend to glance over large amounts of code when scanning for information.
+
+  Especially beginners will likely find reading more complex-looking code strenuous and may therefore avoid it altogether.
+
+  If a code sample appears to require a lot of inline explanation, consider replacing it with a simpler one.
+  If that's not possible, break the example down into multiple parts, explain them separately, and then show the combined result at the end.
+  This should be a last resort, as that would amount to writing a [tutorial](https://diataxis.fr/tutorials/) on the given subject.
+
 - Use British English.
 
   This is a somewhat arbitrary choice to force consistency, and accounts for the fact that a majority of Nix users and developers are from Europe.

From 00c90eae95c5987a8352dd786d9687f3d213f54a Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Wed, 25 Oct 2023 12:01:17 +0200
Subject: [PATCH 007/294] add note on highlighting examples and syntax
 definitions

---
 doc/manual/src/contributing/documentation.md | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/doc/manual/src/contributing/documentation.md b/doc/manual/src/contributing/documentation.md
index f73ab2149..e35a29d93 100644
--- a/doc/manual/src/contributing/documentation.md
+++ b/doc/manual/src/contributing/documentation.md
@@ -151,6 +151,24 @@ Please observe these guidelines to ease reviews:
   > This is a note.
   ```
 
+  Highlight examples as such:
+
+  ````
+  > **Example**
+  >
+  > ```console
+  > $ nix --version
+  > ```
+  ````
+
+  Highlight syntax definiions as such, using [EBNF](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form) notation:
+
+  ````
+  > **Syntax**
+  >
+  > *attribute-set* = `{` [ *attribute-name* `=` *expression* `;` ... ] `}`
+  ````
+
 ### The `@docroot@` variable
 
 `@docroot@` provides a base path for links that occur in reusable snippets or other documentation that doesn't have a base path of its own.

From 95d657c8b3ae4282e24628ba7426edb90c8f3942 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 25 Oct 2023 18:18:15 +0200
Subject: [PATCH 008/294] Input: Replace markFileChanged() by putFile()

Committing a lock file using markFileChanged() required the input to
be writable by the caller in the local filesystem (using the path
returned by getSourcePath()). putFile() abstracts over this.
---
 src/libexpr/flake/flake.cc   | 67 +++++++++++++++++++-----------------
 src/libfetchers/cache.hh     |  1 +
 src/libfetchers/fetchers.cc  | 17 +++++----
 src/libfetchers/fetchers.hh  | 21 +++++++----
 src/libfetchers/git.cc       | 20 +++++++----
 src/libfetchers/indirect.cc  |  1 +
 src/libfetchers/mercurial.cc | 21 +++++++----
 src/libfetchers/path.cc      | 20 +++++++++--
 8 files changed, 109 insertions(+), 59 deletions(-)

diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index be2cf014c..5c2a1623a 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -623,12 +623,7 @@ LockedFlake lockFlake(
 
         debug("new lock file: %s", newLockFile);
 
-        auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock";
         auto sourcePath = topRef.input.getSourcePath();
-        auto outputLockFilePath = sourcePath ? std::optional{*sourcePath + "/" + relPath} : std::nullopt;
-        if (lockFlags.outputLockFilePath) {
-            outputLockFilePath = lockFlags.outputLockFilePath;
-        }
 
         /* Check whether we need to / can write the new lock file. */
         if (newLockFile != oldLockFile || lockFlags.outputLockFilePath) {
@@ -636,7 +631,7 @@ LockedFlake lockFlake(
             auto diff = LockFile::diff(oldLockFile, newLockFile);
 
             if (lockFlags.writeLockFile) {
-                if (outputLockFilePath) {
+                if (sourcePath || lockFlags.outputLockFilePath) {
                     if (auto unlockedInput = newLockFile.isUnlocked()) {
                         if (fetchSettings.warnDirty)
                             warn("will not write lock file of flake '%s' because it has an unlocked input ('%s')", topRef, *unlockedInput);
@@ -644,41 +639,49 @@ LockedFlake lockFlake(
                         if (!lockFlags.updateLockFile)
                             throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef);
 
-                        bool lockFileExists = pathExists(*outputLockFilePath);
+                        auto newLockFileS = fmt("%s\n", newLockFile);
 
-                        if (lockFileExists) {
-                            auto s = chomp(diff);
-                            if (s.empty())
-                                warn("updating lock file '%s'", *outputLockFilePath);
-                            else
-                                warn("updating lock file '%s':\n%s", *outputLockFilePath, s);
-                        } else
-                            warn("creating lock file '%s'", *outputLockFilePath);
+                        if (lockFlags.outputLockFilePath)
+                            writeFile(*lockFlags.outputLockFilePath, newLockFileS);
+                        else {
+                            auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock";
+                            auto outputLockFilePath = sourcePath ? std::optional{*sourcePath + "/" + relPath} : std::nullopt;
 
-                        newLockFile.write(*outputLockFilePath);
+                            bool lockFileExists = pathExists(*outputLockFilePath);
 
-                        std::optional<std::string> commitMessage = std::nullopt;
-                        if (lockFlags.commitLockFile) {
-                            if (lockFlags.outputLockFilePath) {
-                                throw Error("--commit-lock-file and --output-lock-file are currently incompatible");
-                            }
-                            std::string cm;
+                            if (lockFileExists) {
+                                auto s = chomp(diff);
+                                if (s.empty())
+                                    warn("updating lock file '%s'", *outputLockFilePath);
+                                else
+                                    warn("updating lock file '%s':\n%s", *outputLockFilePath, s);
+                            } else
+                                warn("creating lock file '%s'", *outputLockFilePath);
 
-                            cm = fetchSettings.commitLockFileSummary.get();
+                            std::optional<std::string> commitMessage = std::nullopt;
 
-                            if (cm == "") {
-                                cm = fmt("%s: %s", relPath, lockFileExists ? "Update" : "Add");
+                            if (lockFlags.commitLockFile) {
+                                if (lockFlags.outputLockFilePath) {
+                                    throw Error("--commit-lock-file and --output-lock-file are currently incompatible");
+                                }
+                                std::string cm;
+
+                                cm = fetchSettings.commitLockFileSummary.get();
+
+                                if (cm == "") {
+                                    cm = fmt("%s: %s", relPath, lockFileExists ? "Update" : "Add");
+                                }
+
+                                cm += "\n\nFlake lock file updates:\n\n";
+                                cm += filterANSIEscapes(diff, true);
+                                commitMessage = cm;
                             }
 
-                            cm += "\n\nFlake lock file updates:\n\n";
-                            cm += filterANSIEscapes(diff, true);
-                            commitMessage = cm;
+                            topRef.input.putFile(
+                                CanonPath((topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"),
+                                newLockFileS, commitMessage);
                         }
 
-                        topRef.input.markChangedFile(
-                            (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock",
-                            commitMessage);
-
                         /* Rewriting the lockfile changed the top-level
                            repo, so we should re-read it. FIXME: we could
                            also just clear the 'rev' field... */
diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh
index ae398d040..af34e66ce 100644
--- a/src/libfetchers/cache.hh
+++ b/src/libfetchers/cache.hh
@@ -2,6 +2,7 @@
 ///@file
 
 #include "fetchers.hh"
+#include "path.hh"
 
 namespace nix::fetchers {
 
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index 5688c4dc1..c339c441b 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -196,12 +196,13 @@ std::optional<Path> Input::getSourcePath() const
     return scheme->getSourcePath(*this);
 }
 
-void Input::markChangedFile(
-    std::string_view file,
+void Input::putFile(
+    const CanonPath & path,
+    std::string_view contents,
     std::optional<std::string> commitMsg) const
 {
     assert(scheme);
-    return scheme->markChangedFile(*this, file, commitMsg);
+    return scheme->putFile(*this, path, contents, commitMsg);
 }
 
 std::string Input::getName() const
@@ -292,14 +293,18 @@ Input InputScheme::applyOverrides(
     return input;
 }
 
-std::optional<Path> InputScheme::getSourcePath(const Input & input)
+std::optional<Path> InputScheme::getSourcePath(const Input & input) const
 {
     return {};
 }
 
-void InputScheme::markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg)
+void InputScheme::putFile(
+    const Input & input,
+    const CanonPath & path,
+    std::string_view contents,
+    std::optional<std::string> commitMsg) const
 {
-    assert(false);
+    throw Error("input '%s' does not support modifying file '%s'", input.to_string(), path);
 }
 
 void InputScheme::clone(const Input & input, const Path & destDir) const
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index ac605ff8e..4212a3e1f 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -3,13 +3,13 @@
 
 #include "types.hh"
 #include "hash.hh"
-#include "path.hh"
+#include "canon-path.hh"
 #include "attrs.hh"
 #include "url.hh"
 
 #include <memory>
 
-namespace nix { class Store; }
+namespace nix { class Store; class StorePath; }
 
 namespace nix::fetchers {
 
@@ -90,8 +90,13 @@ public:
 
     std::optional<Path> getSourcePath() const;
 
-    void markChangedFile(
-        std::string_view file,
+    /**
+     * Write a file to this input, for input types that support
+     * writing. Optionally commit the change (for e.g. Git inputs).
+     */
+    void putFile(
+        const CanonPath & path,
+        std::string_view contents,
         std::optional<std::string> commitMsg) const;
 
     std::string getName() const;
@@ -135,9 +140,13 @@ struct InputScheme
 
     virtual void clone(const Input & input, const Path & destDir) const;
 
-    virtual std::optional<Path> getSourcePath(const Input & input);
+    virtual std::optional<Path> getSourcePath(const Input & input) const;
 
-    virtual void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg);
+    virtual void putFile(
+        const Input & input,
+        const CanonPath & path,
+        std::string_view contents,
+        std::optional<std::string> commitMsg) const;
 
     virtual std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input) = 0;
 
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 26b8987d6..4bfd53b0e 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -354,7 +354,7 @@ struct GitInputScheme : InputScheme
         runProgram("git", true, args, {}, true);
     }
 
-    std::optional<Path> getSourcePath(const Input & input) override
+    std::optional<Path> getSourcePath(const Input & input) const override
     {
         auto url = parseURL(getStrAttr(input.attrs, "url"));
         if (url.scheme == "file" && !input.getRef() && !input.getRev())
@@ -362,18 +362,26 @@ struct GitInputScheme : InputScheme
         return {};
     }
 
-    void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
+    void putFile(
+        const Input & input,
+        const CanonPath & path,
+        std::string_view contents,
+        std::optional<std::string> commitMsg) const override
     {
-        auto sourcePath = getSourcePath(input);
-        assert(sourcePath);
+        auto root = getSourcePath(input);
+        if (!root)
+            throw Error("cannot commit '%s' to Git repository '%s' because it's not a working tree", path, input.to_string());
+
+        writeFile((CanonPath(*root) + path).abs(), contents);
+
         auto gitDir = ".git";
 
         runProgram("git", true,
-            { "-C", *sourcePath, "--git-dir", gitDir, "add", "--intent-to-add", "--", std::string(file) });
+            { "-C", *root, "--git-dir", gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) });
 
         if (commitMsg)
             runProgram("git", true,
-                { "-C", *sourcePath, "--git-dir", gitDir, "commit", std::string(file), "-m", *commitMsg });
+                { "-C", *root, "--git-dir", gitDir, "commit", std::string(path.rel()), "-m", *commitMsg });
     }
 
     std::pair<bool, std::string> getActualUrl(const Input & input) const
diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc
index 9a71df3d4..b18411bdc 100644
--- a/src/libfetchers/indirect.cc
+++ b/src/libfetchers/indirect.cc
@@ -1,5 +1,6 @@
 #include "fetchers.hh"
 #include "url-parts.hh"
+#include "path.hh"
 
 namespace nix::fetchers {
 
diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc
index f830a3271..97c48afc9 100644
--- a/src/libfetchers/mercurial.cc
+++ b/src/libfetchers/mercurial.cc
@@ -109,7 +109,7 @@ struct MercurialInputScheme : InputScheme
         return res;
     }
 
-    std::optional<Path> getSourcePath(const Input & input) override
+    std::optional<Path> getSourcePath(const Input & input) const override
     {
         auto url = parseURL(getStrAttr(input.attrs, "url"));
         if (url.scheme == "file" && !input.getRef() && !input.getRev())
@@ -117,18 +117,27 @@ struct MercurialInputScheme : InputScheme
         return {};
     }
 
-    void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
+    void putFile(
+        const Input & input,
+        const CanonPath & path,
+        std::string_view contents,
+        std::optional<std::string> commitMsg) const override
     {
-        auto sourcePath = getSourcePath(input);
-        assert(sourcePath);
+        auto [isLocal, repoPath] = getActualUrl(input);
+        if (!isLocal)
+            throw Error("cannot commit '%s' to Mercurial repository '%s' because it's not a working tree", path, input.to_string());
+
+        auto absPath = CanonPath(repoPath) + path;
+
+        writeFile(absPath.abs(), contents);
 
         // FIXME: shut up if file is already tracked.
         runHg(
-            { "add", *sourcePath + "/" + std::string(file) });
+            { "add", absPath.abs() });
 
         if (commitMsg)
             runHg(
-                { "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg });
+                { "commit", absPath.abs(), "-m", *commitMsg });
     }
 
     std::pair<bool, std::string> getActualUrl(const Input & input) const
diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc
index d829609b5..22be0f1fe 100644
--- a/src/libfetchers/path.cc
+++ b/src/libfetchers/path.cc
@@ -66,14 +66,28 @@ struct PathInputScheme : InputScheme
         };
     }
 
-    std::optional<Path> getSourcePath(const Input & input) override
+    std::optional<Path> getSourcePath(const Input & input) const override
     {
         return getStrAttr(input.attrs, "path");
     }
 
-    void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
+    void putFile(
+        const Input & input,
+        const CanonPath & path,
+        std::string_view contents,
+        std::optional<std::string> commitMsg) const override
     {
-        // nothing to do
+        writeFile((CanonPath(getAbsPath(input)) + path).abs(), contents);
+    }
+
+    CanonPath getAbsPath(const Input & input) const
+    {
+        auto path = getStrAttr(input.attrs, "path");
+
+        if (path[0] == '/')
+            return CanonPath(path);
+
+        throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string());
     }
 
     std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override

From 15c430f38971c2f852effec22392cbe1da511aec Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 25 Oct 2023 18:44:09 +0200
Subject: [PATCH 009/294] Remove unused LockFile::write()

---
 src/libexpr/flake/lockfile.cc | 6 ------
 src/libexpr/flake/lockfile.hh | 2 --
 2 files changed, 8 deletions(-)

diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc
index f3ea9063f..3e99fb2d4 100644
--- a/src/libexpr/flake/lockfile.cc
+++ b/src/libexpr/flake/lockfile.cc
@@ -214,12 +214,6 @@ std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile)
     return stream;
 }
 
-void LockFile::write(const Path & path) const
-{
-    createDirs(dirOf(path));
-    writeFile(path, fmt("%s\n", *this));
-}
-
 std::optional<FlakeRef> LockFile::isUnlocked() const
 {
     std::set<ref<const Node>> nodes;
diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh
index ba4c0c848..5a1493404 100644
--- a/src/libexpr/flake/lockfile.hh
+++ b/src/libexpr/flake/lockfile.hh
@@ -65,8 +65,6 @@ struct LockFile
 
     static LockFile read(const Path & path);
 
-    void write(const Path & path) const;
-
     /**
      * Check whether this lock file has any unlocked inputs.
      */

From 46028ff76493439921a5a9200d14c015f0a0c025 Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <silvan.mosberger@tweag.io>
Date: Thu, 26 Oct 2023 07:05:48 +0200
Subject: [PATCH 010/294] doc: Fix fetchGit default name (#9241)

---
 src/libexpr/primops/fetchTree.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index a99b0e500..767f559be 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -392,7 +392,7 @@ static RegisterPrimOp primop_fetchGit({
 
         The URL of the repo.
 
-      - `name` (default: *basename of the URL*)
+      - `name` (default: `source`)
 
         The name of the directory the repo should be exported to in the store.
 

From b66381e8d8728c040197ec78ed47d5eff88e1d0e Mon Sep 17 00:00:00 2001
From: Jacek Galowicz <jacek@galowicz.de>
Date: Tue, 24 Oct 2023 20:16:34 +0200
Subject: [PATCH 011/294] Use using instead of typedef

---
 src/libstore/path-info.hh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh
index a82e643ae..580e94189 100644
--- a/src/libstore/path-info.hh
+++ b/src/libstore/path-info.hh
@@ -29,7 +29,7 @@ struct SubstitutablePathInfo
     uint64_t narSize;
 };
 
-typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos;
+using SubstitutablePathInfos = std::map<StorePath, SubstitutablePathInfo>;
 
 
 struct UnkeyedValidPathInfo
@@ -136,6 +136,6 @@ struct ValidPathInfo : UnkeyedValidPathInfo {
     virtual ~ValidPathInfo() { }
 };
 
-typedef std::map<StorePath, ValidPathInfo> ValidPathInfos;
+using ValidPathInfos = std::map<StorePath, ValidPathInfo>;
 
 }

From 28c39c370c99148ccc6576d429b41a1f46b08174 Mon Sep 17 00:00:00 2001
From: Jacek Galowicz <jacek@galowicz.de>
Date: Tue, 24 Oct 2023 20:16:46 +0200
Subject: [PATCH 012/294] Provide default value for id to fix warning

---
 src/libstore/path-info.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh
index 580e94189..c4c4a6366 100644
--- a/src/libstore/path-info.hh
+++ b/src/libstore/path-info.hh
@@ -42,7 +42,7 @@ struct UnkeyedValidPathInfo
     StorePathSet references;
     time_t registrationTime = 0;
     uint64_t narSize = 0; // 0 = unknown
-    uint64_t id; // internal use only
+    uint64_t id = 0; // internal use only
 
     /**
      * Whether the path is ultimately trusted, that is, it's a

From e69c764708a78ae2ea38b4c37e5c07119e7097d0 Mon Sep 17 00:00:00 2001
From: Sergei Trofimovich <slyich@gmail.com>
Date: Thu, 26 Oct 2023 20:53:03 +0100
Subject: [PATCH 013/294] local-derivation-goal.cc: slightly clarify waiting
 message

Before the change builder ID exhaustion printed the following message:

    [0/1 built] waiting for UID to build '/nix/store/hiy9136x0iyib4ssh3w3r5m8pxjnad50-python3.11-breathe-4.35.0.drv'

After the change it should be:

    [0/1 built] waiting for a free build user ID for '/nix/store/hiy9136x0iyib4ssh3w3r5m8pxjnad50-python3.11-breathe-4.35.0.drv'
---
 src/libstore/build/local-derivation-goal.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index b5b060d95..738e7051e 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -227,7 +227,7 @@ void LocalDerivationGoal::tryLocalBuild()
         if (!buildUser) {
             if (!actLock)
                 actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
-                    fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
+                    fmt("waiting for a free build user ID for '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
             worker.waitForAWhile(shared_from_this());
             return;
         }

From a419b6149705bddff60215a0d21ef355857ef2c5 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 26 Oct 2023 17:58:45 -0400
Subject: [PATCH 014/294] Turn derivation unit tests into unit characterization
 tests

The brings a number of advantages, including:

- Easier to update test data if design changes (and I do think our
  derivation JSON is not yet complaint with the guidelines).

- Easier to reuse test data in other implementations, inching closer to
  compliance tests for Nix *the concept* rather than any one
  implementation.
---
 src/libstore/tests/characterization.hh        |   5 +
 src/libstore/tests/common-protocol.cc         |   8 +-
 src/libstore/tests/derivation.cc              | 273 ++++++++----------
 src/libstore/tests/protocol.hh                |   4 +-
 .../derivation/bad-old-version-dyn-deps.drv   |   1 +
 .../libstore/derivation/bad-version.drv       |   1 +
 .../libstore/derivation/dynDerivationDeps.drv |   1 +
 .../derivation/dynDerivationDeps.json         |  38 +++
 .../derivation/output-caFixedFlat.json        |   5 +
 .../derivation/output-caFixedNAR.json         |   5 +
 .../derivation/output-caFixedText.json        |   5 +
 .../derivation/output-caFloating.json         |   3 +
 .../libstore/derivation/output-deferred.json  |   1 +
 .../libstore/derivation/output-impure.json    |   4 +
 .../derivation/output-inputAddressed.json     |   3 +
 unit-test-data/libstore/derivation/simple.drv |   1 +
 .../libstore/derivation/simple.json           |  25 ++
 17 files changed, 228 insertions(+), 155 deletions(-)
 create mode 100644 unit-test-data/libstore/derivation/bad-old-version-dyn-deps.drv
 create mode 100644 unit-test-data/libstore/derivation/bad-version.drv
 create mode 100644 unit-test-data/libstore/derivation/dynDerivationDeps.drv
 create mode 100644 unit-test-data/libstore/derivation/dynDerivationDeps.json
 create mode 100644 unit-test-data/libstore/derivation/output-caFixedFlat.json
 create mode 100644 unit-test-data/libstore/derivation/output-caFixedNAR.json
 create mode 100644 unit-test-data/libstore/derivation/output-caFixedText.json
 create mode 100644 unit-test-data/libstore/derivation/output-caFloating.json
 create mode 100644 unit-test-data/libstore/derivation/output-deferred.json
 create mode 100644 unit-test-data/libstore/derivation/output-impure.json
 create mode 100644 unit-test-data/libstore/derivation/output-inputAddressed.json
 create mode 100644 unit-test-data/libstore/derivation/simple.drv
 create mode 100644 unit-test-data/libstore/derivation/simple.json

diff --git a/src/libstore/tests/characterization.hh b/src/libstore/tests/characterization.hh
index 5f366cb42..46bf4b2e5 100644
--- a/src/libstore/tests/characterization.hh
+++ b/src/libstore/tests/characterization.hh
@@ -20,4 +20,9 @@ static bool testAccept() {
     return getEnv("_NIX_TEST_ACCEPT") == "1";
 }
 
+constexpr std::string_view cannotReadGoldenMaster =
+    "Cannot read golden master because another test is also updating it";
+
+constexpr std::string_view updatingGoldenMaster =
+    "Updating golden master";
 }
diff --git a/src/libstore/tests/common-protocol.cc b/src/libstore/tests/common-protocol.cc
index 61c2cb70c..b3f4977d2 100644
--- a/src/libstore/tests/common-protocol.cc
+++ b/src/libstore/tests/common-protocol.cc
@@ -24,14 +24,14 @@ public:
     {
         if (testAccept())
         {
-            GTEST_SKIP() << "Cannot read golden master because another test is also updating it";
+            GTEST_SKIP() << cannotReadGoldenMaster;
         }
         else
         {
-            auto expected = readFile(goldenMaster(testStem));
+            auto encoded = readFile(goldenMaster(testStem));
 
             T got = ({
-                StringSource from { expected };
+                StringSource from { encoded };
                 CommonProto::Serialise<T>::read(
                     *store,
                     CommonProto::ReadConn { .from = from });
@@ -59,7 +59,7 @@ public:
         {
             createDirs(dirOf(file));
             writeFile(file, to.s);
-            GTEST_SKIP() << "Updating golden master";
+            GTEST_SKIP() << updatingGoldenMaster;
         }
         else
         {
diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc
index c360c9707..ca0cdff71 100644
--- a/src/libstore/tests/derivation.cc
+++ b/src/libstore/tests/derivation.cc
@@ -5,9 +5,12 @@
 #include "derivations.hh"
 
 #include "tests/libstore.hh"
+#include "tests/characterization.hh"
 
 namespace nix {
 
+using nlohmann::json;
+
 class DerivationTest : public LibStoreTest
 {
 public:
@@ -16,6 +19,12 @@ public:
      * to worry about race conditions if the tests run concurrently.
      */
     ExperimentalFeatureSettings mockXpSettings;
+
+    Path unitTestData = getUnitTestData() + "/libstore/derivation";
+
+    Path goldenMaster(std::string_view testStem) {
+        return unitTestData + "/" + testStem;
+    }
 };
 
 class CaDerivationTest : public DerivationTest
@@ -46,7 +55,7 @@ TEST_F(DerivationTest, BadATerm_version) {
     ASSERT_THROW(
         parseDerivation(
             *store,
-            R"(DrvWithVersion("invalid-version",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))",
+            readFile(goldenMaster("bad-version.drv")),
             "whatever",
             mockXpSettings),
         FormatError);
@@ -56,50 +65,61 @@ TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) {
     ASSERT_THROW(
         parseDerivation(
             *store,
-            R"(Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))",
+            readFile(goldenMaster("bad-old-version-dyn-deps.drv")),
             "dyn-dep-derivation",
             mockXpSettings),
         FormatError);
 }
 
-#define TEST_JSON(FIXTURE, NAME, STR, VAL, DRV_NAME, OUTPUT_NAME) \
-    TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) {    \
-        using nlohmann::literals::operator "" _json;           \
-        ASSERT_EQ(                                             \
-            STR ## _json,                                      \
-            (DerivationOutput { VAL }).toJSON(                 \
-                *store,                                        \
-                DRV_NAME,                                      \
-                OUTPUT_NAME));                                 \
-    }                                                          \
-                                                               \
-    TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _from_json) {  \
-        using nlohmann::literals::operator "" _json;           \
-        ASSERT_EQ(                                             \
-            DerivationOutput { VAL },                          \
-            DerivationOutput::fromJSON(                        \
-                *store,                                        \
-                DRV_NAME,                                      \
-                OUTPUT_NAME,                                   \
-                STR ## _json,                                  \
-                mockXpSettings));                              \
+#define TEST_JSON(FIXTURE, NAME, VAL, DRV_NAME, OUTPUT_NAME)              \
+    TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _from_json) {            \
+        if (testAccept())                                                 \
+        {                                                                 \
+            GTEST_SKIP() << cannotReadGoldenMaster;                       \
+        }                                                                 \
+        else                                                              \
+        {                                                                 \
+            auto encoded = json::parse(                                   \
+                readFile(goldenMaster("output-" #NAME ".json")));         \
+            DerivationOutput got = DerivationOutput::fromJSON(            \
+                *store,                                                   \
+                DRV_NAME,                                                 \
+                OUTPUT_NAME,                                              \
+                encoded,                                                  \
+                mockXpSettings);                                          \
+            DerivationOutput expected { VAL };                            \
+            ASSERT_EQ(got, expected);                                     \
+        }                                                                 \
+    }                                                                     \
+                                                                          \
+    TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) {              \
+        auto file = goldenMaster("output-" #NAME ".json");                \
+                                                                          \
+        json got = DerivationOutput { VAL }.toJSON(                       \
+            *store,                                                       \
+            DRV_NAME,                                                     \
+            OUTPUT_NAME);                                                 \
+                                                                          \
+        if (testAccept())                                                 \
+        {                                                                 \
+            createDirs(dirOf(file));                                      \
+            writeFile(file, got.dump(2) + "\n");                          \
+            GTEST_SKIP() << updatingGoldenMaster;                         \
+        }                                                                 \
+        else                                                              \
+        {                                                                 \
+            auto expected = json::parse(readFile(file));                  \
+            ASSERT_EQ(got, expected);                                     \
+        }                                                                 \
     }
 
 TEST_JSON(DerivationTest, inputAddressed,
-    R"({
-        "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"
-    })",
     (DerivationOutput::InputAddressed {
         .path = store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"),
     }),
     "drv-name", "output-name")
 
 TEST_JSON(DerivationTest, caFixedFlat,
-    R"({
-        "hashAlgo": "sha256",
-        "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
-        "path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name"
-    })",
     (DerivationOutput::CAFixed {
         .ca = {
             .method = FileIngestionMethod::Flat,
@@ -109,11 +129,6 @@ TEST_JSON(DerivationTest, caFixedFlat,
     "drv-name", "output-name")
 
 TEST_JSON(DerivationTest, caFixedNAR,
-    R"({
-        "hashAlgo": "r:sha256",
-        "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
-        "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"
-    })",
     (DerivationOutput::CAFixed {
         .ca = {
             .method = FileIngestionMethod::Recursive,
@@ -123,11 +138,6 @@ TEST_JSON(DerivationTest, caFixedNAR,
     "drv-name", "output-name")
 
 TEST_JSON(DynDerivationTest, caFixedText,
-    R"({
-        "hashAlgo": "text:sha256",
-        "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
-        "path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name"
-    })",
     (DerivationOutput::CAFixed {
         .ca = {
             .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="),
@@ -136,9 +146,6 @@ TEST_JSON(DynDerivationTest, caFixedText,
     "drv-name", "output-name")
 
 TEST_JSON(CaDerivationTest, caFloating,
-    R"({
-        "hashAlgo": "r:sha256"
-    })",
     (DerivationOutput::CAFloating {
         .method = FileIngestionMethod::Recursive,
         .hashType = htSHA256,
@@ -146,15 +153,10 @@ TEST_JSON(CaDerivationTest, caFloating,
     "drv-name", "output-name")
 
 TEST_JSON(DerivationTest, deferred,
-    R"({ })",
     DerivationOutput::Deferred { },
     "drv-name", "output-name")
 
 TEST_JSON(ImpureDerivationTest, impure,
-    R"({
-        "hashAlgo": "r:sha256",
-        "impure": true
-    })",
     (DerivationOutput::Impure {
         .method = FileIngestionMethod::Recursive,
         .hashType = htSHA256,
@@ -163,43 +165,79 @@ TEST_JSON(ImpureDerivationTest, impure,
 
 #undef TEST_JSON
 
-#define TEST_JSON(FIXTURE, NAME, STR, VAL)                       \
-    TEST_F(FIXTURE, Derivation_ ## NAME ## _to_json) {           \
-        using nlohmann::literals::operator "" _json;             \
-        ASSERT_EQ(                                               \
-            STR ## _json,                                        \
-            (VAL).toJSON(*store));                               \
-    }                                                            \
-                                                                 \
-    TEST_F(FIXTURE, Derivation_ ## NAME ## _from_json) {         \
-        using nlohmann::literals::operator "" _json;             \
-        ASSERT_EQ(                                               \
-            (VAL),                                               \
-            Derivation::fromJSON(                                \
-                *store,                                          \
-                STR ## _json,                                    \
-                mockXpSettings));                                \
+#define TEST_JSON(FIXTURE, NAME, VAL)                                     \
+    TEST_F(FIXTURE, Derivation_ ## NAME ## _from_json) {                  \
+        if (testAccept())                                                 \
+        {                                                                 \
+            GTEST_SKIP() << cannotReadGoldenMaster;                       \
+        }                                                                 \
+        else                                                              \
+        {                                                                 \
+            auto encoded = json::parse(                                   \
+                readFile(goldenMaster( #NAME ".json")));                  \
+            Derivation expected { VAL };                                  \
+            Derivation got = Derivation::fromJSON(                        \
+                *store,                                                   \
+                encoded,                                                  \
+                mockXpSettings);                                          \
+            ASSERT_EQ(got, expected);                                     \
+        }                                                                 \
+    }                                                                     \
+                                                                          \
+    TEST_F(FIXTURE, Derivation_ ## NAME ## _to_json) {                    \
+        auto file = goldenMaster( #NAME ".json");                         \
+                                                                          \
+        json got = Derivation { VAL }.toJSON(*store);                     \
+                                                                          \
+        if (testAccept())                                                 \
+        {                                                                 \
+            createDirs(dirOf(file));                                      \
+            writeFile(file, got.dump(2) + "\n");                          \
+            GTEST_SKIP() << updatingGoldenMaster;                         \
+        }                                                                 \
+        else                                                              \
+        {                                                                 \
+            auto expected = json::parse(readFile(file));                  \
+            ASSERT_EQ(got, expected);                                     \
+        }                                                                 \
     }
 
-#define TEST_ATERM(FIXTURE, NAME, STR, VAL, DRV_NAME)            \
-    TEST_F(FIXTURE, Derivation_ ## NAME ## _to_aterm) {          \
-        ASSERT_EQ(                                               \
-            STR,                                                 \
-            (VAL).unparse(*store, false));                       \
-    }                                                            \
-                                                                 \
-    TEST_F(FIXTURE, Derivation_ ## NAME ## _from_aterm) {        \
-        auto parsed = parseDerivation(                           \
-            *store,                                              \
-            STR,                                                 \
-            DRV_NAME,                                            \
-            mockXpSettings);                                     \
-        ASSERT_EQ(                                               \
-            (VAL).toJSON(*store),                                \
-            parsed.toJSON(*store));                              \
-        ASSERT_EQ(                                               \
-            (VAL),                                               \
-            parsed);                                             \
+#define TEST_ATERM(FIXTURE, NAME, VAL, DRV_NAME)                          \
+    TEST_F(FIXTURE, Derivation_ ## NAME ## _from_aterm) {                 \
+        if (testAccept())                                                 \
+        {                                                                 \
+            GTEST_SKIP() << cannotReadGoldenMaster;                       \
+        }                                                                 \
+        else                                                              \
+        {                                                                 \
+            auto encoded = readFile(goldenMaster( #NAME ".drv"));         \
+            Derivation expected { VAL };                                  \
+            auto got = parseDerivation(                                   \
+                *store,                                                   \
+                std::move(encoded),                                       \
+                DRV_NAME,                                                 \
+                mockXpSettings);                                          \
+            ASSERT_EQ(got.toJSON(*store), expected.toJSON(*store)) ;      \
+            ASSERT_EQ(got, expected);                                     \
+        }                                                                 \
+    }                                                                     \
+                                                                          \
+    TEST_F(FIXTURE, Derivation_ ## NAME ## _to_aterm) {                   \
+        auto file = goldenMaster( #NAME ".drv");                          \
+                                                                          \
+        auto got = (VAL).unparse(*store, false);                          \
+                                                                          \
+        if (testAccept())                                                 \
+        {                                                                 \
+            createDirs(dirOf(file));                                      \
+            writeFile(file, got);                                         \
+            GTEST_SKIP() << updatingGoldenMaster;                         \
+        }                                                                 \
+        else                                                              \
+        {                                                                 \
+            auto expected = readFile(file);                               \
+            ASSERT_EQ(got, expected);                                     \
+        }                                                                 \
     }
 
 Derivation makeSimpleDrv(const Store & store) {
@@ -236,36 +274,9 @@ Derivation makeSimpleDrv(const Store & store) {
     return drv;
 }
 
-TEST_JSON(DerivationTest, simple,
-    R"({
-      "name": "simple-derivation",
-      "inputSrcs": [
-        "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"
-      ],
-      "inputDrvs": {
-        "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": {
-          "dynamicOutputs": {},
-          "outputs": [
-            "cat",
-            "dog"
-          ]
-        }
-      },
-      "system": "wasm-sel4",
-      "builder": "foo",
-      "args": [
-        "bar",
-        "baz"
-      ],
-      "env": {
-        "BIG_BAD": "WOLF"
-      },
-      "outputs": {}
-    })",
-    makeSimpleDrv(*store))
+TEST_JSON(DerivationTest, simple, makeSimpleDrv(*store))
 
 TEST_ATERM(DerivationTest, simple,
-    R"(Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))",
     makeSimpleDrv(*store),
     "simple-derivation")
 
@@ -321,45 +332,9 @@ Derivation makeDynDepDerivation(const Store & store) {
     return drv;
 }
 
-TEST_JSON(DynDerivationTest, dynDerivationDeps,
-    R"({
-      "name": "dyn-dep-derivation",
-      "inputSrcs": [
-        "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"
-      ],
-      "inputDrvs": {
-        "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": {
-          "dynamicOutputs": {
-            "cat": {
-              "dynamicOutputs": {},
-              "outputs": ["kitten"]
-            },
-            "goose": {
-              "dynamicOutputs": {},
-              "outputs": ["gosling"]
-            }
-          },
-          "outputs": [
-            "cat",
-            "dog"
-          ]
-        }
-      },
-      "system": "wasm-sel4",
-      "builder": "foo",
-      "args": [
-        "bar",
-        "baz"
-      ],
-      "env": {
-        "BIG_BAD": "WOLF"
-      },
-      "outputs": {}
-    })",
-    makeDynDepDerivation(*store))
+TEST_JSON(DynDerivationTest, dynDerivationDeps, makeDynDepDerivation(*store))
 
 TEST_ATERM(DynDerivationTest, dynDerivationDeps,
-    R"(DrvWithVersion("xp-dyn-drv",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))",
     makeDynDepDerivation(*store),
     "dyn-dep-derivation")
 
diff --git a/src/libstore/tests/protocol.hh b/src/libstore/tests/protocol.hh
index 496915745..7fdd3e11c 100644
--- a/src/libstore/tests/protocol.hh
+++ b/src/libstore/tests/protocol.hh
@@ -29,7 +29,7 @@ public:
     {
         if (testAccept())
         {
-            GTEST_SKIP() << "Cannot read golden master because another test is also updating it";
+            GTEST_SKIP() << cannotReadGoldenMaster;
         }
         else
         {
@@ -70,7 +70,7 @@ public:
         {
             createDirs(dirOf(file));
             writeFile(file, to.s);
-            GTEST_SKIP() << "Updating golden master";
+            GTEST_SKIP() << updatingGoldenMaster;
         }
         else
         {
diff --git a/unit-test-data/libstore/derivation/bad-old-version-dyn-deps.drv b/unit-test-data/libstore/derivation/bad-old-version-dyn-deps.drv
new file mode 100644
index 000000000..3cd1ded02
--- /dev/null
+++ b/unit-test-data/libstore/derivation/bad-old-version-dyn-deps.drv
@@ -0,0 +1 @@
+Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")])
\ No newline at end of file
diff --git a/unit-test-data/libstore/derivation/bad-version.drv b/unit-test-data/libstore/derivation/bad-version.drv
new file mode 100644
index 000000000..bbf75c114
--- /dev/null
+++ b/unit-test-data/libstore/derivation/bad-version.drv
@@ -0,0 +1 @@
+DrvWithVersion("invalid-version",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")])
\ No newline at end of file
diff --git a/unit-test-data/libstore/derivation/dynDerivationDeps.drv b/unit-test-data/libstore/derivation/dynDerivationDeps.drv
new file mode 100644
index 000000000..cfffe48ec
--- /dev/null
+++ b/unit-test-data/libstore/derivation/dynDerivationDeps.drv
@@ -0,0 +1 @@
+DrvWithVersion("xp-dyn-drv",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")])
\ No newline at end of file
diff --git a/unit-test-data/libstore/derivation/dynDerivationDeps.json b/unit-test-data/libstore/derivation/dynDerivationDeps.json
new file mode 100644
index 000000000..9dbeb1f15
--- /dev/null
+++ b/unit-test-data/libstore/derivation/dynDerivationDeps.json
@@ -0,0 +1,38 @@
+{
+  "args": [
+    "bar",
+    "baz"
+  ],
+  "builder": "foo",
+  "env": {
+    "BIG_BAD": "WOLF"
+  },
+  "inputDrvs": {
+    "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": {
+      "dynamicOutputs": {
+        "cat": {
+          "dynamicOutputs": {},
+          "outputs": [
+            "kitten"
+          ]
+        },
+        "goose": {
+          "dynamicOutputs": {},
+          "outputs": [
+            "gosling"
+          ]
+        }
+      },
+      "outputs": [
+        "cat",
+        "dog"
+      ]
+    }
+  },
+  "inputSrcs": [
+    "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"
+  ],
+  "name": "dyn-dep-derivation",
+  "outputs": {},
+  "system": "wasm-sel4"
+}
diff --git a/unit-test-data/libstore/derivation/output-caFixedFlat.json b/unit-test-data/libstore/derivation/output-caFixedFlat.json
new file mode 100644
index 000000000..fe000ea36
--- /dev/null
+++ b/unit-test-data/libstore/derivation/output-caFixedFlat.json
@@ -0,0 +1,5 @@
+{
+  "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
+  "hashAlgo": "sha256",
+  "path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name"
+}
diff --git a/unit-test-data/libstore/derivation/output-caFixedNAR.json b/unit-test-data/libstore/derivation/output-caFixedNAR.json
new file mode 100644
index 000000000..1afd60223
--- /dev/null
+++ b/unit-test-data/libstore/derivation/output-caFixedNAR.json
@@ -0,0 +1,5 @@
+{
+  "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
+  "hashAlgo": "r:sha256",
+  "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"
+}
diff --git a/unit-test-data/libstore/derivation/output-caFixedText.json b/unit-test-data/libstore/derivation/output-caFixedText.json
new file mode 100644
index 000000000..0b2cc8bbc
--- /dev/null
+++ b/unit-test-data/libstore/derivation/output-caFixedText.json
@@ -0,0 +1,5 @@
+{
+  "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
+  "hashAlgo": "text:sha256",
+  "path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name"
+}
diff --git a/unit-test-data/libstore/derivation/output-caFloating.json b/unit-test-data/libstore/derivation/output-caFloating.json
new file mode 100644
index 000000000..9115de851
--- /dev/null
+++ b/unit-test-data/libstore/derivation/output-caFloating.json
@@ -0,0 +1,3 @@
+{
+  "hashAlgo": "r:sha256"
+}
diff --git a/unit-test-data/libstore/derivation/output-deferred.json b/unit-test-data/libstore/derivation/output-deferred.json
new file mode 100644
index 000000000..0967ef424
--- /dev/null
+++ b/unit-test-data/libstore/derivation/output-deferred.json
@@ -0,0 +1 @@
+{}
diff --git a/unit-test-data/libstore/derivation/output-impure.json b/unit-test-data/libstore/derivation/output-impure.json
new file mode 100644
index 000000000..62b61cdca
--- /dev/null
+++ b/unit-test-data/libstore/derivation/output-impure.json
@@ -0,0 +1,4 @@
+{
+  "hashAlgo": "r:sha256",
+  "impure": true
+}
diff --git a/unit-test-data/libstore/derivation/output-inputAddressed.json b/unit-test-data/libstore/derivation/output-inputAddressed.json
new file mode 100644
index 000000000..86c7f3a05
--- /dev/null
+++ b/unit-test-data/libstore/derivation/output-inputAddressed.json
@@ -0,0 +1,3 @@
+{
+  "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"
+}
diff --git a/unit-test-data/libstore/derivation/simple.drv b/unit-test-data/libstore/derivation/simple.drv
new file mode 100644
index 000000000..bda74ad25
--- /dev/null
+++ b/unit-test-data/libstore/derivation/simple.drv
@@ -0,0 +1 @@
+Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")])
\ No newline at end of file
diff --git a/unit-test-data/libstore/derivation/simple.json b/unit-test-data/libstore/derivation/simple.json
new file mode 100644
index 000000000..20d0f8933
--- /dev/null
+++ b/unit-test-data/libstore/derivation/simple.json
@@ -0,0 +1,25 @@
+{
+  "args": [
+    "bar",
+    "baz"
+  ],
+  "builder": "foo",
+  "env": {
+    "BIG_BAD": "WOLF"
+  },
+  "inputDrvs": {
+    "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": {
+      "dynamicOutputs": {},
+      "outputs": [
+        "cat",
+        "dog"
+      ]
+    }
+  },
+  "inputSrcs": [
+    "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"
+  ],
+  "name": "simple-derivation",
+  "outputs": {},
+  "system": "wasm-sel4"
+}

From 325db01d269ca8580fc05ca4b56f28232266ecb7 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Fri, 27 Oct 2023 07:30:16 +0200
Subject: [PATCH 015/294] fix anchor in conf-file

I inadvertently switched it to `opt-` when refactoring, but it should
have been `conf` to begin with.
---
 doc/manual/local.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index 8bf16e9dd..db3daf252 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -103,7 +103,7 @@ $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage
 
 $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/generate-settings.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix
 	@cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp
-	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-settings.nix { prefix = "opt-"; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
+	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-settings.nix { prefix = "conf"; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
 	@mv $@.tmp $@
 
 $(d)/nix.json: $(bindir)/nix

From 8381eeda6fa858b74bc7b516b9af9eecbbddd594 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 30 Oct 2023 10:14:27 -0400
Subject: [PATCH 016/294] Systematize fetcher input attribute validation

We now have `schemeName` and `allowedAttrs` functions for this purpose.
We look up the schema with the former; we restrict the set of input
attributes with the latter.
---
 src/libfetchers/fetchers.cc  | 66 ++++++++++++++++++++++++++----------
 src/libfetchers/fetchers.hh  | 20 ++++++++++-
 src/libfetchers/git.cc       | 30 ++++++++++++----
 src/libfetchers/github.cc    | 35 +++++++++++--------
 src/libfetchers/indirect.cc  | 23 +++++++++----
 src/libfetchers/mercurial.cc | 23 +++++++++----
 src/libfetchers/path.cc      | 35 +++++++++++--------
 src/libfetchers/tarball.cc   | 34 +++++++++++--------
 8 files changed, 184 insertions(+), 82 deletions(-)

diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index 5688c4dc1..7a5c97399 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -5,12 +5,18 @@
 
 namespace nix::fetchers {
 
-std::unique_ptr<std::vector<std::shared_ptr<InputScheme>>> inputSchemes = nullptr;
+using InputSchemeMap = std::map<std::string_view, std::shared_ptr<InputScheme>>;
+
+std::unique_ptr<InputSchemeMap> inputSchemes = nullptr;
 
 void registerInputScheme(std::shared_ptr<InputScheme> && inputScheme)
 {
-    if (!inputSchemes) inputSchemes = std::make_unique<std::vector<std::shared_ptr<InputScheme>>>();
-    inputSchemes->push_back(std::move(inputScheme));
+    if (!inputSchemes)
+        inputSchemes = std::make_unique<InputSchemeMap>();
+    auto schemeName = inputScheme->schemeName();
+    if (inputSchemes->count(schemeName) > 0)
+        throw Error("Input scheme with name %s already registered", schemeName);
+    inputSchemes->insert_or_assign(schemeName, std::move(inputScheme));
 }
 
 Input Input::fromURL(const std::string & url, bool requireTree)
@@ -33,7 +39,7 @@ static void fixupInput(Input & input)
 
 Input Input::fromURL(const ParsedURL & url, bool requireTree)
 {
-    for (auto & inputScheme : *inputSchemes) {
+    for (auto & [_, inputScheme] : *inputSchemes) {
         auto res = inputScheme->inputFromURL(url, requireTree);
         if (res) {
             experimentalFeatureSettings.require(inputScheme->experimentalFeature());
@@ -48,20 +54,44 @@ Input Input::fromURL(const ParsedURL & url, bool requireTree)
 
 Input Input::fromAttrs(Attrs && attrs)
 {
-    for (auto & inputScheme : *inputSchemes) {
-        auto res = inputScheme->inputFromAttrs(attrs);
-        if (res) {
-            experimentalFeatureSettings.require(inputScheme->experimentalFeature());
-            res->scheme = inputScheme;
-            fixupInput(*res);
-            return std::move(*res);
-        }
-    }
+    auto schemeName = ({
+        auto schemeNameOpt = maybeGetStrAttr(attrs, "type");
+        if (!schemeNameOpt)
+            throw Error("'type' attribute to specify input scheme is required but not provided");
+        *std::move(schemeNameOpt);
+    });
 
-    Input input;
-    input.attrs = attrs;
-    fixupInput(input);
-    return input;
+    auto raw = [&]() {
+        // Return an input without a scheme; most operations will fail,
+        // but not all of them. Doing this is to support those other
+        // operations which are supposed to be robust on
+        // unknown/uninterpretable inputs.
+        Input input;
+        input.attrs = attrs;
+        fixupInput(input);
+        return input;
+    };
+
+    std::shared_ptr<InputScheme> inputScheme = ({
+        auto i = inputSchemes->find(schemeName);
+        i == inputSchemes->end() ? nullptr : i->second;
+    });
+
+    if (!inputScheme) return raw();
+
+    experimentalFeatureSettings.require(inputScheme->experimentalFeature());
+
+    auto allowedAttrs = inputScheme->allowedAttrs();
+
+    for (auto & [name, _] : attrs)
+        if (name != "type" && allowedAttrs.count(name) == 0)
+            throw Error("input attribute '%s' not supported by scheme '%s'", name, schemeName);
+
+    auto res = inputScheme->inputFromAttrs(attrs);
+    if (!res) return raw();
+    res->scheme = inputScheme;
+    fixupInput(*res);
+    return std::move(*res);
 }
 
 ParsedURL Input::toURL() const
@@ -307,7 +337,7 @@ void InputScheme::clone(const Input & input, const Path & destDir) const
     throw Error("do not know how to clone input '%s'", input.to_string());
 }
 
-std::optional<ExperimentalFeature> InputScheme::experimentalFeature()
+std::optional<ExperimentalFeature> InputScheme::experimentalFeature() const
 {
     return {};
 }
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index ac605ff8e..b35d87eeb 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -126,6 +126,24 @@ struct InputScheme
 
     virtual std::optional<Input> inputFromAttrs(const Attrs & attrs) const = 0;
 
+    /**
+     * What is the name of the scheme?
+     *
+     * The `type` attribute is used to select which input scheme is
+     * used, and then the other fields are forwarded to that input
+     * scheme.
+     */
+    virtual std::string_view schemeName() const = 0;
+
+    /**
+     * Allowed attributes in an attribute set that is converted to an
+     * input.
+     *
+     * `type` is not included from this set, because the `type` field is
+      parsed first to choose which scheme; `type` is always required.
+     */
+    virtual StringSet allowedAttrs() const = 0;
+
     virtual ParsedURL toURL(const Input & input) const;
 
     virtual Input applyOverrides(
@@ -144,7 +162,7 @@ struct InputScheme
     /**
      * Is this `InputScheme` part of an experimental feature?
      */
-    virtual std::optional<ExperimentalFeature> experimentalFeature();
+    virtual std::optional<ExperimentalFeature> experimentalFeature() const;
 
     virtual bool isDirect(const Input & input) const
     { return true; }
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 26b8987d6..bf25434c8 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -285,14 +285,32 @@ struct GitInputScheme : InputScheme
         return inputFromAttrs(attrs);
     }
 
+
+    std::string_view schemeName() const override
+    {
+        return "git";
+    }
+
+    StringSet allowedAttrs() const override
+    {
+        return {
+            "url",
+            "ref",
+            "rev",
+            "shallow",
+            "submodules",
+            "lastModified",
+            "revCount",
+            "narHash",
+            "allRefs",
+            "name",
+            "dirtyRev",
+            "dirtyShortRev",
+        };
+    }
+
     std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
     {
-        if (maybeGetStrAttr(attrs, "type") != "git") return {};
-
-        for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name" && name != "dirtyRev" && name != "dirtyShortRev")
-                throw Error("unsupported Git input attribute '%s'", name);
-
         maybeGetBoolAttr(attrs, "shallow");
         maybeGetBoolAttr(attrs, "submodules");
         maybeGetBoolAttr(attrs, "allRefs");
diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 617fc7468..6c9b29721 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -27,13 +27,11 @@ std::regex hostRegex(hostRegexS, std::regex::ECMAScript);
 
 struct GitArchiveInputScheme : InputScheme
 {
-    virtual std::string type() const = 0;
-
     virtual std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const = 0;
 
     std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override
     {
-        if (url.scheme != type()) return {};
+        if (url.scheme != schemeName()) return {};
 
         auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
 
@@ -91,7 +89,7 @@ struct GitArchiveInputScheme : InputScheme
             throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url.url, *ref, rev->gitRev());
 
         Input input;
-        input.attrs.insert_or_assign("type", type());
+        input.attrs.insert_or_assign("type", std::string { schemeName() });
         input.attrs.insert_or_assign("owner", path[0]);
         input.attrs.insert_or_assign("repo", path[1]);
         if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
@@ -101,14 +99,21 @@ struct GitArchiveInputScheme : InputScheme
         return input;
     }
 
+    StringSet allowedAttrs() const override
+    {
+        return {
+            "owner",
+            "repo",
+            "ref",
+            "rev",
+            "narHash",
+            "lastModified",
+            "host",
+        };
+    }
+
     std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
     {
-        if (maybeGetStrAttr(attrs, "type") != type()) return {};
-
-        for (auto & [name, value] : attrs)
-            if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified" && name != "host")
-                throw Error("unsupported input attribute '%s'", name);
-
         getStrAttr(attrs, "owner");
         getStrAttr(attrs, "repo");
 
@@ -128,7 +133,7 @@ struct GitArchiveInputScheme : InputScheme
         if (ref) path += "/" + *ref;
         if (rev) path += "/" + rev->to_string(HashFormat::Base16, false);
         return ParsedURL {
-            .scheme = type(),
+            .scheme = std::string { schemeName() },
             .path = path,
         };
     }
@@ -220,7 +225,7 @@ struct GitArchiveInputScheme : InputScheme
         return {result.storePath, input};
     }
 
-    std::optional<ExperimentalFeature> experimentalFeature() override
+    std::optional<ExperimentalFeature> experimentalFeature() const override
     {
         return Xp::Flakes;
     }
@@ -228,7 +233,7 @@ struct GitArchiveInputScheme : InputScheme
 
 struct GitHubInputScheme : GitArchiveInputScheme
 {
-    std::string type() const override { return "github"; }
+    std::string_view schemeName() const override { return "github"; }
 
     std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
     {
@@ -309,7 +314,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
 
 struct GitLabInputScheme : GitArchiveInputScheme
 {
-    std::string type() const override { return "gitlab"; }
+    std::string_view schemeName() const override { return "gitlab"; }
 
     std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
     {
@@ -377,7 +382,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
 
 struct SourceHutInputScheme : GitArchiveInputScheme
 {
-    std::string type() const override { return "sourcehut"; }
+    std::string_view schemeName() const override { return "sourcehut"; }
 
     std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
     {
diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc
index 9a71df3d4..06f7f908d 100644
--- a/src/libfetchers/indirect.cc
+++ b/src/libfetchers/indirect.cc
@@ -49,14 +49,23 @@ struct IndirectInputScheme : InputScheme
         return input;
     }
 
+    std::string_view schemeName() const override
+    {
+        return "indirect";
+    }
+
+    StringSet allowedAttrs() const override
+    {
+        return {
+            "id",
+            "ref",
+            "rev",
+            "narHash",
+        };
+    }
+
     std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
     {
-        if (maybeGetStrAttr(attrs, "type") != "indirect") return {};
-
-        for (auto & [name, value] : attrs)
-            if (name != "type" && name != "id" && name != "ref" && name != "rev" && name != "narHash")
-                throw Error("unsupported indirect input attribute '%s'", name);
-
         auto id = getStrAttr(attrs, "id");
         if (!std::regex_match(id, flakeRegex))
             throw BadURL("'%s' is not a valid flake ID", id);
@@ -92,7 +101,7 @@ struct IndirectInputScheme : InputScheme
         throw Error("indirect input '%s' cannot be fetched directly", input.to_string());
     }
 
-    std::optional<ExperimentalFeature> experimentalFeature() override
+    std::optional<ExperimentalFeature> experimentalFeature() const override
     {
         return Xp::Flakes;
     }
diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc
index f830a3271..99002a94f 100644
--- a/src/libfetchers/mercurial.cc
+++ b/src/libfetchers/mercurial.cc
@@ -69,14 +69,25 @@ struct MercurialInputScheme : InputScheme
         return inputFromAttrs(attrs);
     }
 
+    std::string_view schemeName() const override
+    {
+        return "hg";
+    }
+
+    StringSet allowedAttrs() const override
+    {
+        return {
+            "url",
+            "ref",
+            "rev",
+            "revCount",
+            "narHash",
+            "name",
+        };
+    }
+
     std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
     {
-        if (maybeGetStrAttr(attrs, "type") != "hg") return {};
-
-        for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash" && name != "name")
-                throw Error("unsupported Mercurial input attribute '%s'", name);
-
         parseURL(getStrAttr(attrs, "url"));
 
         if (auto ref = maybeGetStrAttr(attrs, "ref")) {
diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc
index d829609b5..699efbc3b 100644
--- a/src/libfetchers/path.cc
+++ b/src/libfetchers/path.cc
@@ -32,23 +32,30 @@ struct PathInputScheme : InputScheme
         return input;
     }
 
+    std::string_view schemeName() const override
+    {
+        return "path";
+    }
+
+    StringSet allowedAttrs() const override
+    {
+        return {
+            "path",
+            /* Allow the user to pass in "fake" tree info
+               attributes. This is useful for making a pinned tree work
+               the same as the repository from which is exported (e.g.
+               path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...).
+             */
+            "rev",
+            "revCount",
+            "lastModified",
+            "narHash",
+        };
+    }
     std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
     {
-        if (maybeGetStrAttr(attrs, "type") != "path") return {};
-
         getStrAttr(attrs, "path");
 
-        for (auto & [name, value] : attrs)
-            /* Allow the user to pass in "fake" tree info
-               attributes. This is useful for making a pinned tree
-               work the same as the repository from which is exported
-               (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */
-            if (name == "type" || name == "rev" || name == "revCount" || name == "lastModified" || name == "narHash" || name == "path")
-                // checked in Input::fromAttrs
-                ;
-            else
-                throw Error("unsupported path input attribute '%s'", name);
-
         Input input;
         input.attrs = attrs;
         return input;
@@ -121,7 +128,7 @@ struct PathInputScheme : InputScheme
         return {std::move(*storePath), input};
     }
 
-    std::optional<ExperimentalFeature> experimentalFeature() override
+    std::optional<ExperimentalFeature> experimentalFeature() const override
     {
         return Xp::Flakes;
     }
diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index e1ea9b58b..0062878a9 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -184,7 +184,6 @@ DownloadTarballResult downloadTarball(
 // An input scheme corresponding to a curl-downloadable resource.
 struct CurlInputScheme : InputScheme
 {
-    virtual const std::string inputType() const = 0;
     const std::set<std::string> transportUrlSchemes = {"file", "http", "https"};
 
     const bool hasTarballExtension(std::string_view path) const
@@ -222,22 +221,27 @@ struct CurlInputScheme : InputScheme
         url.query.erase("rev");
         url.query.erase("revCount");
 
-        input.attrs.insert_or_assign("type", inputType());
+        input.attrs.insert_or_assign("type", std::string { schemeName() });
         input.attrs.insert_or_assign("url", url.to_string());
         return input;
     }
 
+    StringSet allowedAttrs() const override
+    {
+        return {
+            "type",
+            "url",
+            "narHash",
+            "name",
+            "unpack",
+            "rev",
+            "revCount",
+            "lastModified",
+        };
+    }
+
     std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
     {
-        auto type = maybeGetStrAttr(attrs, "type");
-        if (type != inputType()) return {};
-
-        // FIXME: some of these only apply to TarballInputScheme.
-        std::set<std::string> allowedNames = {"type", "url", "narHash", "name", "unpack", "rev", "revCount", "lastModified"};
-        for (auto & [name, value] : attrs)
-            if (!allowedNames.count(name))
-                throw Error("unsupported %s input attribute '%s'", *type, name);
-
         Input input;
         input.attrs = attrs;
 
@@ -258,14 +262,14 @@ struct CurlInputScheme : InputScheme
 
 struct FileInputScheme : CurlInputScheme
 {
-    const std::string inputType() const override { return "file"; }
+    std::string_view schemeName() const override { return "file"; }
 
     bool isValidURL(const ParsedURL & url, bool requireTree) const override
     {
         auto parsedUrlScheme = parseUrlScheme(url.scheme);
         return transportUrlSchemes.count(std::string(parsedUrlScheme.transport))
             && (parsedUrlScheme.application
-                ? parsedUrlScheme.application.value() == inputType()
+                ? parsedUrlScheme.application.value() == schemeName()
                 : (!requireTree && !hasTarballExtension(url.path)));
     }
 
@@ -278,7 +282,7 @@ struct FileInputScheme : CurlInputScheme
 
 struct TarballInputScheme : CurlInputScheme
 {
-    const std::string inputType() const override { return "tarball"; }
+    std::string_view schemeName() const override { return "tarball"; }
 
     bool isValidURL(const ParsedURL & url, bool requireTree) const override
     {
@@ -286,7 +290,7 @@ struct TarballInputScheme : CurlInputScheme
 
         return transportUrlSchemes.count(std::string(parsedUrlScheme.transport))
             && (parsedUrlScheme.application
-                ? parsedUrlScheme.application.value() == inputType()
+                ? parsedUrlScheme.application.value() == schemeName()
                 : (requireTree || hasTarballExtension(url.path)));
     }
 

From 077de2968e8cf2d125818999adf8c149baf6384e Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 30 Oct 2023 10:30:59 -0400
Subject: [PATCH 017/294] Include fetcher input scheme info in the CLI dump

Leverages the previous commit.
---
 src/libfetchers/fetchers.cc | 13 +++++++++++++
 src/libfetchers/fetchers.hh |  3 +++
 src/nix/main.cc             |  1 +
 3 files changed, 17 insertions(+)

diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index 7a5c97399..44b3fa4a5 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -19,6 +19,19 @@ void registerInputScheme(std::shared_ptr<InputScheme> && inputScheme)
     inputSchemes->insert_or_assign(schemeName, std::move(inputScheme));
 }
 
+nlohmann::json dumpRegisterInputSchemeInfo() {
+    using nlohmann::json;
+
+    auto res = json::object();
+
+    for (auto & [name, scheme] : *inputSchemes) {
+        auto & r = res[name] = json::object();
+        r["allowedAttrs"] = scheme->allowedAttrs();
+    }
+
+    return res;
+}
+
 Input Input::fromURL(const std::string & url, bool requireTree)
 {
     return fromURL(parseURL(url), requireTree);
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index b35d87eeb..3a02967f4 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -8,6 +8,7 @@
 #include "url.hh"
 
 #include <memory>
+#include <nlohmann/json_fwd.hpp>
 
 namespace nix { class Store; }
 
@@ -170,4 +171,6 @@ struct InputScheme
 
 void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);
 
+nlohmann::json dumpRegisterInputSchemeInfo();
+
 }
diff --git a/src/nix/main.cc b/src/nix/main.cc
index ffba10099..d20bc1f8a 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -188,6 +188,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs
             j["experimentalFeature"] = storeConfig->experimentalFeature();
         }
         res["stores"] = std::move(stores);
+        res["fetchers"] = fetchers::dumpRegisterInputSchemeInfo();
 
         return res.dump();
     }

From 05316d401fa509557c71140e17bb19814412fcb8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 30 Oct 2023 17:03:06 +0100
Subject: [PATCH 018/294] Cleanup

---
 src/libexpr/flake/flake.cc | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index 5c2a1623a..45c9ec8f3 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -641,29 +641,28 @@ LockedFlake lockFlake(
 
                         auto newLockFileS = fmt("%s\n", newLockFile);
 
-                        if (lockFlags.outputLockFilePath)
+                        if (lockFlags.outputLockFilePath) {
+                            if (lockFlags.commitLockFile)
+                                throw Error("'--commit-lock-file' and '--output-lock-file' are incompatible");
                             writeFile(*lockFlags.outputLockFilePath, newLockFileS);
-                        else {
+                        } else {
                             auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock";
-                            auto outputLockFilePath = sourcePath ? std::optional{*sourcePath + "/" + relPath} : std::nullopt;
+                            auto outputLockFilePath = *sourcePath + "/" + relPath;
 
-                            bool lockFileExists = pathExists(*outputLockFilePath);
+                            bool lockFileExists = pathExists(outputLockFilePath);
 
                             if (lockFileExists) {
                                 auto s = chomp(diff);
                                 if (s.empty())
-                                    warn("updating lock file '%s'", *outputLockFilePath);
+                                    warn("updating lock file '%s'", outputLockFilePath);
                                 else
-                                    warn("updating lock file '%s':\n%s", *outputLockFilePath, s);
+                                    warn("updating lock file '%s':\n%s", outputLockFilePath, s);
                             } else
-                                warn("creating lock file '%s'", *outputLockFilePath);
+                                warn("creating lock file '%s'", outputLockFilePath);
 
                             std::optional<std::string> commitMessage = std::nullopt;
 
                             if (lockFlags.commitLockFile) {
-                                if (lockFlags.outputLockFilePath) {
-                                    throw Error("--commit-lock-file and --output-lock-file are currently incompatible");
-                                }
                                 std::string cm;
 
                                 cm = fetchSettings.commitLockFileSummary.get();

From 95f3f9eac978466c812814c06716f26e9f668e54 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 30 Oct 2023 22:21:34 +0000
Subject: [PATCH 019/294] build(deps): bump zeebe-io/backport-action from 1.4.0
 to 2.0.0

Bumps [zeebe-io/backport-action](https://github.com/zeebe-io/backport-action) from 1.4.0 to 2.0.0.
- [Release notes](https://github.com/zeebe-io/backport-action/releases)
- [Commits](https://github.com/zeebe-io/backport-action/compare/v1.4.0...v2.0.0)

---
updated-dependencies:
- dependency-name: zeebe-io/backport-action
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/backport.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml
index 12c60c649..312c211dd 100644
--- a/.github/workflows/backport.yml
+++ b/.github/workflows/backport.yml
@@ -21,7 +21,7 @@ jobs:
           fetch-depth: 0
       - name: Create backport PRs
         # should be kept in sync with `version`
-        uses: zeebe-io/backport-action@v1.4.0
+        uses: zeebe-io/backport-action@v2.0.0
         with:
           # Config README: https://github.com/zeebe-io/backport-action#backport-action
           github_token: ${{ secrets.GITHUB_TOKEN }}

From e1b8442fa1dbd2e69598dbeb701da4df8e6d2c38 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 24 Oct 2023 08:20:31 +0200
Subject: [PATCH 020/294] Fetcher cache: Add support for caching facts not
 related to store paths

---
 src/libfetchers/cache.cc | 57 ++++++++++++++++++++++++++++++++++++++++
 src/libfetchers/cache.hh | 38 +++++++++++++++++++++++++++
 2 files changed, 95 insertions(+)

diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc
index 0c8ecac9d..8a3e462d3 100644
--- a/src/libfetchers/cache.cc
+++ b/src/libfetchers/cache.cc
@@ -19,6 +19,9 @@ create table if not exists Cache (
 );
 )sql";
 
+// FIXME: we should periodically purge/nuke this cache to prevent it
+// from growing too big.
+
 struct CacheImpl : Cache
 {
     struct State
@@ -47,6 +50,60 @@ struct CacheImpl : Cache
             "select info, path, immutable, timestamp from Cache where input = ?");
     }
 
+    void upsert(
+        const Attrs & inAttrs,
+        const Attrs & infoAttrs) override
+    {
+        _state.lock()->add.use()
+            (attrsToJSON(inAttrs).dump())
+            (attrsToJSON(infoAttrs).dump())
+            ("") // no path
+            (false)
+            (time(0)).exec();
+    }
+
+    std::optional<Attrs> lookup(const Attrs & inAttrs) override
+    {
+        if (auto res = lookupExpired(inAttrs))
+            return std::move(res->infoAttrs);
+        return {};
+    }
+
+    std::optional<Attrs> lookupWithTTL(const Attrs & inAttrs) override
+    {
+        if (auto res = lookupExpired(inAttrs)) {
+            if (!res->expired)
+                return std::move(res->infoAttrs);
+            debug("ignoring expired cache entry '%s'",
+                attrsToJSON(inAttrs).dump());
+        }
+        return {};
+    }
+
+    std::optional<Result2> lookupExpired(const Attrs & inAttrs) override
+    {
+        auto state(_state.lock());
+
+        auto inAttrsJSON = attrsToJSON(inAttrs).dump();
+
+        auto stmt(state->lookup.use()(inAttrsJSON));
+        if (!stmt.next()) {
+            debug("did not find cache entry for '%s'", inAttrsJSON);
+            return {};
+        }
+
+        auto infoJSON = stmt.getStr(0);
+        auto locked = stmt.getInt(2) != 0;
+        auto timestamp = stmt.getInt(3);
+
+        debug("using cache entry '%s' -> '%s'", inAttrsJSON, infoJSON);
+
+        return Result2 {
+            .expired = !locked && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)),
+            .infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJSON)),
+        };
+    }
+
     void add(
         ref<Store> store,
         const Attrs & inAttrs,
diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh
index af34e66ce..b517d496e 100644
--- a/src/libfetchers/cache.hh
+++ b/src/libfetchers/cache.hh
@@ -10,6 +10,44 @@ struct Cache
 {
     virtual ~Cache() { }
 
+    /* A cache for arbitrary Attrs -> Attrs mappings with a timestamp
+       for expiration. */
+
+    /*
+     * Add a value to the cache. The cache is an arbitrary mapping of
+     * Attrs to Attrs.
+     */
+    virtual void upsert(
+        const Attrs & inAttrs,
+        const Attrs & infoAttrs) = 0;
+
+    /*
+     * Look up a key with infinite TTL.
+     */
+    virtual std::optional<Attrs> lookup(
+        const Attrs & inAttrs) = 0;
+
+    /*
+     * Look up a key. Return nothing if its TTL has exceeded
+     * `settings.tarballTTL`.
+     */
+    virtual std::optional<Attrs> lookupWithTTL(
+        const Attrs & inAttrs) = 0;
+
+    struct Result2
+    {
+        bool expired = false;
+        Attrs infoAttrs;
+    };
+
+    /*
+     * Look up a key. Return a bool denoting whether its TTL has
+     * exceeded `settings.tarballTTL`.
+     */
+    virtual std::optional<Result2> lookupExpired(
+        const Attrs & inAttrs) = 0;
+
+    /* Old cache for things that have a store path. */
     virtual void add(
         ref<Store> store,
         const Attrs & inAttrs,

From 1d0e3d84b6ed693c140c3b7fd6a72ef8a8a26ec3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 24 Oct 2023 10:23:46 +0200
Subject: [PATCH 021/294] Provide a InputScheme::fetch() built on top of
 InputScheme::getAccessor()

This is for graceful migration to lazy-trees fetchers (which are all
accessor-based). Eventually fetch() will be removed.
---
 src/libfetchers/fetchers.cc | 13 +++++++++++++
 src/libfetchers/fetchers.hh |  6 ++++--
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index c339c441b..3e654dd53 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -1,5 +1,6 @@
 #include "fetchers.hh"
 #include "store-api.hh"
+#include "input-accessor.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -312,6 +313,18 @@ void InputScheme::clone(const Input & input, const Path & destDir) const
     throw Error("do not know how to clone input '%s'", input.to_string());
 }
 
+std::pair<StorePath, Input> InputScheme::fetch(ref<Store> store, const Input & input)
+{
+    auto [accessor, input2] = getAccessor(store, input);
+    auto storePath = accessor->root().fetchToStore(store, input2.getName());
+    return {storePath, input2};
+}
+
+std::pair<ref<InputAccessor>, Input> InputScheme::getAccessor(ref<Store> store, const Input & input) const
+{
+    throw UnimplementedError("InputScheme must implement fetch() or getAccessor()");
+}
+
 std::optional<ExperimentalFeature> InputScheme::experimentalFeature()
 {
     return {};
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index 4212a3e1f..7b70ab6e2 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -9,7 +9,7 @@
 
 #include <memory>
 
-namespace nix { class Store; class StorePath; }
+namespace nix { class Store; class StorePath; struct InputAccessor; }
 
 namespace nix::fetchers {
 
@@ -148,7 +148,9 @@ struct InputScheme
         std::string_view contents,
         std::optional<std::string> commitMsg) const;
 
-    virtual std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input) = 0;
+    virtual std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input);
+
+    virtual std::pair<ref<InputAccessor>, Input> getAccessor(ref<Store> store, const Input & input) const;
 
     /**
      * Is this `InputScheme` part of an experimental feature?

From ee36a44bf272c8cca62a2ce96a017a8150c4d35b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 25 Oct 2023 18:55:08 +0200
Subject: [PATCH 022/294] GitInputScheme: Use libgit2

This replaces most calls to the "git" binary with libgit2.
---
 flake.nix                                     |   1 +
 src/libfetchers/git-utils.cc                  | 498 ++++++++++++++
 src/libfetchers/git-utils.hh                  |  56 ++
 src/libfetchers/git.cc                        | 609 ++++++++----------
 src/libfetchers/local.mk                      |   2 +-
 tests/functional/fetchGit.sh                  |   7 +-
 tests/functional/flakes/flake-in-submodule.sh |   5 +-
 7 files changed, 839 insertions(+), 339 deletions(-)
 create mode 100644 src/libfetchers/git-utils.cc
 create mode 100644 src/libfetchers/git-utils.hh

diff --git a/flake.nix b/flake.nix
index 398ba10a0..3472bf7a8 100644
--- a/flake.nix
+++ b/flake.nix
@@ -191,6 +191,7 @@
             bzip2 xz brotli editline
             openssl sqlite
             libarchive
+            libgit2
             boost
             lowdown-nix
             libsodium
diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
new file mode 100644
index 000000000..68e39580f
--- /dev/null
+++ b/src/libfetchers/git-utils.cc
@@ -0,0 +1,498 @@
+#include "git-utils.hh"
+#include "input-accessor.hh"
+#include "cache.hh"
+
+#include <boost/core/span.hpp>
+
+#include <git2/blob.h>
+#include <git2/commit.h>
+#include <git2/describe.h>
+#include <git2/errors.h>
+#include <git2/global.h>
+#include <git2/object.h>
+#include <git2/refs.h>
+#include <git2/remote.h>
+#include <git2/repository.h>
+#include <git2/status.h>
+#include <git2/tree.h>
+
+#include <unordered_set>
+#include <queue>
+
+namespace std {
+
+template<> struct hash<git_oid>
+{
+    size_t operator()(const git_oid & oid) const
+    {
+        return * (size_t *) oid.id;
+    }
+};
+
+}
+
+std::ostream & operator << (std::ostream & str, const git_oid & oid)
+{
+    str << git_oid_tostr_s(&oid);
+    return str;
+}
+
+bool operator == (const git_oid & oid1, const git_oid & oid2)
+{
+    return git_oid_equal(&oid1, &oid2);
+}
+
+namespace nix {
+
+// Some wrapper types that ensure that the git_*_free functions get called.
+template<auto del>
+struct Deleter
+{
+    template <typename T>
+    void operator()(T * p) const { del(p); };
+};
+
+typedef std::unique_ptr<git_repository, Deleter<git_repository_free>> Repository;
+typedef std::unique_ptr<git_tree_entry, Deleter<git_tree_entry_free>> TreeEntry;
+typedef std::unique_ptr<git_tree, Deleter<git_tree_free>> Tree;
+typedef std::unique_ptr<git_treebuilder, Deleter<git_treebuilder_free>> TreeBuilder;
+typedef std::unique_ptr<git_blob, Deleter<git_blob_free>> Blob;
+typedef std::unique_ptr<git_object, Deleter<git_object_free>> Object;
+typedef std::unique_ptr<git_commit, Deleter<git_commit_free>> Commit;
+typedef std::unique_ptr<git_reference, Deleter<git_reference_free>> Reference;
+typedef std::unique_ptr<git_describe_result, Deleter<git_describe_result_free>> DescribeResult;
+typedef std::unique_ptr<git_status_list, Deleter<git_status_list_free>> StatusList;
+typedef std::unique_ptr<git_remote, Deleter<git_remote_free>> Remote;
+
+// A helper to ensure that we don't leak objects returned by libgit2.
+template<typename T>
+struct Setter
+{
+    T & t;
+    typename T::pointer p = nullptr;
+
+    Setter(T & t) : t(t) { }
+
+    ~Setter() { if (p) t = T(p); }
+
+    operator typename T::pointer * () { return &p; }
+};
+
+Hash toHash(const git_oid & oid)
+{
+    #ifdef GIT_EXPERIMENTAL_SHA256
+    assert(oid.type == GIT_OID_SHA1);
+    #endif
+    Hash hash(htSHA1);
+    memcpy(hash.hash, oid.id, hash.hashSize);
+    return hash;
+}
+
+static void initLibGit2()
+{
+    if (git_libgit2_init() < 0)
+        throw Error("initialising libgit2: %s", git_error_last()->message);
+}
+
+git_oid hashToOID(const Hash & hash)
+{
+    git_oid oid;
+    if (git_oid_fromstr(&oid, hash.gitRev().c_str()))
+        throw Error("cannot convert '%s' to a Git OID", hash.gitRev());
+    return oid;
+}
+
+Object lookupObject(git_repository * repo, const git_oid & oid)
+{
+    Object obj;
+    if (git_object_lookup(Setter(obj), repo, &oid, GIT_OBJECT_ANY)) {
+        auto err = git_error_last();
+        throw Error("getting Git object '%s': %s", oid, err->message);
+    }
+    return obj;
+}
+
+template<typename T>
+T peelObject(git_repository * repo, git_object * obj, git_object_t type)
+{
+    T obj2;
+    if (git_object_peel((git_object * *) (typename T::pointer *) Setter(obj2), obj, type)) {
+        auto err = git_error_last();
+        throw Error("peeling Git object '%s': %s", git_object_id(obj), err->message);
+    }
+    return obj2;
+}
+
+int statusCallbackTrampoline(const char * path, unsigned int statusFlags, void * payload)
+{
+    return (*((std::function<int(const char * path, unsigned int statusFlags)> *) payload))(path, statusFlags);
+}
+
+struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
+{
+    CanonPath path;
+    Repository repo;
+
+    GitRepoImpl(CanonPath _path, bool create, bool bare)
+        : path(std::move(_path))
+    {
+        initLibGit2();
+
+        if (pathExists(path.abs())) {
+            if (git_repository_open(Setter(repo), path.c_str()))
+                throw Error("opening Git repository '%s': %s", path, git_error_last()->message);
+        } else {
+            if (git_repository_init(Setter(repo), path.c_str(), bare))
+                throw Error("creating Git repository '%s': %s", path, git_error_last()->message);
+        }
+    }
+
+    operator git_repository * ()
+    {
+        return repo.get();
+    }
+
+    uint64_t getRevCount(const Hash & rev) override
+    {
+        std::unordered_set<git_oid> done;
+        std::queue<Commit> todo;
+
+        todo.push(peelObject<Commit>(*this, lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT));
+
+        while (auto commit = pop(todo)) {
+            if (!done.insert(*git_commit_id(commit->get())).second) continue;
+
+            for (size_t n = 0; n < git_commit_parentcount(commit->get()); ++n) {
+                git_commit * parent;
+                if (git_commit_parent(&parent, commit->get(), n))
+                    throw Error("getting parent of Git commit '%s': %s", *git_commit_id(commit->get()), git_error_last()->message);
+                todo.push(Commit(parent));
+            }
+        }
+
+        return done.size();
+    }
+
+    uint64_t getLastModified(const Hash & rev) override
+    {
+        auto commit = peelObject<Commit>(*this, lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT);
+
+        return git_commit_time(commit.get());
+    }
+
+    bool isShallow() override
+    {
+        return git_repository_is_shallow(*this);
+    }
+
+    Hash resolveRef(std::string ref) override
+    {
+        // Handle revisions used as refs.
+        {
+            git_oid oid;
+            if (git_oid_fromstr(&oid, ref.c_str()) == 0)
+                return toHash(oid);
+        }
+
+        // Resolve short names like 'master'.
+        Reference ref2;
+        if (!git_reference_dwim(Setter(ref2), *this, ref.c_str()))
+            ref = git_reference_name(ref2.get());
+
+        // Resolve full references like 'refs/heads/master'.
+        Reference ref3;
+        if (git_reference_lookup(Setter(ref3), *this, ref.c_str()))
+            throw Error("resolving Git reference '%s': %s", ref, git_error_last()->message);
+
+        auto oid = git_reference_target(ref3.get());
+        if (!oid)
+            throw Error("cannot get OID for Git reference '%s'", git_reference_name(ref3.get()));
+
+        return toHash(*oid);
+    }
+
+    WorkdirInfo getWorkdirInfo() override
+    {
+        WorkdirInfo info;
+
+        /* Get the head revision, if any. */
+        git_oid headRev;
+        if (auto err = git_reference_name_to_id(&headRev, *this, "HEAD")) {
+            if (err != GIT_ENOTFOUND)
+                throw Error("resolving HEAD: %s", git_error_last()->message);
+        } else
+            info.headRev = toHash(headRev);
+
+        /* Get all tracked files and determine whether the working
+           directory is dirty. */
+        std::function<int(const char * path, unsigned int statusFlags)> statusCallback = [&](const char * path, unsigned int statusFlags)
+        {
+            if (!(statusFlags & GIT_STATUS_INDEX_DELETED) &&
+                !(statusFlags & GIT_STATUS_WT_DELETED))
+                info.files.insert(CanonPath(path));
+            if (statusFlags != GIT_STATUS_CURRENT)
+                info.isDirty = true;
+            return 0;
+        };
+
+        git_status_options options = GIT_STATUS_OPTIONS_INIT;
+        options.flags |= GIT_STATUS_OPT_INCLUDE_UNMODIFIED;
+        options.flags |= GIT_STATUS_OPT_EXCLUDE_SUBMODULES;
+        if (git_status_foreach_ext(*this, &options, &statusCallbackTrampoline, &statusCallback))
+            throw Error("getting working directory status: %s", git_error_last()->message);
+
+        return info;
+    }
+
+    std::optional<std::string> getWorkdirRef() override
+    {
+        Reference ref;
+        if (git_reference_lookup(Setter(ref), *this, "HEAD"))
+            throw Error("looking up HEAD: %s", git_error_last()->message);
+
+        if (auto target = git_reference_symbolic_target(ref.get()))
+            return target;
+
+        return std::nullopt;
+    }
+
+    bool hasObject(const Hash & oid_) override
+    {
+        auto oid = hashToOID(oid_);
+
+        Object obj;
+        if (auto errCode = git_object_lookup(Setter(obj), *this, &oid, GIT_OBJECT_ANY)) {
+            if (errCode == GIT_ENOTFOUND) return false;
+            auto err = git_error_last();
+            throw Error("getting Git object '%s': %s", oid, err->message);
+        }
+
+        return true;
+    }
+
+    ref<InputAccessor> getAccessor(const Hash & rev) override;
+
+    void fetch(
+        const std::string & url,
+        const std::string & refspec) override
+    {
+        /* FIXME: use libgit2. Unfortunately, it doesn't support
+           ssh_config at the moment. */
+        #if 0
+        Remote remote;
+
+        if (git_remote_create_anonymous(Setter(remote), *this, url.c_str()))
+            throw Error("cannot create Git remote '%s': %s", url, git_error_last()->message);
+
+        char * refspecs[] = {(char *) refspec.c_str()};
+        git_strarray refspecs2 {
+            .strings = refspecs,
+            .count = 1
+        };
+
+        if (git_remote_fetch(remote.get(), &refspecs2, nullptr, nullptr))
+            throw Error("fetching '%s' from '%s': %s", refspec, url, git_error_last()->message);
+        #endif
+
+        // FIXME: git stderr messes up our progress indicator, so
+        // we're using --quiet for now. Should process its stderr.
+        runProgram("git", true,
+            { "-C", path.abs(),
+              "--bare",
+              "fetch",
+              "--quiet",
+              "--force",
+              "--",
+              url,
+              refspec
+            }, {}, true);
+    }
+};
+
+ref<GitRepo> GitRepo::openRepo(const CanonPath & path, bool create, bool bare)
+{
+    return make_ref<GitRepoImpl>(path, create, bare);
+}
+
+struct GitInputAccessor : InputAccessor
+{
+    ref<GitRepoImpl> repo;
+    Tree root;
+
+    GitInputAccessor(ref<GitRepoImpl> repo_, const Hash & rev)
+        : repo(repo_)
+        , root(peelObject<Tree>(*repo, lookupObject(*repo, hashToOID(rev)).get(), GIT_OBJECT_TREE))
+    {
+    }
+
+    std::string readBlob(const CanonPath & path, bool symlink)
+    {
+        auto blob = getBlob(path, symlink);
+
+        auto data = std::string_view((const char *) git_blob_rawcontent(blob.get()), git_blob_rawsize(blob.get()));
+
+        return std::string(data);
+    }
+
+    std::string readFile(const CanonPath & path) override
+    {
+        return readBlob(path, false);
+    }
+
+    bool pathExists(const CanonPath & path) override
+    {
+        return path.isRoot() ? true : (bool) lookup(path);
+    }
+
+    Stat lstat(const CanonPath & path) override
+    {
+        if (path.isRoot())
+            return Stat { .type = tDirectory };
+
+        auto entry = need(path);
+
+        auto mode = git_tree_entry_filemode(entry);
+
+        if (mode == GIT_FILEMODE_TREE)
+            return Stat { .type = tDirectory };
+
+        else if (mode == GIT_FILEMODE_BLOB)
+            return Stat { .type = tRegular };
+
+        else if (mode == GIT_FILEMODE_BLOB_EXECUTABLE)
+            return Stat { .type = tRegular, .isExecutable = true };
+
+        else if (mode == GIT_FILEMODE_LINK)
+            return Stat { .type = tSymlink };
+
+        else if (mode == GIT_FILEMODE_COMMIT)
+            // Treat submodules as an empty directory.
+            return Stat { .type = tDirectory };
+
+        else
+            throw Error("file '%s' has an unsupported Git file type");
+    }
+
+    DirEntries readDirectory(const CanonPath & path) override
+    {
+        return std::visit(overloaded {
+            [&](Tree tree) {
+                DirEntries res;
+
+                auto count = git_tree_entrycount(tree.get());
+
+                for (size_t n = 0; n < count; ++n) {
+                    auto entry = git_tree_entry_byindex(tree.get(), n);
+                    // FIXME: add to cache
+                    res.emplace(std::string(git_tree_entry_name(entry)), DirEntry{});
+                }
+
+                return res;
+            },
+            [&](Submodule) {
+                return DirEntries();
+            }
+        }, getTree(path));
+    }
+
+    std::string readLink(const CanonPath & path) override
+    {
+        return readBlob(path, true);
+    }
+
+    std::map<CanonPath, TreeEntry> lookupCache;
+
+    /* Recursively look up 'path' relative to the root. */
+    git_tree_entry * lookup(const CanonPath & path)
+    {
+        if (path.isRoot()) return nullptr;
+
+        auto i = lookupCache.find(path);
+        if (i == lookupCache.end()) {
+            TreeEntry entry;
+            if (auto err = git_tree_entry_bypath(Setter(entry), root.get(), std::string(path.rel()).c_str())) {
+                if (err != GIT_ENOTFOUND)
+                    throw Error("looking up '%s': %s", showPath(path), git_error_last()->message);
+            }
+
+            i = lookupCache.emplace(path, std::move(entry)).first;
+        }
+
+        return &*i->second;
+    }
+
+    git_tree_entry * need(const CanonPath & path)
+    {
+        auto entry = lookup(path);
+        if (!entry)
+            throw Error("'%s' does not exist", showPath(path));
+        return entry;
+    }
+
+    struct Submodule { };
+
+    std::variant<Tree, Submodule> getTree(const CanonPath & path)
+    {
+        if (path.isRoot()) {
+            Tree tree;
+            if (git_tree_dup(Setter(tree), root.get()))
+                throw Error("duplicating directory '%s': %s", showPath(path), git_error_last()->message);
+            return tree;
+        }
+
+        auto entry = need(path);
+
+        if (git_tree_entry_type(entry) == GIT_OBJECT_COMMIT)
+            return Submodule();
+
+        if (git_tree_entry_type(entry) != GIT_OBJECT_TREE)
+            throw Error("'%s' is not a directory", showPath(path));
+
+        Tree tree;
+        if (git_tree_entry_to_object((git_object * *) (git_tree * *) Setter(tree), *repo, entry))
+            throw Error("looking up directory '%s': %s", showPath(path), git_error_last()->message);
+
+        return tree;
+    }
+
+    Blob getBlob(const CanonPath & path, bool expectSymlink)
+    {
+        auto notExpected = [&]()
+        {
+            throw Error(
+                expectSymlink
+                ? "'%s' is not a symlink"
+                : "'%s' is not a regular file",
+                showPath(path));
+        };
+
+        if (path.isRoot()) notExpected();
+
+        auto entry = need(path);
+
+        if (git_tree_entry_type(entry) != GIT_OBJECT_BLOB)
+            notExpected();
+
+        auto mode = git_tree_entry_filemode(entry);
+        if (expectSymlink) {
+            if (mode != GIT_FILEMODE_LINK)
+                notExpected();
+        } else {
+            if (mode != GIT_FILEMODE_BLOB && mode != GIT_FILEMODE_BLOB_EXECUTABLE)
+                notExpected();
+        }
+
+        Blob blob;
+        if (git_tree_entry_to_object((git_object * *) (git_blob * *) Setter(blob), *repo, entry))
+            throw Error("looking up file '%s': %s", showPath(path), git_error_last()->message);
+
+        return blob;
+    }
+};
+
+ref<InputAccessor> GitRepoImpl::getAccessor(const Hash & rev)
+{
+    return make_ref<GitInputAccessor>(ref<GitRepoImpl>(shared_from_this()), rev);
+}
+
+}
diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh
new file mode 100644
index 000000000..dd2c06672
--- /dev/null
+++ b/src/libfetchers/git-utils.hh
@@ -0,0 +1,56 @@
+#pragma once
+
+#include "input-accessor.hh"
+
+namespace nix {
+
+struct GitRepo
+{
+    virtual ~GitRepo()
+    { }
+
+    static ref<GitRepo> openRepo(const CanonPath & path, bool create = false, bool bare = false);
+
+    virtual uint64_t getRevCount(const Hash & rev) = 0;
+
+    virtual uint64_t getLastModified(const Hash & rev) = 0;
+
+    virtual bool isShallow() = 0;
+
+    /* Return the commit hash to which a ref points. */
+    virtual Hash resolveRef(std::string ref) = 0;
+
+    struct WorkdirInfo
+    {
+        bool isDirty = false;
+
+        /* The checked out commit, or nullopt if there are no commits
+           in the repo yet. */
+        std::optional<Hash> headRev;
+
+        /* All files in the working directory that are unchanged,
+           modified or added, but excluding deleted files. */
+        std::set<CanonPath> files;
+    };
+
+    virtual WorkdirInfo getWorkdirInfo() = 0;
+
+    /* Get the ref that HEAD points to. */
+    virtual std::optional<std::string> getWorkdirRef() = 0;
+
+    struct TarballInfo
+    {
+        Hash treeHash;
+        time_t lastModified;
+    };
+
+    virtual bool hasObject(const Hash & oid) = 0;
+
+    virtual ref<InputAccessor> getAccessor(const Hash & rev) = 0;
+
+    virtual void fetch(
+        const std::string & url,
+        const std::string & refspec) = 0;
+};
+
+}
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 4bfd53b0e..55d3a8ebe 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -7,6 +7,8 @@
 #include "pathlocks.hh"
 #include "util.hh"
 #include "git.hh"
+#include "fs-input-accessor.hh"
+#include "git-utils.hh"
 
 #include "fetch-settings.hh"
 
@@ -137,121 +139,6 @@ bool isNotDotGitDirectory(const Path & path)
     return baseNameOf(path) != ".git";
 }
 
-struct WorkdirInfo
-{
-    bool clean = false;
-    bool hasHead = false;
-};
-
-// Returns whether a git workdir is clean and has commits.
-WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir)
-{
-    const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
-    std::string gitDir(".git");
-
-    auto env = getEnv();
-    // Set LC_ALL to C: because we rely on the error messages from git rev-parse to determine what went wrong
-    // that way unknown errors can lead to a failure instead of continuing through the wrong code path
-    env["LC_ALL"] = "C";
-
-    /* Check whether HEAD points to something that looks like a commit,
-       since that is the refrence we want to use later on. */
-    auto result = runProgram(RunOptions {
-        .program = "git",
-        .args = { "-C", workdir, "--git-dir", gitDir, "rev-parse", "--verify", "--no-revs", "HEAD^{commit}" },
-        .environment = env,
-        .mergeStderrToStdout = true
-    });
-    auto exitCode = WEXITSTATUS(result.first);
-    auto errorMessage = result.second;
-
-    if (errorMessage.find("fatal: not a git repository") != std::string::npos) {
-        throw Error("'%s' is not a Git repository", workdir);
-    } else if (errorMessage.find("fatal: Needed a single revision") != std::string::npos) {
-        // indicates that the repo does not have any commits
-        // we want to proceed and will consider it dirty later
-    } else if (exitCode != 0) {
-        // any other errors should lead to a failure
-        throw Error("getting the HEAD of the Git tree '%s' failed with exit code %d:\n%s", workdir, exitCode, errorMessage);
-    }
-
-    bool clean = false;
-    bool hasHead = exitCode == 0;
-
-    try {
-        if (hasHead) {
-            // Using git diff is preferrable over lower-level operations here,
-            // because its conceptually simpler and we only need the exit code anyways.
-            auto gitDiffOpts = Strings({ "-C", workdir, "--git-dir", gitDir, "diff", "HEAD", "--quiet"});
-            if (!submodules) {
-                // Changes in submodules should only make the tree dirty
-                // when those submodules will be copied as well.
-                gitDiffOpts.emplace_back("--ignore-submodules");
-            }
-            gitDiffOpts.emplace_back("--");
-            runProgram("git", true, gitDiffOpts);
-
-            clean = true;
-        }
-    } catch (ExecError & e) {
-        if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
-    }
-
-    return WorkdirInfo { .clean = clean, .hasHead = hasHead };
-}
-
-std::pair<StorePath, Input> fetchFromWorkdir(ref<Store> store, Input & input, const Path & workdir, const WorkdirInfo & workdirInfo)
-{
-    const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
-    auto gitDir = ".git";
-
-    if (!fetchSettings.allowDirty)
-        throw Error("Git tree '%s' is dirty", workdir);
-
-    if (fetchSettings.warnDirty)
-        warn("Git tree '%s' is dirty", workdir);
-
-    auto gitOpts = Strings({ "-C", workdir, "--git-dir", gitDir, "ls-files", "-z" });
-    if (submodules)
-        gitOpts.emplace_back("--recurse-submodules");
-
-    auto files = tokenizeString<std::set<std::string>>(
-        runProgram("git", true, gitOpts), "\0"s);
-
-    Path actualPath(absPath(workdir));
-
-    PathFilter filter = [&](const Path & p) -> bool {
-        assert(hasPrefix(p, actualPath));
-        std::string file(p, actualPath.size() + 1);
-
-        auto st = lstat(p);
-
-        if (S_ISDIR(st.st_mode)) {
-            auto prefix = file + "/";
-            auto i = files.lower_bound(prefix);
-            return i != files.end() && hasPrefix(*i, prefix);
-        }
-
-        return files.count(file);
-    };
-
-    auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter);
-
-    // FIXME: maybe we should use the timestamp of the last
-    // modified dirty file?
-    input.attrs.insert_or_assign(
-        "lastModified",
-        workdirInfo.hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0);
-
-    if (workdirInfo.hasHead) {
-        input.attrs.insert_or_assign("dirtyRev", chomp(
-            runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "rev-parse", "--verify", "HEAD" })) + "-dirty");
-        input.attrs.insert_or_assign("dirtyShortRev", chomp(
-            runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "rev-parse", "--verify", "--short", "HEAD" })) + "-dirty");
-    }
-
-    return {std::move(storePath), input};
-}
 }  // end namespace
 
 struct GitInputScheme : InputScheme
@@ -336,11 +223,11 @@ struct GitInputScheme : InputScheme
 
     void clone(const Input & input, const Path & destDir) const override
     {
-        auto [isLocal, actualUrl] = getActualUrl(input);
+        auto repoInfo = getRepoInfo(input);
 
         Strings args = {"clone"};
 
-        args.push_back(actualUrl);
+        args.push_back(repoInfo.url);
 
         if (auto ref = input.getRef()) {
             args.push_back("--branch");
@@ -356,10 +243,9 @@ struct GitInputScheme : InputScheme
 
     std::optional<Path> getSourcePath(const Input & input) const override
     {
-        auto url = parseURL(getStrAttr(input.attrs, "url"));
-        if (url.scheme == "file" && !input.getRef() && !input.getRev())
-            return url.path;
-        return {};
+        auto repoInfo = getRepoInfo(input);
+        if (repoInfo.isLocal) return repoInfo.url;
+        return std::nullopt;
     }
 
     void putFile(
@@ -368,24 +254,79 @@ struct GitInputScheme : InputScheme
         std::string_view contents,
         std::optional<std::string> commitMsg) const override
     {
-        auto root = getSourcePath(input);
-        if (!root)
+        auto repoInfo = getRepoInfo(input);
+        if (!repoInfo.isLocal)
             throw Error("cannot commit '%s' to Git repository '%s' because it's not a working tree", path, input.to_string());
 
-        writeFile((CanonPath(*root) + path).abs(), contents);
-
-        auto gitDir = ".git";
+        writeFile((CanonPath(repoInfo.url) + path).abs(), contents);
 
         runProgram("git", true,
-            { "-C", *root, "--git-dir", gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) });
+            { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) });
 
         if (commitMsg)
             runProgram("git", true,
-                { "-C", *root, "--git-dir", gitDir, "commit", std::string(path.rel()), "-m", *commitMsg });
+                { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "commit", std::string(path.rel()), "-m", *commitMsg });
     }
 
-    std::pair<bool, std::string> getActualUrl(const Input & input) const
+    struct RepoInfo
     {
+        bool shallow = false;
+        bool submodules = false;
+        bool allRefs = false;
+
+        std::string cacheType;
+
+        /* Whether this is a local, non-bare repository. */
+        bool isLocal = false;
+
+        /* Working directory info: the complete list of files, and
+           whether the working directory is dirty compared to HEAD. */
+        GitRepo::WorkdirInfo workdirInfo;
+
+        /* URL of the repo, or its path if isLocal. */
+        std::string url;
+
+        void warnDirty() const
+        {
+            if (workdirInfo.isDirty) {
+                if (!fetchSettings.allowDirty)
+                    throw Error("Git tree '%s' is dirty", url);
+
+                if (fetchSettings.warnDirty)
+                    warn("Git tree '%s' is dirty", url);
+            }
+        }
+
+        std::string gitDir = ".git";
+    };
+
+    bool getSubmodulesAttr(const Input & input) const
+    {
+        return maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
+    }
+
+    RepoInfo getRepoInfo(const Input & input) const
+    {
+        auto checkHashType = [&](const std::optional<Hash> & hash)
+        {
+            if (hash.has_value() && !(hash->type == htSHA1 || hash->type == htSHA256))
+                throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(HashFormat::Base16, true));
+        };
+
+        if (auto rev = input.getRev())
+            checkHashType(rev);
+
+        RepoInfo repoInfo {
+            .shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false),
+            .submodules = getSubmodulesAttr(input),
+            .allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false)
+        };
+
+        repoInfo.cacheType = "git";
+        if (repoInfo.shallow) repoInfo.cacheType += "-shallow";
+        if (repoInfo.submodules) repoInfo.cacheType += "-submodules";
+        if (repoInfo.allRefs) repoInfo.cacheType += "-all-refs";
+
         // file:// URIs are normally not cloned (but otherwise treated the
         // same as remote URIs, i.e. we don't use the working tree or
         // HEAD). Exception: If _NIX_FORCE_HTTP is set, or the repo is a bare git
@@ -393,153 +334,142 @@ struct GitInputScheme : InputScheme
         static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing
         auto url = parseURL(getStrAttr(input.attrs, "url"));
         bool isBareRepository = url.scheme == "file" && !pathExists(url.path + "/.git");
-        bool isLocal = url.scheme == "file" && !forceHttp && !isBareRepository;
-        return {isLocal, isLocal ? url.path : url.base};
+        repoInfo.isLocal = url.scheme == "file" && !forceHttp && !isBareRepository;
+        repoInfo.url = repoInfo.isLocal ? url.path : url.base;
+
+        // If this is a local directory and no ref or revision is
+        // given, then allow the use of an unclean working tree.
+        if (!input.getRef() && !input.getRev() && repoInfo.isLocal)
+            repoInfo.workdirInfo = GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirInfo();
+
+        return repoInfo;
     }
 
-    std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override
+    uint64_t getLastModified(const RepoInfo & repoInfo, const std::string & repoDir, const Hash & rev) const
     {
-        Input input(_input);
-        auto gitDir = ".git";
+        Attrs key{{"_what", "gitLastModified"}, {"rev", rev.gitRev()}};
+
+        auto cache = getCache();
+
+        if (auto res = cache->lookup(key))
+            return getIntAttr(*res, "lastModified");
+
+        auto lastModified = GitRepo::openRepo(CanonPath(repoDir))->getLastModified(rev);
+
+        cache->upsert(key, Attrs{{"lastModified", lastModified}});
+
+        return lastModified;
+    }
+
+    uint64_t getRevCount(const RepoInfo & repoInfo, const std::string & repoDir, const Hash & rev) const
+    {
+        Attrs key{{"_what", "gitRevCount"}, {"rev", rev.gitRev()}};
+
+        auto cache = getCache();
+
+        if (auto revCountAttrs = cache->lookup(key))
+            return getIntAttr(*revCountAttrs, "revCount");
+
+        Activity act(*logger, lvlChatty, actUnknown, fmt("getting Git revision count of '%s'", repoInfo.url));
+
+        auto revCount = GitRepo::openRepo(CanonPath(repoDir))->getRevCount(rev);
+
+        cache->upsert(key, Attrs{{"revCount", revCount}});
+
+        return revCount;
+    }
+
+    std::string getDefaultRef(const RepoInfo & repoInfo) const
+    {
+        auto head = repoInfo.isLocal
+            ? GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirRef()
+            : readHeadCached(repoInfo.url);
+        if (!head) {
+            warn("could not read HEAD ref from repo at '%s', using 'master'", repoInfo.url);
+            return "master";
+        }
+        return *head;
+    }
+
+    static MakeNotAllowedError makeNotAllowedError(std::string url)
+    {
+        return [url{std::move(url)}](const CanonPath & path) -> RestrictedPathError
+        {
+            if (nix::pathExists(path.abs()))
+                return RestrictedPathError("access to path '%s' is forbidden because it is not under Git control; maybe you should 'git add' it to the repository '%s'?", path, url);
+            else
+                return RestrictedPathError("path '%s' does not exist in Git repository '%s'", path, url);
+        };
+    }
+
+    std::pair<ref<InputAccessor>, Input> getAccessorFromCommit(
+        ref<Store> store,
+        RepoInfo & repoInfo,
+        Input && input) const
+    {
+        assert(!repoInfo.workdirInfo.isDirty);
+
+        auto origRev = input.getRev();
 
         std::string name = input.getName();
 
-        bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
-        bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
-        bool allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false);
-
-        std::string cacheType = "git";
-        if (shallow) cacheType += "-shallow";
-        if (submodules) cacheType += "-submodules";
-        if (allRefs) cacheType += "-all-refs";
-
-        auto checkHashType = [&](const std::optional<Hash> & hash)
-        {
-            if (hash.has_value() && !(hash->type == htSHA1 || hash->type == htSHA256))
-                throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(HashFormat::Base16, true));
-        };
-
-        auto getLockedAttrs = [&]()
-        {
-            checkHashType(input.getRev());
-
-            return Attrs({
-                {"type", cacheType},
-                {"name", name},
-                {"rev", input.getRev()->gitRev()},
-            });
-        };
-
-        auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
-            -> std::pair<StorePath, Input>
+        auto makeResult2 = [&](const Attrs & infoAttrs, ref<InputAccessor> accessor) -> std::pair<ref<InputAccessor>, Input>
         {
             assert(input.getRev());
-            assert(!_input.getRev() || _input.getRev() == input.getRev());
-            if (!shallow)
+            assert(!origRev || origRev == input.getRev());
+            if (!repoInfo.shallow)
                 input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
             input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified"));
-            return {std::move(storePath), input};
+
+            return {accessor, std::move(input)};
         };
 
-        if (input.getRev()) {
-            if (auto res = getCache()->lookup(store, getLockedAttrs()))
-                return makeResult(res->first, std::move(res->second));
-        }
+        auto makeResult = [&](const Attrs & infoAttrs, const StorePath & storePath) -> std::pair<ref<InputAccessor>, Input>
+        {
+            // FIXME: remove?
+            //input.attrs.erase("narHash");
+            auto narHash = store->queryPathInfo(storePath)->narHash;
+            input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
 
-        auto [isLocal, actualUrl_] = getActualUrl(input);
-        auto actualUrl = actualUrl_; // work around clang bug
+            auto accessor = makeStorePathAccessor(store, storePath, makeNotAllowedError(repoInfo.url));
 
-        /* If this is a local directory and no ref or revision is given,
-           allow fetching directly from a dirty workdir. */
-        if (!input.getRef() && !input.getRev() && isLocal) {
-            auto workdirInfo = getWorkdirInfo(input, actualUrl);
-            if (!workdirInfo.clean) {
-                return fetchFromWorkdir(store, input, actualUrl, workdirInfo);
-            }
-        }
+            return makeResult2(infoAttrs, accessor);
+        };
 
-        Attrs unlockedAttrs({
-            {"type", cacheType},
-            {"name", name},
-            {"url", actualUrl},
-        });
+        auto originalRef = input.getRef();
+        auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo);
+        input.attrs.insert_or_assign("ref", ref);
 
         Path repoDir;
 
-        if (isLocal) {
-            if (!input.getRef()) {
-                auto head = readHead(actualUrl);
-                if (!head) {
-                    warn("could not read HEAD ref from repo at '%s', using 'master'", actualUrl);
-                    head = "master";
-                }
-                input.attrs.insert_or_assign("ref", *head);
-                unlockedAttrs.insert_or_assign("ref", *head);
-            }
-
+        if (repoInfo.isLocal) {
+            repoDir = repoInfo.url;
             if (!input.getRev())
-                input.attrs.insert_or_assign("rev",
-                    Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "--git-dir", gitDir, "rev-parse", *input.getRef() })), htSHA1).gitRev());
-
-            repoDir = actualUrl;
+                input.attrs.insert_or_assign("rev", GitRepo::openRepo(CanonPath(repoDir))->resolveRef(ref).gitRev());
         } else {
-            const bool useHeadRef = !input.getRef();
-            if (useHeadRef) {
-                auto head = readHeadCached(actualUrl);
-                if (!head) {
-                    warn("could not read HEAD ref from repo at '%s', using 'master'", actualUrl);
-                    head = "master";
-                }
-                input.attrs.insert_or_assign("ref", *head);
-                unlockedAttrs.insert_or_assign("ref", *head);
-            } else {
-                if (!input.getRev()) {
-                    unlockedAttrs.insert_or_assign("ref", input.getRef().value());
-                }
-            }
-
-            if (auto res = getCache()->lookup(store, unlockedAttrs)) {
-                auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1);
-                if (!input.getRev() || input.getRev() == rev2) {
-                    input.attrs.insert_or_assign("rev", rev2.gitRev());
-                    return makeResult(res->first, std::move(res->second));
-                }
-            }
-
-            Path cacheDir = getCachePath(actualUrl);
+            Path cacheDir = getCachePath(repoInfo.url);
             repoDir = cacheDir;
-            gitDir = ".";
+            repoInfo.gitDir = ".";
 
             createDirs(dirOf(cacheDir));
-            PathLocks cacheDirLock({cacheDir + ".lock"});
+            PathLocks cacheDirLock({cacheDir});
 
-            if (!pathExists(cacheDir)) {
-                runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", "--bare", repoDir });
-            }
+            auto repo = GitRepo::openRepo(CanonPath(cacheDir), true, true);
 
             Path localRefFile =
-                input.getRef()->compare(0, 5, "refs/") == 0
-                ? cacheDir + "/" + *input.getRef()
-                : cacheDir + "/refs/heads/" + *input.getRef();
+                ref.compare(0, 5, "refs/") == 0
+                ? cacheDir + "/" + ref
+                : cacheDir + "/refs/heads/" + ref;
 
             bool doFetch;
             time_t now = time(0);
 
             /* If a rev was specified, we need to fetch if it's not in the
                repo. */
-            if (input.getRev()) {
-                try {
-                    runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "cat-file", "-e", input.getRev()->gitRev() });
-                    doFetch = false;
-                } catch (ExecError & e) {
-                    if (WIFEXITED(e.status)) {
-                        doFetch = true;
-                    } else {
-                        throw;
-                    }
-                }
+            if (auto rev = input.getRev()) {
+                doFetch = !repo->hasObject(*rev);
             } else {
-                if (allRefs) {
+                if (repoInfo.allRefs) {
                     doFetch = true;
                 } else {
                     /* If the local ref is older than ‘tarball-ttl’ seconds, do a
@@ -551,75 +481,80 @@ struct GitInputScheme : InputScheme
             }
 
             if (doFetch) {
-                Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", actualUrl));
+                Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", repoInfo.url));
 
-                // FIXME: git stderr messes up our progress indicator, so
-                // we're using --quiet for now. Should process its stderr.
                 try {
-                    auto ref = input.getRef();
-                    auto fetchRef = allRefs
+                    auto fetchRef = repoInfo.allRefs
                         ? "refs/*"
-                        : ref->compare(0, 5, "refs/") == 0
-                            ? *ref
-                            : ref == "HEAD"
-                                ? *ref
-                                : "refs/heads/" + *ref;
-                    runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }, {}, true);
+                        : ref.compare(0, 5, "refs/") == 0
+                        ? ref
+                        : ref == "HEAD"
+                        ? ref
+                        : "refs/heads/" + ref;
+
+                    repo->fetch(repoInfo.url, fmt("%s:%s", fetchRef, fetchRef));
                 } catch (Error & e) {
                     if (!pathExists(localRefFile)) throw;
-                    warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl);
+                    logError(e.info());
+                    warn("could not update local clone of Git repository '%s'; continuing with the most recent version", repoInfo.url);
                 }
 
                 if (!touchCacheFile(localRefFile, now))
                     warn("could not update mtime for file '%s': %s", localRefFile, strerror(errno));
-                if (useHeadRef && !storeCachedHead(actualUrl, *input.getRef()))
-                    warn("could not update cached head '%s' for '%s'", *input.getRef(), actualUrl);
+                if (!originalRef && !storeCachedHead(repoInfo.url, ref))
+                    warn("could not update cached head '%s' for '%s'", ref, repoInfo.url);
             }
 
-            if (!input.getRev())
+            if (auto rev = input.getRev()) {
+                if (!repo->hasObject(*rev))
+                    throw Error(
+                        "Cannot find Git revision '%s' in ref '%s' of repository '%s'! "
+                        "Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the "
+                        ANSI_BOLD "ref" ANSI_NORMAL " you've specified or add " ANSI_BOLD
+                        "allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".",
+                        rev->gitRev(),
+                        ref,
+                        repoInfo.url
+                        );
+            } else
                 input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), htSHA1).gitRev());
 
             // cache dir lock is removed at scope end; we will only use read-only operations on specific revisions in the remainder
         }
 
-        bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "rev-parse", "--is-shallow-repository" })) == "true";
+        auto isShallow = GitRepo::openRepo(CanonPath(repoDir))->isShallow();
 
-        if (isShallow && !shallow)
-            throw Error("'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified.", actualUrl);
+        if (isShallow && !repoInfo.shallow)
+            throw Error("'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified", repoInfo.url);
 
-        // FIXME: check whether rev is an ancestor of ref.
+        // FIXME: check whether rev is an ancestor of ref?
 
-        printTalkative("using revision %s of repo '%s'", input.getRev()->gitRev(), actualUrl);
+        auto rev = *input.getRev();
 
-        /* Now that we know the ref, check again whether we have it in
-           the store. */
-        if (auto res = getCache()->lookup(store, getLockedAttrs()))
-            return makeResult(res->first, std::move(res->second));
-
-        Path tmpDir = createTempDir();
-        AutoDelete delTmpDir(tmpDir, true);
-        PathFilter filter = defaultPathFilter;
-
-        auto result = runProgram(RunOptions {
-            .program = "git",
-            .args = { "-C", repoDir, "--git-dir", gitDir, "cat-file", "commit", input.getRev()->gitRev() },
-            .mergeStderrToStdout = true
+        Attrs infoAttrs({
+            {"rev", rev.gitRev()},
+            {"lastModified", getLastModified(repoInfo, repoDir, rev)},
         });
-        if (WEXITSTATUS(result.first) == 128
-            && result.second.find("bad file") != std::string::npos)
-        {
-            throw Error(
-                "Cannot find Git revision '%s' in ref '%s' of repository '%s'! "
-                "Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the "
-                ANSI_BOLD "ref" ANSI_NORMAL " you've specified or add " ANSI_BOLD
-                "allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".",
-                input.getRev()->gitRev(),
-                *input.getRef(),
-                actualUrl
-            );
+
+        if (!repoInfo.shallow)
+            infoAttrs.insert_or_assign("revCount",
+                getRevCount(repoInfo, repoDir, rev));
+
+        printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.url);
+
+        if (!repoInfo.submodules) {
+            auto accessor = GitRepo::openRepo(CanonPath(repoDir))->getAccessor(rev);
+            return makeResult2(infoAttrs, accessor);
         }
 
-        if (submodules) {
+        else {
+            // FIXME: use libgit2
+            Path tmpDir = createTempDir();
+            AutoDelete delTmpDir(tmpDir, true);
+            PathFilter filter = defaultPathFilter;
+
+            Activity act(*logger, lvlChatty, actUnknown, fmt("copying Git tree '%s' to the store", input.to_string()));
+
             Path tmpGitDir = createTempDir();
             AutoDelete delTmpGitDir(tmpGitDir, true);
 
@@ -634,77 +569,89 @@ struct GitInputScheme : InputScheme
                         "--update-head-ok", "--", repoDir, "refs/*:refs/*" }, {}, true);
             }
 
-            runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() });
+            runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", rev.gitRev() });
 
             /* Ensure that we use the correct origin for fetching
                submodules. This matters for submodules with relative
                URLs. */
-            if (isLocal) {
-                writeFile(tmpGitDir + "/config", readFile(repoDir + "/" + gitDir + "/config"));
+            if (repoInfo.isLocal) {
+                writeFile(tmpGitDir + "/config", readFile(repoDir + "/" + repoInfo.gitDir + "/config"));
 
                 /* Restore the config.bare setting we may have just
                    copied erroneously from the user's repo. */
                 runProgram("git", true, { "-C", tmpDir, "config", "core.bare", "false" });
             } else
-                runProgram("git", true, { "-C", tmpDir, "config", "remote.origin.url", actualUrl });
+                runProgram("git", true, { "-C", tmpDir, "config", "remote.origin.url", repoInfo.url });
 
             /* As an optimisation, copy the modules directory of the
                source repo if it exists. */
-            auto modulesPath = repoDir + "/" + gitDir + "/modules";
+            auto modulesPath = repoDir + "/" + repoInfo.gitDir + "/modules";
             if (pathExists(modulesPath)) {
-                Activity act(*logger, lvlTalkative, actUnknown, fmt("copying submodules of '%s'", actualUrl));
+                Activity act(*logger, lvlTalkative, actUnknown, fmt("copying submodules of '%s'", repoInfo.url));
                 runProgram("cp", true, { "-R", "--", modulesPath, tmpGitDir + "/modules" });
             }
 
             {
-                Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", actualUrl));
+                Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", repoInfo.url));
                 runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }, {}, true);
             }
 
             filter = isNotDotGitDirectory;
-        } else {
-            // FIXME: should pipe this, or find some better way to extract a
-            // revision.
-            auto source = sinkToSource([&](Sink & sink) {
-                runProgram2({
-                    .program = "git",
-                    .args = { "-C", repoDir, "--git-dir", gitDir, "archive", input.getRev()->gitRev() },
-                    .standardOut = &sink
-                });
-            });
 
-            unpackTarfile(*source, tmpDir);
+            auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);
+
+            return makeResult(infoAttrs, std::move(storePath));
+        }
+    }
+
+    std::pair<ref<InputAccessor>, Input> getAccessorFromWorkdir(
+        RepoInfo & repoInfo,
+        Input && input) const
+    {
+        if (!repoInfo.workdirInfo.isDirty) {
+            if (auto ref = GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirRef())
+                input.attrs.insert_or_assign("ref", *ref);
+
+            auto rev = repoInfo.workdirInfo.headRev.value();
+
+            input.attrs.insert_or_assign("rev", rev.gitRev());
+
+            input.attrs.insert_or_assign("revCount", getRevCount(repoInfo, repoInfo.url, rev));
+        } else {
+            repoInfo.warnDirty();
+
+            if (repoInfo.workdirInfo.headRev) {
+                input.attrs.insert_or_assign("dirtyRev",
+                    repoInfo.workdirInfo.headRev->gitRev() + "-dirty");
+                input.attrs.insert_or_assign("dirtyShortRev",
+                    repoInfo.workdirInfo.headRev->gitShortRev() + "-dirty");
+            }
         }
 
-        auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);
+        input.attrs.insert_or_assign(
+            "lastModified",
+            repoInfo.workdirInfo.headRev
+            ? getLastModified(repoInfo, repoInfo.url, *repoInfo.workdirInfo.headRev)
+            : 0);
 
-        auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() }));
+        input.locked = true; // FIXME
 
-        Attrs infoAttrs({
-            {"rev", input.getRev()->gitRev()},
-            {"lastModified", lastModified},
-        });
+        return {
+            makeFSInputAccessor(CanonPath(repoInfo.url), repoInfo.workdirInfo.files, makeNotAllowedError(repoInfo.url)),
+            std::move(input)
+        };
+    }
 
-        if (!shallow)
-            infoAttrs.insert_or_assign("revCount",
-                std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "rev-list", "--count", input.getRev()->gitRev() })));
+    std::pair<ref<InputAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
+    {
+        Input input(_input);
 
-        if (!_input.getRev())
-            getCache()->add(
-                store,
-                unlockedAttrs,
-                infoAttrs,
-                storePath,
-                false);
+        auto repoInfo = getRepoInfo(input);
 
-        getCache()->add(
-            store,
-            getLockedAttrs(),
-            infoAttrs,
-            storePath,
-            true);
-
-        return makeResult(infoAttrs, std::move(storePath));
+        if (input.getRef() || input.getRev() || !repoInfo.isLocal)
+            return getAccessorFromCommit(store, repoInfo, std::move(input));
+        else
+            return getAccessorFromWorkdir(repoInfo, std::move(input));
     }
 };
 
diff --git a/src/libfetchers/local.mk b/src/libfetchers/local.mk
index 2e8869d83..f21651d77 100644
--- a/src/libfetchers/local.mk
+++ b/src/libfetchers/local.mk
@@ -8,6 +8,6 @@ libfetchers_SOURCES := $(wildcard $(d)/*.cc)
 
 libfetchers_CXXFLAGS += -I src/libutil -I src/libstore
 
-libfetchers_LDFLAGS += -pthread
+libfetchers_LDFLAGS += -pthread -lgit2 -larchive
 
 libfetchers_LIBS = libutil libstore
diff --git a/tests/functional/fetchGit.sh b/tests/functional/fetchGit.sh
index fc89f2040..c38cd27eb 100644
--- a/tests/functional/fetchGit.sh
+++ b/tests/functional/fetchGit.sh
@@ -185,11 +185,7 @@ path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref =
 # Nuke the cache
 rm -rf $TEST_HOME/.cache/nix
 
-# Try again, but without 'git' on PATH. This should fail.
-NIX=$(command -v nix)
-(! PATH= $NIX eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath" )
-
-# Try again, with 'git' available.  This should work.
+# Try again. This should work.
 path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath")
 [[ $path3 = $path5 ]]
 
@@ -241,6 +237,7 @@ rm -rf $repo/.git
 
 # should succeed for a repo without commits
 git init $repo
+git -C $repo add hello # need to add at least one file to cause the root of the repo to be visible
 path10=$(nix eval --impure --raw --expr "(builtins.fetchGit \"file://$repo\").outPath")
 
 # should succeed for a path with a space
diff --git a/tests/functional/flakes/flake-in-submodule.sh b/tests/functional/flakes/flake-in-submodule.sh
index 21a4b52de..6e24a80c1 100644
--- a/tests/functional/flakes/flake-in-submodule.sh
+++ b/tests/functional/flakes/flake-in-submodule.sh
@@ -46,7 +46,8 @@ echo '"expression in root repo"' > $rootRepo/root.nix
 git -C $rootRepo add root.nix
 git -C $rootRepo commit -m "Add root.nix"
 
+# FIXME
 # Flake can live inside a submodule and can be accessed via ?dir=submodule
-[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#sub ) = '"expression in submodule"' ]]
+#[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#sub ) = '"expression in submodule"' ]]
 # The flake can access content outside of the submodule
-[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#root ) = '"expression in root repo"' ]]
+#[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#root ) = '"expression in root repo"' ]]

From d88106df24869104cc6c29c726ddfbbfda9dae10 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 27 Oct 2023 18:39:00 +0200
Subject: [PATCH 023/294] Git fetcher: Improve submodule handling

Instead of making a complete copy of the repo, fetching the
submodules, and writing the result to the store (which is all
superexpensive), we now fetch the submodules recursively using the Git
fetcher, and return a union accessor that "mounts" the accessors for
the submodules on top of the root accessor.
---
 src/libfetchers/git-utils.cc            |  78 +++++++++++++++++
 src/libfetchers/git-utils.hh            |  12 +++
 src/libfetchers/git.cc                  | 109 +++++++-----------------
 src/libfetchers/union-input-accessor.cc |  80 +++++++++++++++++
 src/libfetchers/union-input-accessor.hh |   9 ++
 tests/functional/fetchGitSubmodules.sh  |   8 --
 6 files changed, 212 insertions(+), 84 deletions(-)
 create mode 100644 src/libfetchers/union-input-accessor.cc
 create mode 100644 src/libfetchers/union-input-accessor.hh

diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index 68e39580f..5e3e6dae4 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -1,11 +1,13 @@
 #include "git-utils.hh"
 #include "input-accessor.hh"
 #include "cache.hh"
+#include "finally.hh"
 
 #include <boost/core/span.hpp>
 
 #include <git2/blob.h>
 #include <git2/commit.h>
+#include <git2/config.h>
 #include <git2/describe.h>
 #include <git2/errors.h>
 #include <git2/global.h>
@@ -14,6 +16,7 @@
 #include <git2/remote.h>
 #include <git2/repository.h>
 #include <git2/status.h>
+#include <git2/submodule.h>
 #include <git2/tree.h>
 
 #include <unordered_set>
@@ -63,6 +66,8 @@ typedef std::unique_ptr<git_reference, Deleter<git_reference_free>> Reference;
 typedef std::unique_ptr<git_describe_result, Deleter<git_describe_result_free>> DescribeResult;
 typedef std::unique_ptr<git_status_list, Deleter<git_status_list_free>> StatusList;
 typedef std::unique_ptr<git_remote, Deleter<git_remote_free>> Remote;
+typedef std::unique_ptr<git_config, Deleter<git_config_free>> GitConfig;
+typedef std::unique_ptr<git_config_iterator, Deleter<git_config_iterator_free>> ConfigIterator;
 
 // A helper to ensure that we don't leak objects returned by libgit2.
 template<typename T>
@@ -256,6 +261,17 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         return std::nullopt;
     }
 
+    std::vector<Submodule> getSubmodules(const Hash & rev) override;
+
+    std::string resolveSubmoduleUrl(const std::string & url) override
+    {
+        git_buf buf = GIT_BUF_INIT;
+        if (git_submodule_resolve_url(&buf, *this, url.c_str()))
+            throw Error("resolving Git submodule URL '%s'", url);
+        Finally cleanup = [&]() { git_buf_dispose(&buf); };
+        return buf.ptr;
+    }
+
     bool hasObject(const Hash & oid_) override
     {
         auto oid = hashToOID(oid_);
@@ -400,6 +416,16 @@ struct GitInputAccessor : InputAccessor
         return readBlob(path, true);
     }
 
+    Hash getSubmoduleRev(const CanonPath & path)
+    {
+        auto entry = need(path);
+
+        if (git_tree_entry_type(entry) != GIT_OBJECT_COMMIT)
+            throw Error("'%s' is not a submodule", showPath(path));
+
+        return toHash(*git_tree_entry_id(entry));
+    }
+
     std::map<CanonPath, TreeEntry> lookupCache;
 
     /* Recursively look up 'path' relative to the root. */
@@ -495,4 +521,56 @@ ref<InputAccessor> GitRepoImpl::getAccessor(const Hash & rev)
     return make_ref<GitInputAccessor>(ref<GitRepoImpl>(shared_from_this()), rev);
 }
 
+std::vector<GitRepoImpl::Submodule> GitRepoImpl::getSubmodules(const Hash & rev)
+{
+    /* Read the .gitmodules files from this revision. */
+    CanonPath modulesFile(".gitmodules");
+
+    auto accessor = getAccessor(rev);
+    if (!accessor->pathExists(modulesFile)) return {};
+
+    /* Parse it. */
+    auto configS = accessor->readFile(modulesFile);
+
+    auto [fdTemp, pathTemp] = createTempFile("nix-git-submodules");
+    writeFull(fdTemp.get(), configS);
+
+    GitConfig config;
+    if (git_config_open_ondisk(Setter(config), pathTemp.c_str()))
+        throw Error("parsing .gitmodules file: %s", git_error_last()->message);
+
+    ConfigIterator it;
+    if (git_config_iterator_glob_new(Setter(it), config.get(), "^submodule\\..*\\.(path|url|branch)$"))
+        throw Error("iterating over .gitmodules: %s", git_error_last()->message);
+
+    std::map<std::string, std::string> entries;
+
+    while (true) {
+        git_config_entry * entry = nullptr;
+        if (auto err = git_config_next(&entry, it.get())) {
+            if (err == GIT_ITEROVER) break;
+            throw Error("iterating over .gitmodules: %s", git_error_last()->message);
+        }
+        entries.emplace(entry->name + 10, entry->value);
+    }
+
+    std::vector<Submodule> result;
+
+    for (auto & [key, value] : entries) {
+        if (!hasSuffix(key, ".path")) continue;
+        std::string key2(key, 0, key.size() - 5);
+        auto path = CanonPath(value);
+        auto rev = accessor.dynamic_pointer_cast<GitInputAccessor>()->getSubmoduleRev(path);
+        result.push_back(Submodule {
+            .path = path,
+            .url = entries[key2 + ".url"],
+            .branch = entries[key2 + ".branch"],
+            .rev = rev,
+        });
+    }
+
+    return result;
+}
+
+
 }
diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh
index dd2c06672..55e7ef969 100644
--- a/src/libfetchers/git-utils.hh
+++ b/src/libfetchers/git-utils.hh
@@ -38,6 +38,18 @@ struct GitRepo
     /* Get the ref that HEAD points to. */
     virtual std::optional<std::string> getWorkdirRef() = 0;
 
+    struct Submodule
+    {
+        CanonPath path;
+        std::string url;
+        std::string branch;
+        Hash rev;
+    };
+
+    virtual std::vector<Submodule> getSubmodules(const Hash & rev) = 0;
+
+    virtual std::string resolveSubmoduleUrl(const std::string & url) = 0;
+
     struct TarballInfo
     {
         Hash treeHash;
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 55d3a8ebe..42b4aa23a 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -8,6 +8,7 @@
 #include "util.hh"
 #include "git.hh"
 #include "fs-input-accessor.hh"
+#include "union-input-accessor.hh"
 #include "git-utils.hh"
 
 #include "fetch-settings.hh"
@@ -134,11 +135,6 @@ std::optional<std::string> readHeadCached(const std::string & actualUrl)
     return std::nullopt;
 }
 
-bool isNotDotGitDirectory(const Path & path)
-{
-    return baseNameOf(path) != ".git";
-}
-
 }  // end namespace
 
 struct GitInputScheme : InputScheme
@@ -413,7 +409,7 @@ struct GitInputScheme : InputScheme
 
         std::string name = input.getName();
 
-        auto makeResult2 = [&](const Attrs & infoAttrs, ref<InputAccessor> accessor) -> std::pair<ref<InputAccessor>, Input>
+        auto makeResult = [&](const Attrs & infoAttrs, ref<InputAccessor> accessor) -> std::pair<ref<InputAccessor>, Input>
         {
             assert(input.getRev());
             assert(!origRev || origRev == input.getRev());
@@ -424,18 +420,6 @@ struct GitInputScheme : InputScheme
             return {accessor, std::move(input)};
         };
 
-        auto makeResult = [&](const Attrs & infoAttrs, const StorePath & storePath) -> std::pair<ref<InputAccessor>, Input>
-        {
-            // FIXME: remove?
-            //input.attrs.erase("narHash");
-            auto narHash = store->queryPathInfo(storePath)->narHash;
-            input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
-
-            auto accessor = makeStorePathAccessor(store, storePath, makeNotAllowedError(repoInfo.url));
-
-            return makeResult2(infoAttrs, accessor);
-        };
-
         auto originalRef = input.getRef();
         auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo);
         input.attrs.insert_or_assign("ref", ref);
@@ -542,66 +526,39 @@ struct GitInputScheme : InputScheme
 
         printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.url);
 
-        if (!repoInfo.submodules) {
-            auto accessor = GitRepo::openRepo(CanonPath(repoDir))->getAccessor(rev);
-            return makeResult2(infoAttrs, accessor);
+        auto repo = GitRepo::openRepo(CanonPath(repoDir));
+
+        auto accessor = repo->getAccessor(rev);
+
+        /* If the repo has submodules, fetch them and return a union
+           input accessor consisting of the accessor for the top-level
+           repo and the accessors for the submodules. */
+        if (repoInfo.submodules) {
+            std::map<CanonPath, nix::ref<InputAccessor>> mounts;
+
+            for (auto & submodule : repo->getSubmodules(rev)) {
+                auto resolved = repo->resolveSubmoduleUrl(submodule.url);
+                debug("Git submodule %s: %s %s %s -> %s",
+                    submodule.path, submodule.url, submodule.branch, submodule.rev.gitRev(), resolved);
+                fetchers::Attrs attrs;
+                attrs.insert_or_assign("type", "git");
+                attrs.insert_or_assign("url", resolved);
+                if (submodule.branch != "")
+                    attrs.insert_or_assign("ref", submodule.branch);
+                attrs.insert_or_assign("rev", submodule.rev.gitRev());
+                auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs));
+                auto [submoduleAccessor, submoduleInput2] =
+                    submoduleInput.scheme->getAccessor(store, submoduleInput);
+                mounts.insert_or_assign(submodule.path, submoduleAccessor);
+            }
+
+            if (!mounts.empty()) {
+                mounts.insert_or_assign(CanonPath::root, accessor);
+                accessor = makeUnionInputAccessor(std::move(mounts));
+            }
         }
 
-        else {
-            // FIXME: use libgit2
-            Path tmpDir = createTempDir();
-            AutoDelete delTmpDir(tmpDir, true);
-            PathFilter filter = defaultPathFilter;
-
-            Activity act(*logger, lvlChatty, actUnknown, fmt("copying Git tree '%s' to the store", input.to_string()));
-
-            Path tmpGitDir = createTempDir();
-            AutoDelete delTmpGitDir(tmpGitDir, true);
-
-            runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", tmpDir, "--separate-git-dir", tmpGitDir });
-
-            {
-                // TODO: repoDir might lack the ref (it only checks if rev
-                // exists, see FIXME above) so use a big hammer and fetch
-                // everything to ensure we get the rev.
-                Activity act(*logger, lvlTalkative, actUnknown, fmt("making temporary clone of '%s'", repoDir));
-                runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force",
-                        "--update-head-ok", "--", repoDir, "refs/*:refs/*" }, {}, true);
-            }
-
-            runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", rev.gitRev() });
-
-            /* Ensure that we use the correct origin for fetching
-               submodules. This matters for submodules with relative
-               URLs. */
-            if (repoInfo.isLocal) {
-                writeFile(tmpGitDir + "/config", readFile(repoDir + "/" + repoInfo.gitDir + "/config"));
-
-                /* Restore the config.bare setting we may have just
-                   copied erroneously from the user's repo. */
-                runProgram("git", true, { "-C", tmpDir, "config", "core.bare", "false" });
-            } else
-                runProgram("git", true, { "-C", tmpDir, "config", "remote.origin.url", repoInfo.url });
-
-            /* As an optimisation, copy the modules directory of the
-               source repo if it exists. */
-            auto modulesPath = repoDir + "/" + repoInfo.gitDir + "/modules";
-            if (pathExists(modulesPath)) {
-                Activity act(*logger, lvlTalkative, actUnknown, fmt("copying submodules of '%s'", repoInfo.url));
-                runProgram("cp", true, { "-R", "--", modulesPath, tmpGitDir + "/modules" });
-            }
-
-            {
-                Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", repoInfo.url));
-                runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }, {}, true);
-            }
-
-            filter = isNotDotGitDirectory;
-
-            auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);
-
-            return makeResult(infoAttrs, std::move(storePath));
-        }
+        return makeResult(infoAttrs, accessor);
     }
 
     std::pair<ref<InputAccessor>, Input> getAccessorFromWorkdir(
diff --git a/src/libfetchers/union-input-accessor.cc b/src/libfetchers/union-input-accessor.cc
new file mode 100644
index 000000000..940c0e06c
--- /dev/null
+++ b/src/libfetchers/union-input-accessor.cc
@@ -0,0 +1,80 @@
+#include "union-input-accessor.hh"
+
+namespace nix {
+
+struct UnionInputAccessor : InputAccessor
+{
+    std::map<CanonPath, ref<InputAccessor>> mounts;
+
+    UnionInputAccessor(std::map<CanonPath, ref<InputAccessor>> _mounts)
+        : mounts(std::move(_mounts))
+    {
+        // Currently we require a root filesystem. This could be relaxed.
+        assert(mounts.contains(CanonPath::root));
+
+        // FIXME: should check that every mount point exists. Or we
+        // could return dummy parent directories automatically.
+    }
+
+    std::string readFile(const CanonPath & path) override
+    {
+        auto [accessor, subpath] = resolve(path);
+        return accessor->readFile(subpath);
+    }
+
+    bool pathExists(const CanonPath & path) override
+    {
+        auto [accessor, subpath] = resolve(path);
+        return accessor->pathExists(subpath);
+    }
+
+    Stat lstat(const CanonPath & path) override
+    {
+        auto [accessor, subpath] = resolve(path);
+        return accessor->lstat(subpath);
+    }
+
+    DirEntries readDirectory(const CanonPath & path) override
+    {
+        auto [accessor, subpath] = resolve(path);
+        return accessor->readDirectory(subpath);
+    }
+
+    std::string readLink(const CanonPath & path) override
+    {
+        auto [accessor, subpath] = resolve(path);
+        return accessor->readLink(subpath);
+    }
+
+    std::string showPath(const CanonPath & path) override
+    {
+        auto [accessor, subpath] = resolve(path);
+        return accessor->showPath(subpath);
+    }
+
+    std::pair<ref<InputAccessor>, CanonPath> resolve(CanonPath path)
+    {
+        // Find the nearest parent of `path` that is a mount point.
+        std::vector<std::string> ss;
+        while (true) {
+            auto i = mounts.find(path);
+            if (i != mounts.end()) {
+                auto subpath = CanonPath::root;
+                for (auto j = ss.rbegin(); j != ss.rend(); ++j)
+                    subpath.push(*j);
+                return {i->second, std::move(subpath)};
+            }
+
+            assert(!path.isRoot());
+            ss.push_back(std::string(*path.baseName()));
+            path.pop();
+        }
+    }
+};
+
+ref<InputAccessor> makeUnionInputAccessor(std::map<CanonPath, ref<InputAccessor>> mounts)
+{
+    return make_ref<UnionInputAccessor>(std::move(mounts));
+}
+
+}
diff --git a/src/libfetchers/union-input-accessor.hh b/src/libfetchers/union-input-accessor.hh
new file mode 100644
index 000000000..6a1649c1d
--- /dev/null
+++ b/src/libfetchers/union-input-accessor.hh
@@ -0,0 +1,9 @@
+#pragma once
+
+#include "input-accessor.hh"
+
+namespace nix {
+
+ref<InputAccessor> makeUnionInputAccessor(std::map<CanonPath, ref<InputAccessor>> mounts);
+
+}
diff --git a/tests/functional/fetchGitSubmodules.sh b/tests/functional/fetchGitSubmodules.sh
index df81232e5..369cdc5db 100644
--- a/tests/functional/fetchGitSubmodules.sh
+++ b/tests/functional/fetchGitSubmodules.sh
@@ -118,11 +118,3 @@ cloneRepo=$TEST_ROOT/a/b/gitSubmodulesClone # NB /a/b to make the relative path
 git clone $rootRepo $cloneRepo
 pathIndirect=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$cloneRepo; rev = \"$rev2\"; submodules = true; }).outPath")
 [[ $pathIndirect = $pathWithRelative ]]
-
-# Test that if the clone has the submodule already, we're not fetching
-# it again.
-git -C $cloneRepo submodule update --init
-rm $TEST_HOME/.cache/nix/fetcher-cache*
-rm -rf $subRepo
-pathSubmoduleGone=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$cloneRepo; rev = \"$rev2\"; submodules = true; }).outPath")
-[[ $pathSubmoduleGone = $pathWithRelative ]]

From 669b074f51c4fea6b362313f47eebb4a67f0e89d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sat, 28 Oct 2023 16:16:20 +0200
Subject: [PATCH 024/294] Cleanup

---
 src/libfetchers/git.cc | 19 +++++++------------
 1 file changed, 7 insertions(+), 12 deletions(-)

diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 42b4aa23a..a66a51cca 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -409,17 +409,6 @@ struct GitInputScheme : InputScheme
 
         std::string name = input.getName();
 
-        auto makeResult = [&](const Attrs & infoAttrs, ref<InputAccessor> accessor) -> std::pair<ref<InputAccessor>, Input>
-        {
-            assert(input.getRev());
-            assert(!origRev || origRev == input.getRev());
-            if (!repoInfo.shallow)
-                input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
-            input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified"));
-
-            return {accessor, std::move(input)};
-        };
-
         auto originalRef = input.getRef();
         auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo);
         input.attrs.insert_or_assign("ref", ref);
@@ -558,7 +547,13 @@ struct GitInputScheme : InputScheme
             }
         }
 
-        return makeResult(infoAttrs, accessor);
+        assert(input.getRev());
+        assert(!origRev || origRev == input.getRev());
+        if (!repoInfo.shallow)
+            input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
+        input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified"));
+
+        return {accessor, std::move(input)};
     }
 
     std::pair<ref<InputAccessor>, Input> getAccessorFromWorkdir(

From 1fd0867389c2dd3e98d06decd4d35067885550a0 Mon Sep 17 00:00:00 2001
From: Felix Uhl <felix.uhl@netlight.com>
Date: Fri, 11 Aug 2023 21:47:16 +0200
Subject: [PATCH 025/294] Fix missing output when creating lockfile

---
 src/libexpr/flake/flake.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index 45c9ec8f3..8cc803ccf 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -651,14 +651,14 @@ LockedFlake lockFlake(
 
                             bool lockFileExists = pathExists(outputLockFilePath);
 
+                            auto s = chomp(diff);
                             if (lockFileExists) {
-                                auto s = chomp(diff);
                                 if (s.empty())
                                     warn("updating lock file '%s'", outputLockFilePath);
                                 else
                                     warn("updating lock file '%s':\n%s", outputLockFilePath, s);
                             } else
-                                warn("creating lock file '%s'", outputLockFilePath);
+                                warn("creating lock file '%s': \n%s", outputLockFilePath, s);
 
                             std::optional<std::string> commitMessage = std::nullopt;
 

From c762b65dc5314ed631381cf4bf26f5976e825bdc Mon Sep 17 00:00:00 2001
From: Felix Uhl <felix.uhl@netlight.com>
Date: Fri, 11 Aug 2023 21:51:03 +0200
Subject: [PATCH 026/294] Fix documentation of flake command output

---
 src/nix/flake-lock.md   |  7 +++++--
 src/nix/flake-update.md | 11 ++++++++---
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/src/nix/flake-lock.md b/src/nix/flake-lock.md
index 2af0ad81e..100987a88 100644
--- a/src/nix/flake-lock.md
+++ b/src/nix/flake-lock.md
@@ -7,8 +7,11 @@ R""(
 
   ```console
   # nix flake lock --update-input nixpkgs --update-input nix
-  * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c'
-  * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293'
+  warning: creating lock file '/home/myself/repos/testflake/flake.lock':
+  • Added input 'nix':
+      'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' (2023-06-28)
+  • Added input 'nixpkgs':
+      'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30)
   ```
 
 # Description
diff --git a/src/nix/flake-update.md b/src/nix/flake-update.md
index 8c6042d94..b5a5ff0ec 100644
--- a/src/nix/flake-update.md
+++ b/src/nix/flake-update.md
@@ -6,9 +6,14 @@ R""(
   lock file:
 
   ```console
-  # nix flake update --commit-lock-file
-  * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c'
-  * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293'
+  # nix flake update
+  warning: updating lock file '/home/myself/repos/testflake/flake.lock':
+  • Updated input 'nix':
+      'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' (2023-06-28)
+    → 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' (2023-07-11)
+  • Updated input 'nixpkgs':
+      'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30)
+    → 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05)
   …
   warning: committed new revision '158bcbd9d6cc08ab859c0810186c1beebc982aad'
   ```

From c7dcdb8325be7b8ecc3d480217808be899fc865a Mon Sep 17 00:00:00 2001
From: Felix Uhl <felix.uhl@netlight.com>
Date: Sat, 12 Aug 2023 20:51:19 +0200
Subject: [PATCH 027/294] Overhaul nix flake update and lock commands

Closes #5110
---
 doc/manual/src/release-notes/rl-next.md | 14 +++++++
 src/libcmd/command.hh                   |  6 +++
 src/libcmd/installables.cc              | 22 +---------
 src/libexpr/flake/flake.cc              | 11 +++--
 src/nix/flake-lock.md                   | 45 ++++++++++-----------
 src/nix/flake-update.md                 | 53 +++++++++++++++++--------
 src/nix/flake.cc                        | 33 +++++++++++++--
 tests/functional/completions.sh         |  7 ++--
 tests/functional/flakes/circular.sh     |  3 +-
 tests/functional/flakes/flakes.sh       |  6 +--
 tests/functional/flakes/follow-paths.sh |  4 +-
 11 files changed, 124 insertions(+), 80 deletions(-)

diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
index 276252c37..3cfb53998 100644
--- a/doc/manual/src/release-notes/rl-next.md
+++ b/doc/manual/src/release-notes/rl-next.md
@@ -15,3 +15,17 @@
 - `nix-shell` shebang lines now support single-quoted arguments.
 
 - `builtins.fetchTree` is now marked as stable.
+
+
+- The interface for creating and updating lock files has been overhauled:
+
+  - [`nix flake lock`](@docroot@/command-ref/new-cli/nix3-flake-lock.md) only creates lock files and adds missing inputs now.
+    It will *never* update existing inputs.
+
+  - [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) does the same, but *will* update inputs.
+    - Passing no arguments will update all inputs of the current flake, just like it already did.
+    - Passing input names as arguments will ensure only those are updated. This replaces the functionality of `nix flake lock --update-input`
+    - To operate on a flake outside the current directory, you must now pass `--flake path/to/flake`.
+
+  - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables.
+    They are superceded by `nix flake update`.
diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh
index dafc0db3b..120c832ac 100644
--- a/src/libcmd/command.hh
+++ b/src/libcmd/command.hh
@@ -326,6 +326,12 @@ struct MixEnvironment : virtual Args {
     void setEnviron();
 };
 
+void completeFlakeInputPath(
+    AddCompletions & completions,
+    ref<EvalState> evalState,
+    const std::vector<FlakeRef> & flakeRefs,
+    std::string_view prefix);
+
 void completeFlakeRef(AddCompletions & completions, ref<Store> store, std::string_view prefix);
 
 void completeFlakeRefWithFragment(
diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index eff18bbf6..3aff601e0 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -28,7 +28,7 @@
 
 namespace nix {
 
-static void completeFlakeInputPath(
+void completeFlakeInputPath(
     AddCompletions & completions,
     ref<EvalState> evalState,
     const std::vector<FlakeRef> & flakeRefs,
@@ -46,13 +46,6 @@ MixFlakeOptions::MixFlakeOptions()
 {
     auto category = "Common flake-related options";
 
-    addFlag({
-        .longName = "recreate-lock-file",
-        .description = "Recreate the flake's lock file from scratch.",
-        .category = category,
-        .handler = {&lockFlags.recreateLockFile, true}
-    });
-
     addFlag({
         .longName = "no-update-lock-file",
         .description = "Do not allow any updates to the flake's lock file.",
@@ -85,19 +78,6 @@ MixFlakeOptions::MixFlakeOptions()
         .handler = {&lockFlags.commitLockFile, true}
     });
 
-    addFlag({
-        .longName = "update-input",
-        .description = "Update a specific flake input (ignoring its previous entry in the lock file).",
-        .category = category,
-        .labels = {"input-path"},
-        .handler = {[&](std::string s) {
-            lockFlags.inputUpdates.insert(flake::parseInputPath(s));
-        }},
-        .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
-            completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix);
-        }}
-    });
-
     addFlag({
         .longName = "override-input",
         .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`). This implies `--no-write-lock-file`.",
diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index 8cc803ccf..70ae7b584 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -447,8 +447,8 @@ LockedFlake lockFlake(
 
                     assert(input.ref);
 
-                    /* Do we have an entry in the existing lock file? And we
-                       don't have a --update-input flag for this input? */
+                    /* Do we have an entry in the existing lock file?
+                       And the input is not in updateInputs? */
                     std::shared_ptr<LockedNode> oldLock;
 
                     updatesUsed.insert(inputPath);
@@ -472,9 +472,8 @@ LockedFlake lockFlake(
 
                         node->inputs.insert_or_assign(id, childNode);
 
-                        /* If we have an --update-input flag for an input
-                           of this input, then we must fetch the flake to
-                           update it. */
+                        /* If we have this input in updateInputs, then we
+                           must fetch the flake to update it. */
                         auto lb = lockFlags.inputUpdates.lower_bound(inputPath);
 
                         auto mustRefetch =
@@ -616,7 +615,7 @@ LockedFlake lockFlake(
 
         for (auto & i : lockFlags.inputUpdates)
             if (!updatesUsed.count(i))
-                warn("the flag '--update-input %s' does not match any input", printInputPath(i));
+                warn("'%s' does not match any input of this flake", printInputPath(i));
 
         /* Check 'follows' inputs. */
         newLockFile.check();
diff --git a/src/nix/flake-lock.md b/src/nix/flake-lock.md
index 100987a88..6d10258e3 100644
--- a/src/nix/flake-lock.md
+++ b/src/nix/flake-lock.md
@@ -2,11 +2,10 @@ R""(
 
 # Examples
 
-* Update the `nixpkgs` and `nix` inputs of the flake in the current
-  directory:
+* Create the lock file for the flake in the current directory:
 
   ```console
-  # nix flake lock --update-input nixpkgs --update-input nix
+  # nix flake lock
   warning: creating lock file '/home/myself/repos/testflake/flake.lock':
   • Added input 'nix':
       'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' (2023-06-28)
@@ -14,28 +13,28 @@ R""(
       'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30)
   ```
 
+* Add missing inputs to the lock file for a flake in a different directory:
+
+  ```console
+  # nix flake lock ~/repos/another
+  warning: updating lock file '/home/myself/repos/another/flake.lock':
+  • Added input 'nixpkgs':
+      'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30)
+  ```
+
+  > **Note**
+  >
+  > When trying to refer to a flake in a subdirectory, write `./another`
+  > instead of `another`.
+  > Otherwise Nix will try to look up the flake in the registry.
+
 # Description
 
-This command updates the lock file of a flake (`flake.lock`) so that
-it contains a lock for every flake input specified in
-`flake.nix`. Existing lock file entries are not updated unless
-required by a flag such as `--update-input`.
+This command adds inputs to the lock file of a flake (`flake.lock`)
+so that it contains a lock for every flake input specified in
+`flake.nix`. Existing lock file entries are not updated.
 
-Note that every command that operates on a flake will also update the
-lock file if needed, and supports the same flags. Therefore,
-
-```console
-# nix flake lock --update-input nixpkgs
-# nix build
-```
-
-is equivalent to:
-
-```console
-# nix build --update-input nixpkgs
-```
-
-Thus, this command is only useful if you want to update the lock file
-separately from any other action such as building.
+If you want to update existing lock entries, use
+[`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md)
 
 )""
diff --git a/src/nix/flake-update.md b/src/nix/flake-update.md
index b5a5ff0ec..63df3b12a 100644
--- a/src/nix/flake-update.md
+++ b/src/nix/flake-update.md
@@ -2,8 +2,7 @@ R""(
 
 # Examples
 
-* Recreate the lock file (i.e. update all inputs) and commit the new
-  lock file:
+* Update all inputs (i.e. recreate the lock file from scratch):
 
   ```console
   # nix flake update
@@ -14,26 +13,46 @@ R""(
   • Updated input 'nixpkgs':
       'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30)
     → 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05)
-  …
-  warning: committed new revision '158bcbd9d6cc08ab859c0810186c1beebc982aad'
   ```
 
+* Update only a single input:
+
+  ```console
+  # nix flake update nixpkgs
+  warning: updating lock file '/home/myself/repos/testflake/flake.lock':
+  • Updated input 'nixpkgs':
+      'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30)
+    → 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05)
+  ```
+
+* Update only a single input of a flake in a different directory:
+
+  ```console
+  # nix flake update nixpkgs --flake ~/repos/another
+  warning: updating lock file '/home/myself/repos/another/flake.lock':
+  • Updated input 'nixpkgs':
+      'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30)
+    → 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05)
+  ```
+
+  > **Note**
+  >
+  > When trying to refer to a flake in a subdirectory, write `./another`
+  > instead of `another`.
+  > Otherwise Nix will try to look up the flake in the registry.
+
 # Description
 
-This command recreates the lock file of a flake (`flake.lock`), thus
-updating the lock for every unlocked input (like `nixpkgs`) to its
-current version. This is equivalent to passing `--recreate-lock-file`
-to any command that operates on a flake. That is,
+This command updates the inputs in a lock file (`flake.lock`).
+**By default, all inputs are updated**. If the lock file doesn't exist
+yet, it will be created. If inputs are not in the lock file yet, they will be added.
 
-```console
-# nix flake update
-# nix build
-```
+Unlike other `nix flake` commands, `nix flake update` takes a list of names of inputs
+to update as its positional arguments and operates on the flake in the current directory.
+You can pass a different flake-url with `--flake` to override that default.
 
-is equivalent to:
-
-```console
-# nix build --recreate-lock-file
-```
+The related command [`nix flake lock`](@docroot@/command-ref/new-cli/nix3-flake-lock.md)
+also creates lock files and adds missing inputs, but is safer as it
+will never update inputs already in the lock file.
 
 )""
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index 0116eff2e..e8906a252 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -24,8 +24,10 @@ using namespace nix;
 using namespace nix::flake;
 using json = nlohmann::json;
 
+struct CmdFlakeUpdate;
 class FlakeCommand : virtual Args, public MixFlakeOptions
 {
+protected:
     std::string flakeUrl = ".";
 
 public:
@@ -63,6 +65,8 @@ public:
 
 struct CmdFlakeUpdate : FlakeCommand
 {
+public:
+
     std::string description() override
     {
         return "update flake lock file";
@@ -70,9 +74,31 @@ struct CmdFlakeUpdate : FlakeCommand
 
     CmdFlakeUpdate()
     {
+        expectedArgs.clear();
+        addFlag({
+            .longName="flake",
+            .description="The flake to operate on. Default is the current directory.",
+            .labels={"flake-url"},
+            .handler={&flakeUrl},
+            .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
+                completeFlakeRef(completions, getStore(), prefix);
+            }}
+        });
+        expectArgs({
+            .label="inputs",
+            .optional=true,
+            .handler={[&](std::string inputToUpdate){
+                auto inputPath = flake::parseInputPath(inputToUpdate);
+                if (lockFlags.inputUpdates.contains(inputPath))
+                    warn("Input '%s' was specified multiple times. You may have done this by accident.");
+                lockFlags.inputUpdates.insert(inputPath);
+            }},
+            .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
+                completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix);
+            }}
+        });
+
         /* Remove flags that don't make sense. */
-        removeFlag("recreate-lock-file");
-        removeFlag("update-input");
         removeFlag("no-update-lock-file");
         removeFlag("no-write-lock-file");
     }
@@ -87,8 +113,9 @@ struct CmdFlakeUpdate : FlakeCommand
     void run(nix::ref<nix::Store> store) override
     {
         settings.tarballTtl = 0;
+        auto updateAll = lockFlags.inputUpdates.empty();
 
-        lockFlags.recreateLockFile = true;
+        lockFlags.recreateLockFile = updateAll;
         lockFlags.writeLockFile = true;
         lockFlags.applyNixConfig = true;
 
diff --git a/tests/functional/completions.sh b/tests/functional/completions.sh
index 7c1e4b287..b9886623a 100644
--- a/tests/functional/completions.sh
+++ b/tests/functional/completions.sh
@@ -48,11 +48,10 @@ EOF
 [[ "$(NIX_GET_COMPLETIONS=5 nix build ./foo ./bar --override-input '')" == $'normal\na\t\nb\t' ]]
 ## With tilde expansion
 [[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix build '~/foo' --override-input '')" == $'normal\na\t' ]]
-[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=5 nix flake show '~/foo' --update-input '')" == $'normal\na\t' ]]
-[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix run '~/foo' --update-input '')" == $'normal\na\t' ]]
+[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=5 nix flake update --flake '~/foo' '')" == $'normal\na\t' ]]
 ## Out of order
-[[ "$(NIX_GET_COMPLETIONS=3 nix build --update-input '' ./foo)" == $'normal\na\t' ]]
-[[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --update-input '' ./bar)" == $'normal\na\t\nb\t' ]]
+[[ "$(NIX_GET_COMPLETIONS=3 nix build --override-input '' '' ./foo)" == $'normal\na\t' ]]
+[[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '' '' ./bar)" == $'normal\na\t\nb\t' ]]
 
 # Cli flag completion
 NIX_GET_COMPLETIONS=2 nix build --log-form | grep -- "--log-format"
diff --git a/tests/functional/flakes/circular.sh b/tests/functional/flakes/circular.sh
index 09cd02edf..d3bb8e8a3 100644
--- a/tests/functional/flakes/circular.sh
+++ b/tests/functional/flakes/circular.sh
@@ -42,7 +42,8 @@ git -C $flakeB commit -a -m 'Foo'
 sed -i $flakeB/flake.nix -e 's/456/789/'
 git -C $flakeB commit -a -m 'Foo'
 
-[[ $(nix eval --update-input b $flakeA#foo) = 1912 ]]
+nix flake update b --flake $flakeA
+[[ $(nix eval $flakeA#foo) = 1912 ]]
 
 # Test list-inputs with circular dependencies
 nix flake metadata $flakeA
diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh
index 70de28628..b0038935c 100644
--- a/tests/functional/flakes/flakes.sh
+++ b/tests/functional/flakes/flakes.sh
@@ -300,7 +300,7 @@ nix build -o "$TEST_ROOT/result" flake4#xyzzy
 nix flake lock "$flake3Dir"
 [[ -z $(git -C "$flake3Dir" diff master || echo failed) ]]
 
-nix flake update "$flake3Dir" --override-flake flake2 nixpkgs
+nix flake update --flake "$flake3Dir" --override-flake flake2 nixpkgs
 [[ ! -z $(git -C "$flake3Dir" diff master || echo failed) ]]
 
 # Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore
@@ -437,7 +437,7 @@ cat > "$flake3Dir/flake.nix" <<EOF
 }
 EOF
 
-nix flake update "$flake3Dir"
+nix flake update --flake "$flake3Dir"
 [[ $(jq -c .nodes.flake2.inputs.flake1 "$flake3Dir/flake.lock") =~ '["foo"]' ]]
 [[ $(jq .nodes.foo.locked.url "$flake3Dir/flake.lock") =~ flake7 ]]
 
@@ -480,7 +480,7 @@ nix flake lock "$flake3Dir" --override-input flake2/flake1 flake1/master/$hash1
 nix flake lock "$flake3Dir"
 [[ $(jq -r .nodes.flake1_2.locked.rev "$flake3Dir/flake.lock") = $hash1 ]]
 
-nix flake lock "$flake3Dir" --update-input flake2/flake1
+nix flake update flake2/flake1 --flake "$flake3Dir"
 [[ $(jq -r .nodes.flake1_2.locked.rev "$flake3Dir/flake.lock") =~ $hash2 ]]
 
 # Test 'nix flake metadata --json'.
diff --git a/tests/functional/flakes/follow-paths.sh b/tests/functional/flakes/follow-paths.sh
index 8573b5511..9f23d738b 100644
--- a/tests/functional/flakes/follow-paths.sh
+++ b/tests/functional/flakes/follow-paths.sh
@@ -77,7 +77,7 @@ git -C $flakeFollowsA add flake.nix flakeB/flake.nix \
 
 nix flake metadata $flakeFollowsA
 
-nix flake update $flakeFollowsA
+nix flake update --flake $flakeFollowsA
 
 nix flake lock $flakeFollowsA
 
@@ -228,7 +228,7 @@ git -C "$flakeFollowsOverloadA" add flake.nix flakeB/flake.nix \
   flakeB/flakeC/flake.nix flakeB/flakeC/flakeD/flake.nix
 
 nix flake metadata "$flakeFollowsOverloadA"
-nix flake update "$flakeFollowsOverloadA"
+nix flake update --flake "$flakeFollowsOverloadA"
 nix flake lock "$flakeFollowsOverloadA"
 
 # Now test follow cycle detection

From f282ef5a56f6314d25044af62b4de38c66b29d38 Mon Sep 17 00:00:00 2001
From: Felix Uhl <felix.uhl@outlook.com>
Date: Tue, 24 Oct 2023 22:22:05 +0200
Subject: [PATCH 028/294] fix: segfault in positional arg completion

Adding the inputPath as a positional feature uncovered this bug.
As positional argument forms were discarded from the `expectedArgs`
list, their closures were not. When the `.completer` closure was then
called, part of the surrounding object did not exist anymore.

This didn't cause an issue before, but with the new call to
`getEvalState()` in the "inputs" completer in nix/flake.cc, a segfault
was triggered reproducibly on invalid memory access to the `this`
pointer, which was always 0.

The solution of splicing the argument forms into a new list to extend
their lifetime is a bit of a hack, but I was unable to get the "nicer"
iterator-based solution to work.
---
 src/libutil/args.cc             | 13 ++++++++++++-
 src/libutil/args.hh             | 14 +++++++++++++-
 tests/functional/completions.sh |  4 ++++
 3 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 6bc3cae07..811353c18 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -255,7 +255,18 @@ bool Args::processArgs(const Strings & args, bool finish)
         }
         if (!anyCompleted)
             exp.handler.fun(ss);
-        expectedArgs.pop_front();
+
+        /* Move the list element to the processedArgs. This is almost the same as 
+           `processedArgs.push_back(expectedArgs.front()); expectedArgs.pop_front()`,
+           except that it will only adjust the next and prev pointers of the list
+           elements, meaning the actual contents don't move in memory. This is
+           critical to prevent invalidating internal pointers! */
+        processedArgs.splice(
+            processedArgs.end(),
+            expectedArgs,
+            expectedArgs.begin(),
+            ++expectedArgs.begin());
+
         res = true;
     }
 
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index ff2bf3cab..e3b41313f 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -200,13 +200,25 @@ protected:
     /**
      * Queue of expected positional argument forms.
      *
-     * Positional arugment descriptions are inserted on the back.
+     * Positional argument descriptions are inserted on the back.
      *
      * As positional arguments are passed, these are popped from the
      * front, until there are hopefully none left as all args that were
      * expected in fact were passed.
      */
     std::list<ExpectedArg> expectedArgs;
+    /**
+     * List of processed positional argument forms.
+     * 
+     * All items removed from `expectedArgs` are added here. After all
+     * arguments were processed, this list should be exactly the same as
+     * `expectedArgs` was before.
+     * 
+     * This list is used to extend the lifetime of the argument forms.
+     * If this is not done, some closures that reference the command
+     * itself will segfault.
+    */
+   std::list<ExpectedArg> processedArgs;
 
     /**
      * Process some positional arugments
diff --git a/tests/functional/completions.sh b/tests/functional/completions.sh
index b9886623a..d3d5bbd48 100644
--- a/tests/functional/completions.sh
+++ b/tests/functional/completions.sh
@@ -44,6 +44,10 @@ EOF
 # Input override completion
 [[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '')" == $'normal\na\t' ]]
 [[ "$(NIX_GET_COMPLETIONS=5 nix flake show ./foo --override-input '')" == $'normal\na\t' ]]
+cd ./foo
+[[ "$(NIX_GET_COMPLETIONS=3 nix flake update '')" == $'normal\na\t' ]]
+cd ..
+[[ "$(NIX_GET_COMPLETIONS=5 nix flake update --flake './foo' '')" == $'normal\na\t' ]]
 ## With multiple input flakes
 [[ "$(NIX_GET_COMPLETIONS=5 nix build ./foo ./bar --override-input '')" == $'normal\na\t\nb\t' ]]
 ## With tilde expansion

From 0c5eac9c4550a6de2cd829d25e628f779e2a29c7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 31 Oct 2023 15:59:25 +0100
Subject: [PATCH 029/294] Git fetcher: Handle submodules for workdirs

---
 src/libfetchers/git-utils.cc                  | 83 +++++++++++--------
 src/libfetchers/git-utils.hh                  | 27 ++++--
 src/libfetchers/git.cc                        | 49 +++++++++--
 tests/functional/flakes/flake-in-submodule.sh | 14 +++-
 4 files changed, 119 insertions(+), 54 deletions(-)

diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index 5e3e6dae4..5b14cfdb1 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -216,6 +216,43 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         return toHash(*oid);
     }
 
+    std::vector<Submodule> parseSubmodules(const CanonPath & configFile)
+    {
+        GitConfig config;
+        if (git_config_open_ondisk(Setter(config), configFile.abs().c_str()))
+            throw Error("parsing .gitmodules file: %s", git_error_last()->message);
+
+        ConfigIterator it;
+        if (git_config_iterator_glob_new(Setter(it), config.get(), "^submodule\\..*\\.(path|url|branch)$"))
+            throw Error("iterating over .gitmodules: %s", git_error_last()->message);
+
+        std::map<std::string, std::string> entries;
+
+        while (true) {
+            git_config_entry * entry = nullptr;
+            if (auto err = git_config_next(&entry, it.get())) {
+                if (err == GIT_ITEROVER) break;
+                throw Error("iterating over .gitmodules: %s", git_error_last()->message);
+            }
+            entries.emplace(entry->name + 10, entry->value);
+        }
+
+        std::vector<Submodule> result;
+
+        for (auto & [key, value] : entries) {
+            if (!hasSuffix(key, ".path")) continue;
+            std::string key2(key, 0, key.size() - 5);
+            auto path = CanonPath(value);
+            result.push_back(Submodule {
+                .path = path,
+                .url = entries[key2 + ".url"],
+                .branch = entries[key2 + ".branch"],
+            });
+        }
+
+        return result;
+    }
+
     WorkdirInfo getWorkdirInfo() override
     {
         WorkdirInfo info;
@@ -246,6 +283,11 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         if (git_status_foreach_ext(*this, &options, &statusCallbackTrampoline, &statusCallback))
             throw Error("getting working directory status: %s", git_error_last()->message);
 
+        /* Get submodule info. */
+        auto modulesFile = path + ".gitmodules";
+        if (pathExists(modulesFile.abs()))
+            info.submodules = parseSubmodules(modulesFile);
+
         return info;
     }
 
@@ -261,7 +303,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         return std::nullopt;
     }
 
-    std::vector<Submodule> getSubmodules(const Hash & rev) override;
+    std::vector<std::tuple<Submodule, Hash>> getSubmodules(const Hash & rev) override;
 
     std::string resolveSubmoduleUrl(const std::string & url) override
     {
@@ -521,7 +563,7 @@ ref<InputAccessor> GitRepoImpl::getAccessor(const Hash & rev)
     return make_ref<GitInputAccessor>(ref<GitRepoImpl>(shared_from_this()), rev);
 }
 
-std::vector<GitRepoImpl::Submodule> GitRepoImpl::getSubmodules(const Hash & rev)
+std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules(const Hash & rev)
 {
     /* Read the .gitmodules files from this revision. */
     CanonPath modulesFile(".gitmodules");
@@ -529,44 +571,17 @@ std::vector<GitRepoImpl::Submodule> GitRepoImpl::getSubmodules(const Hash & rev)
     auto accessor = getAccessor(rev);
     if (!accessor->pathExists(modulesFile)) return {};
 
-    /* Parse it. */
+    /* Parse it and get the revision of each submodule. */
     auto configS = accessor->readFile(modulesFile);
 
     auto [fdTemp, pathTemp] = createTempFile("nix-git-submodules");
     writeFull(fdTemp.get(), configS);
 
-    GitConfig config;
-    if (git_config_open_ondisk(Setter(config), pathTemp.c_str()))
-        throw Error("parsing .gitmodules file: %s", git_error_last()->message);
+    std::vector<std::tuple<Submodule, Hash>> result;
 
-    ConfigIterator it;
-    if (git_config_iterator_glob_new(Setter(it), config.get(), "^submodule\\..*\\.(path|url|branch)$"))
-        throw Error("iterating over .gitmodules: %s", git_error_last()->message);
-
-    std::map<std::string, std::string> entries;
-
-    while (true) {
-        git_config_entry * entry = nullptr;
-        if (auto err = git_config_next(&entry, it.get())) {
-            if (err == GIT_ITEROVER) break;
-            throw Error("iterating over .gitmodules: %s", git_error_last()->message);
-        }
-        entries.emplace(entry->name + 10, entry->value);
-    }
-
-    std::vector<Submodule> result;
-
-    for (auto & [key, value] : entries) {
-        if (!hasSuffix(key, ".path")) continue;
-        std::string key2(key, 0, key.size() - 5);
-        auto path = CanonPath(value);
-        auto rev = accessor.dynamic_pointer_cast<GitInputAccessor>()->getSubmoduleRev(path);
-        result.push_back(Submodule {
-            .path = path,
-            .url = entries[key2 + ".url"],
-            .branch = entries[key2 + ".branch"],
-            .rev = rev,
-        });
+    for (auto & submodule : parseSubmodules(CanonPath(pathTemp))) {
+        auto rev = accessor.dynamic_pointer_cast<GitInputAccessor>()->getSubmoduleRev(submodule.path);
+        result.push_back({std::move(submodule), rev});
     }
 
     return result;
diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh
index 55e7ef969..a425e5814 100644
--- a/src/libfetchers/git-utils.hh
+++ b/src/libfetchers/git-utils.hh
@@ -20,6 +20,16 @@ struct GitRepo
     /* Return the commit hash to which a ref points. */
     virtual Hash resolveRef(std::string ref) = 0;
 
+    /**
+     * Info about a submodule.
+     */
+    struct Submodule
+    {
+        CanonPath path;
+        std::string url;
+        std::string branch;
+    };
+
     struct WorkdirInfo
     {
         bool isDirty = false;
@@ -31,6 +41,9 @@ struct GitRepo
         /* All files in the working directory that are unchanged,
            modified or added, but excluding deleted files. */
         std::set<CanonPath> files;
+
+        /* The submodules listed in .gitmodules of this workdir. */
+        std::vector<Submodule> submodules;
     };
 
     virtual WorkdirInfo getWorkdirInfo() = 0;
@@ -38,15 +51,11 @@ struct GitRepo
     /* Get the ref that HEAD points to. */
     virtual std::optional<std::string> getWorkdirRef() = 0;
 
-    struct Submodule
-    {
-        CanonPath path;
-        std::string url;
-        std::string branch;
-        Hash rev;
-    };
-
-    virtual std::vector<Submodule> getSubmodules(const Hash & rev) = 0;
+    /**
+     * Return the submodules of this repo at the indicated revision,
+     * along with the revision of each submodule.
+     */
+    virtual std::vector<std::tuple<Submodule, Hash>> getSubmodules(const Hash & rev) = 0;
 
     virtual std::string resolveSubmoduleUrl(const std::string & url) = 0;
 
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index a66a51cca..5471eb260 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -525,16 +525,16 @@ struct GitInputScheme : InputScheme
         if (repoInfo.submodules) {
             std::map<CanonPath, nix::ref<InputAccessor>> mounts;
 
-            for (auto & submodule : repo->getSubmodules(rev)) {
+            for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev)) {
                 auto resolved = repo->resolveSubmoduleUrl(submodule.url);
                 debug("Git submodule %s: %s %s %s -> %s",
-                    submodule.path, submodule.url, submodule.branch, submodule.rev.gitRev(), resolved);
+                    submodule.path, submodule.url, submodule.branch, submoduleRev.gitRev(), resolved);
                 fetchers::Attrs attrs;
                 attrs.insert_or_assign("type", "git");
                 attrs.insert_or_assign("url", resolved);
                 if (submodule.branch != "")
                     attrs.insert_or_assign("ref", submodule.branch);
-                attrs.insert_or_assign("rev", submodule.rev.gitRev());
+                attrs.insert_or_assign("rev", submoduleRev.gitRev());
                 auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs));
                 auto [submoduleAccessor, submoduleInput2] =
                     submoduleInput.scheme->getAccessor(store, submoduleInput);
@@ -557,9 +557,45 @@ struct GitInputScheme : InputScheme
     }
 
     std::pair<ref<InputAccessor>, Input> getAccessorFromWorkdir(
+        ref<Store> store,
         RepoInfo & repoInfo,
         Input && input) const
     {
+        if (repoInfo.submodules)
+            /* Create mountpoints for the submodules. */
+            for (auto & submodule : repoInfo.workdirInfo.submodules)
+                repoInfo.workdirInfo.files.insert(submodule.path);
+
+        ref<InputAccessor> accessor =
+            makeFSInputAccessor(CanonPath(repoInfo.url), repoInfo.workdirInfo.files, makeNotAllowedError(repoInfo.url));
+
+        /* If the repo has submodules, return a union input accessor
+           consisting of the accessor for the top-level repo and the
+           accessors for the submodule workdirs. */
+        if (repoInfo.submodules && !repoInfo.workdirInfo.submodules.empty()) {
+            std::map<CanonPath, nix::ref<InputAccessor>> mounts;
+
+            for (auto & submodule : repoInfo.workdirInfo.submodules) {
+                auto submodulePath = CanonPath(repoInfo.url) + submodule.path;
+                fetchers::Attrs attrs;
+                attrs.insert_or_assign("type", "git");
+                attrs.insert_or_assign("url", submodulePath.abs());
+                auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs));
+                auto [submoduleAccessor, submoduleInput2] =
+                    submoduleInput.scheme->getAccessor(store, submoduleInput);
+
+                /* If the submodule is dirty, mark this repo dirty as
+                   well. */
+                if (!submoduleInput2.getRev())
+                    repoInfo.workdirInfo.isDirty = true;
+
+                mounts.insert_or_assign(submodule.path, submoduleAccessor);
+            }
+
+            mounts.insert_or_assign(CanonPath::root, accessor);
+            accessor = makeUnionInputAccessor(std::move(mounts));
+        }
+
         if (!repoInfo.workdirInfo.isDirty) {
             if (auto ref = GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirRef())
                 input.attrs.insert_or_assign("ref", *ref);
@@ -588,10 +624,7 @@ struct GitInputScheme : InputScheme
 
         input.locked = true; // FIXME
 
-        return {
-            makeFSInputAccessor(CanonPath(repoInfo.url), repoInfo.workdirInfo.files, makeNotAllowedError(repoInfo.url)),
-            std::move(input)
-        };
+        return {accessor, std::move(input)};
     }
 
     std::pair<ref<InputAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
@@ -603,7 +636,7 @@ struct GitInputScheme : InputScheme
         if (input.getRef() || input.getRev() || !repoInfo.isLocal)
             return getAccessorFromCommit(store, repoInfo, std::move(input));
         else
-            return getAccessorFromWorkdir(repoInfo, std::move(input));
+            return getAccessorFromWorkdir(store, repoInfo, std::move(input));
     }
 };
 
diff --git a/tests/functional/flakes/flake-in-submodule.sh b/tests/functional/flakes/flake-in-submodule.sh
index 6e24a80c1..85a4d3389 100644
--- a/tests/functional/flakes/flake-in-submodule.sh
+++ b/tests/functional/flakes/flake-in-submodule.sh
@@ -46,8 +46,16 @@ echo '"expression in root repo"' > $rootRepo/root.nix
 git -C $rootRepo add root.nix
 git -C $rootRepo commit -m "Add root.nix"
 
-# FIXME
+flakeref=git+file://$rootRepo\?submodules=1\&dir=submodule
+
 # Flake can live inside a submodule and can be accessed via ?dir=submodule
-#[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#sub ) = '"expression in submodule"' ]]
+[[ $(nix eval --json $flakeref#sub ) = '"expression in submodule"' ]]
+
 # The flake can access content outside of the submodule
-#[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#root ) = '"expression in root repo"' ]]
+[[ $(nix eval --json $flakeref#root ) = '"expression in root repo"' ]]
+
+# Check that dirtying a submodule makes the entire thing dirty.
+[[ $(nix flake metadata --json $flakeref | jq -r .locked.rev) != null ]]
+echo '"foo"' > $rootRepo/submodule/sub.nix
+[[ $(nix eval --json $flakeref#sub ) = '"foo"' ]]
+[[ $(nix flake metadata --json $flakeref | jq -r .locked.rev) = null ]]

From 1f4525531e9b5e744830a55a2595880b135d93c0 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 31 Oct 2023 12:01:13 -0400
Subject: [PATCH 030/294] Add configure test to ensure GCC bug is fixed

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431

(test is adapted from issue, test does not test for GCC-specific
behavior but rather absence of bug, so test is good with other compilers
too.)
---
 configure.ac        |  3 +++
 m4/gcc_bug_80431.m4 | 64 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 67 insertions(+)
 create mode 100644 m4/gcc_bug_80431.m4

diff --git a/configure.ac b/configure.ac
index 225baf6b5..75ce7d01d 100644
--- a/configure.ac
+++ b/configure.ac
@@ -68,6 +68,9 @@ case "$host_os" in
 esac
 
 
+ENSURE_NO_GCC_BUG_80431
+
+
 # Check for pubsetbuf.
 AC_MSG_CHECKING([for pubsetbuf])
 AC_LANG_PUSH(C++)
diff --git a/m4/gcc_bug_80431.m4 b/m4/gcc_bug_80431.m4
new file mode 100644
index 000000000..e42f01956
--- /dev/null
+++ b/m4/gcc_bug_80431.m4
@@ -0,0 +1,64 @@
+# Ensure that this bug is not present in the C++ toolchain we are using.
+#
+# URL for bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431
+#
+# The test program is from that issue, with only a slight modification
+# to set an exit status instead of printing strings.
+AC_DEFUN([ENSURE_NO_GCC_BUG_80431],
+[
+  AC_MSG_CHECKING([that GCC bug 80431 is fixed])
+  AC_LANG_PUSH(C++)
+  AC_RUN_IFELSE(
+    [AC_LANG_PROGRAM(
+      [[
+        #include <cstdio>
+
+        static bool a = true;
+        static bool b = true;
+
+        struct Options { };
+
+        struct Option
+        {
+            Option(Options * options)
+            {
+                a = false;
+            }
+
+            ~Option()
+            {
+                b = false;
+            }
+        };
+
+        struct MyOptions : Options { };
+
+        struct MyOptions2 : virtual MyOptions
+        {
+            Option foo{this};
+        };
+      ]],
+      [[
+        {
+            MyOptions2 opts;
+        }
+        return (a << 1) | b;
+      ]])],
+    [status_80431=0],
+    [status_80431=$?],
+    [
+      # Assume we're bug-free when cross-compiling
+    ])
+  AC_LANG_POP(C++)
+  AS_CASE([$status_80431],
+    [0],[
+      AC_MSG_RESULT(yes)
+    ],
+    [2],[
+      AC_MSG_RESULT(no)
+      AC_MSG_ERROR(Cannot build Nix with C++ compiler with this bug)
+    ],
+    [
+      AC_MSG_RESULT(unexpected result $status_80431: not expected failure with bug, ignoring)
+    ])
+])

From b2cae33aef63644bf6e09dea253ed6e1af847fb8 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 30 Oct 2023 18:12:37 -0400
Subject: [PATCH 031/294] Remove bug-avoiding `StoreConfig *` casts for
 settings

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431 has been fixed, and
per the previous commit we now check that is the case at build time.
---
 src/libstore/binary-cache-store.hh    | 14 +++++++-------
 src/libstore/legacy-ssh-store.cc      |  6 +++---
 src/libstore/local-fs-store.hh        | 12 ++++--------
 src/libstore/local-store.hh           |  4 ++--
 src/libstore/remote-store.hh          |  4 ++--
 src/libstore/s3-binary-cache-store.cc | 18 +++++++++---------
 src/libstore/ssh-store-config.hh      |  8 ++++----
 src/libstore/ssh-store.cc             |  2 +-
 8 files changed, 32 insertions(+), 36 deletions(-)

diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index 49f271d24..218a888e3 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -17,28 +17,28 @@ struct BinaryCacheStoreConfig : virtual StoreConfig
 {
     using StoreConfig::StoreConfig;
 
-    const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression",
+    const Setting<std::string> compression{this, "xz", "compression",
         "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`)."};
 
-    const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing",
+    const Setting<bool> writeNARListing{this, false, "write-nar-listing",
         "Whether to write a JSON file that lists the files in each NAR."};
 
-    const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info",
+    const Setting<bool> writeDebugInfo{this, false, "index-debug-info",
         R"(
           Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to
           fetch debug info on demand
         )"};
 
-    const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key",
+    const Setting<Path> secretKeyFile{this, "", "secret-key",
         "Path to the secret key used to sign the binary cache."};
 
-    const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache",
+    const Setting<Path> localNarCache{this, "", "local-nar-cache",
         "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`."};
 
-    const Setting<bool> parallelCompression{(StoreConfig*) this, false, "parallel-compression",
+    const Setting<bool> parallelCompression{this, false, "parallel-compression",
         "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."};
 
-    const Setting<int> compressionLevel{(StoreConfig*) this, -1, "compression-level",
+    const Setting<int> compressionLevel{this, -1, "compression-level",
         R"(
           The *preset level* to be used when compressing NARs.
           The meaning and accepted values depend on the compression method selected.
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index c712f7eb1..38fdf118f 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -17,10 +17,10 @@ struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig
 {
     using CommonSSHStoreConfig::CommonSSHStoreConfig;
 
-    const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-store", "remote-program",
+    const Setting<Path> remoteProgram{this, "nix-store", "remote-program",
         "Path to the `nix-store` executable on the remote machine."};
 
-    const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections",
+    const Setting<int> maxConnections{this, 1, "max-connections",
         "Maximum number of concurrent SSH connections."};
 
     const std::string name() override { return "SSH Store"; }
@@ -38,7 +38,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
     // Hack for getting remote build log output.
     // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in
     // the documentation
-    const Setting<int> logFD{(StoreConfig*) this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
+    const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
 
     struct Connection
     {
diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh
index 488109501..d6bda05d1 100644
--- a/src/libstore/local-fs-store.hh
+++ b/src/libstore/local-fs-store.hh
@@ -11,25 +11,21 @@ struct LocalFSStoreConfig : virtual StoreConfig
 {
     using StoreConfig::StoreConfig;
 
-    // FIXME: the (StoreConfig*) cast works around a bug in gcc that causes
-    // it to omit the call to the Setting constructor. Clang works fine
-    // either way.
-
-    const OptionalPathSetting rootDir{(StoreConfig*) this, std::nullopt,
+    const OptionalPathSetting rootDir{this, std::nullopt,
         "root",
         "Directory prefixed to all other paths."};
 
-    const PathSetting stateDir{(StoreConfig*) this,
+    const PathSetting stateDir{this,
         rootDir.get() ? *rootDir.get() + "/nix/var/nix" : settings.nixStateDir,
         "state",
         "Directory where Nix will store state."};
 
-    const PathSetting logDir{(StoreConfig*) this,
+    const PathSetting logDir{this,
         rootDir.get() ? *rootDir.get() + "/nix/var/log/nix" : settings.nixLogDir,
         "log",
         "directory where Nix will store log files."};
 
-    const PathSetting realStoreDir{(StoreConfig*) this,
+    const PathSetting realStoreDir{this,
         rootDir.get() ? *rootDir.get() + "/nix/store" : storeDir, "real",
         "Physical path of the Nix store."};
 };
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index e97195f5b..fe26a0f27 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -40,12 +40,12 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig
 {
     using LocalFSStoreConfig::LocalFSStoreConfig;
 
-    Setting<bool> requireSigs{(StoreConfig*) this,
+    Setting<bool> requireSigs{this,
         settings.requireSigs,
         "require-sigs",
         "Whether store paths copied into this store should have a trusted signature."};
 
-    Setting<bool> readOnly{(StoreConfig*) this,
+    Setting<bool> readOnly{this,
         false,
         "read-only",
         R"(
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index a1ae82a0f..f0985fdc1 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -22,10 +22,10 @@ struct RemoteStoreConfig : virtual StoreConfig
 {
     using StoreConfig::StoreConfig;
 
-    const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections",
+    const Setting<int> maxConnections{this, 1, "max-connections",
         "Maximum number of concurrent connections to the Nix daemon."};
 
-    const Setting<unsigned int> maxConnectionAge{(StoreConfig*) this,
+    const Setting<unsigned int> maxConnectionAge{this,
         std::numeric_limits<unsigned int>::max(),
         "max-connection-age",
         "Maximum age of a connection before it is closed."};
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index d2fc6abaf..1a62d92d4 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -193,20 +193,20 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
 {
     using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
 
-    const Setting<std::string> profile{(StoreConfig*) this, "", "profile",
+    const Setting<std::string> profile{this, "", "profile",
         R"(
           The name of the AWS configuration profile to use. By default
           Nix will use the `default` profile.
         )"};
 
-    const Setting<std::string> region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region",
+    const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region",
         R"(
           The region of the S3 bucket. If your bucket is not in
           `us–east-1`, you should always explicitly specify the region
           parameter.
         )"};
 
-    const Setting<std::string> scheme{(StoreConfig*) this, "", "scheme",
+    const Setting<std::string> scheme{this, "", "scheme",
         R"(
           The scheme used for S3 requests, `https` (default) or `http`. This
           option allows you to disable HTTPS for binary caches which don't
@@ -218,7 +218,7 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
           > information.
         )"};
 
-    const Setting<std::string> endpoint{(StoreConfig*) this, "", "endpoint",
+    const Setting<std::string> endpoint{this, "", "endpoint",
         R"(
           The URL of the endpoint of an S3-compatible service such as MinIO.
           Do not specify this setting if you're using Amazon S3.
@@ -229,13 +229,13 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
           > addressing instead of virtual host based addressing.
         )"};
 
-    const Setting<std::string> narinfoCompression{(StoreConfig*) this, "", "narinfo-compression",
+    const Setting<std::string> narinfoCompression{this, "", "narinfo-compression",
         "Compression method for `.narinfo` files."};
 
-    const Setting<std::string> lsCompression{(StoreConfig*) this, "", "ls-compression",
+    const Setting<std::string> lsCompression{this, "", "ls-compression",
         "Compression method for `.ls` files."};
 
-    const Setting<std::string> logCompression{(StoreConfig*) this, "", "log-compression",
+    const Setting<std::string> logCompression{this, "", "log-compression",
         R"(
           Compression method for `log/*` files. It is recommended to
           use a compression method supported by most web browsers
@@ -243,11 +243,11 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
         )"};
 
     const Setting<bool> multipartUpload{
-        (StoreConfig*) this, false, "multipart-upload",
+        this, false, "multipart-upload",
         "Whether to use multi-part uploads."};
 
     const Setting<uint64_t> bufferSize{
-        (StoreConfig*) this, 5 * 1024 * 1024, "buffer-size",
+        this, 5 * 1024 * 1024, "buffer-size",
         "Size (in bytes) of each part in multi-part uploads."};
 
     const std::string name() override { return "S3 Binary Cache Store"; }
diff --git a/src/libstore/ssh-store-config.hh b/src/libstore/ssh-store-config.hh
index c27a5d00f..bf55d20cf 100644
--- a/src/libstore/ssh-store-config.hh
+++ b/src/libstore/ssh-store-config.hh
@@ -9,16 +9,16 @@ struct CommonSSHStoreConfig : virtual StoreConfig
 {
     using StoreConfig::StoreConfig;
 
-    const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key",
+    const Setting<Path> sshKey{this, "", "ssh-key",
         "Path to the SSH private key used to authenticate to the remote machine."};
 
-    const Setting<std::string> sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key",
+    const Setting<std::string> sshPublicHostKey{this, "", "base64-ssh-public-host-key",
         "The public host key of the remote machine."};
 
-    const Setting<bool> compress{(StoreConfig*) this, false, "compress",
+    const Setting<bool> compress{this, false, "compress",
         "Whether to enable SSH compression."};
 
-    const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store",
+    const Setting<std::string> remoteStore{this, "", "remote-store",
         R"(
           [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
           to be used on the remote machine. The default is `auto`
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index 9c6c42ef4..4a6aad449 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -16,7 +16,7 @@ struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig
     using RemoteStoreConfig::RemoteStoreConfig;
     using CommonSSHStoreConfig::CommonSSHStoreConfig;
 
-    const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program",
+    const Setting<Path> remoteProgram{this, "nix-daemon", "remote-program",
         "Path to the `nix-daemon` executable on the remote machine."};
 
     const std::string name() override { return "Experimental SSH Store"; }

From 1093d6585ff6478e50a5845de64cfcf114e35a95 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 31 Oct 2023 20:39:39 -0400
Subject: [PATCH 032/294] Make `ParseSink` a bit better

I wouldn't call it *good* yet, but this will do for now.

- `RetrieveRegularNARSink` renamed to `RegularFileSink` and moved
  accordingly because it actually has nothing to do with NARs in
  particular.

  - its `fd` field is also marked private

- `copyRecursive` introduced to dump a `SourceAccessor` into a
  `ParseSink`.

- `NullParseSink` made so `ParseSink` no longer has sketchy default
  methods.

This was done while updating #8918 to work with the new
`SourceAccessor`.
---
 src/libstore/daemon.cc        |  6 +--
 src/libstore/export-import.cc |  2 +-
 src/libstore/local-store.cc   |  2 +-
 src/libstore/store-api.cc     |  8 ++--
 src/libutil/archive.cc        |  8 +---
 src/libutil/archive.hh        | 27 -----------
 src/libutil/fs-sink.cc        | 48 +++++++++++++++++++
 src/libutil/fs-sink.hh        | 86 ++++++++++++++++++++++++++++++-----
 8 files changed, 132 insertions(+), 55 deletions(-)

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 007ffc05a..105d92f25 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -454,13 +454,13 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                        eagerly consume the entire stream it's given, past the
                        length of the Nar. */
                     TeeSource savedNARSource(from, saved);
-                    ParseSink sink; /* null sink; just parse the NAR */
+                    NullParseSink sink; /* just parse the NAR */
                     parseDump(sink, savedNARSource);
                 } else {
                     /* Incrementally parse the NAR file, stripping the
                        metadata, and streaming the sole file we expect into
                        `saved`. */
-                    RetrieveRegularNARSink savedRegular { saved };
+                    RegularFileSink savedRegular { saved };
                     parseDump(savedRegular, from);
                     if (!savedRegular.regular) throw Error("regular file expected");
                 }
@@ -899,7 +899,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                 source = std::make_unique<TunnelSource>(from, to);
             else {
                 TeeSource tee { from, saved };
-                ParseSink ether;
+                NullParseSink ether;
                 parseDump(ether, tee);
                 source = std::make_unique<StringSource>(saved.s);
             }
diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc
index 91b7e30db..52130f8f6 100644
--- a/src/libstore/export-import.cc
+++ b/src/libstore/export-import.cc
@@ -65,7 +65,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
         /* Extract the NAR from the source. */
         StringSink saved;
         TeeSource tee { source, saved };
-        ParseSink ether;
+        NullParseSink ether;
         parseDump(ether, tee);
 
         uint32_t magic = readInt(source);
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 1c2f6023a..a5e9426f8 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1200,7 +1200,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
     bool narRead = false;
     Finally cleanup = [&]() {
         if (!narRead) {
-            ParseSink sink;
+            NullParseSink sink;
             parseDump(sink, source);
         }
     };
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 0399120d1..e6a4cf9d9 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -410,7 +410,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
     /* Note that fileSink and unusualHashTee must be mutually exclusive, since
        they both write to caHashSink. Note that that requisite is currently true
        because the former is only used in the flat case. */
-    RetrieveRegularNARSink fileSink { caHashSink };
+    RegularFileSink fileSink { caHashSink };
     TeeSink unusualHashTee { narHashSink, caHashSink };
 
     auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != htSHA256
@@ -428,10 +428,10 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
        information to narSink. */
     TeeSource tapped { *fileSource, narSink };
 
-    ParseSink blank;
+    NullParseSink blank;
     auto & parseSink = method == FileIngestionMethod::Flat
-        ? fileSink
-        : blank;
+        ? (ParseSink &) fileSink
+        : (ParseSink &) blank;
 
     /* The information that flows from tapped (besides being replicated in
        narSink), is now put in parseSink. */
diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index 3b1a1e0ef..4ca84d357 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -5,12 +5,6 @@
 
 #include <strings.h> // for strcasecmp
 
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <unistd.h>
-#include <dirent.h>
-#include <fcntl.h>
-
 #include "archive.hh"
 #include "util.hh"
 #include "config.hh"
@@ -299,7 +293,7 @@ void copyNAR(Source & source, Sink & sink)
     // FIXME: if 'source' is the output of dumpPath() followed by EOF,
     // we should just forward all data directly without parsing.
 
-    ParseSink parseSink; /* null sink; just parse the NAR */
+    NullParseSink parseSink; /* just parse the NAR */
 
     TeeSource wrapper { source, sink };
 
diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh
index 3530783c1..2cf8ee891 100644
--- a/src/libutil/archive.hh
+++ b/src/libutil/archive.hh
@@ -73,33 +73,6 @@ time_t dumpPathAndGetMtime(const Path & path, Sink & sink,
  */
 void dumpString(std::string_view s, Sink & sink);
 
-/**
- * If the NAR archive contains a single file at top-level, then save
- * the contents of the file to `s`.  Otherwise barf.
- */
-struct RetrieveRegularNARSink : ParseSink
-{
-    bool regular = true;
-    Sink & sink;
-
-    RetrieveRegularNARSink(Sink & sink) : sink(sink) { }
-
-    void createDirectory(const Path & path) override
-    {
-        regular = false;
-    }
-
-    void receiveContents(std::string_view data) override
-    {
-        sink(data);
-    }
-
-    void createSymlink(const Path & path, const std::string & target) override
-    {
-        regular = false;
-    }
-};
-
 void parseDump(ParseSink & sink, Source & source);
 
 void restorePath(const Path & path, Source & source);
diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc
index a08a723a4..925e6f05d 100644
--- a/src/libutil/fs-sink.cc
+++ b/src/libutil/fs-sink.cc
@@ -5,6 +5,54 @@
 
 namespace nix {
 
+void copyRecursive(
+    SourceAccessor & accessor, const CanonPath & from,
+    ParseSink & sink, const Path & to)
+{
+    auto stat = accessor.lstat(from);
+
+    switch (stat.type) {
+    case SourceAccessor::tSymlink:
+    {
+        sink.createSymlink(to, accessor.readLink(from));
+    }
+
+    case SourceAccessor::tRegular:
+    {
+        sink.createRegularFile(to);
+        if (stat.isExecutable)
+            sink.isExecutable();
+        LambdaSink sink2 {
+            [&](auto d) {
+                sink.receiveContents(d);
+            }
+        };
+        accessor.readFile(from, sink2, [&](uint64_t size) {
+            sink.preallocateContents(size);
+        });
+        break;
+    }
+
+    case SourceAccessor::tDirectory:
+    {
+        sink.createDirectory(to);
+        for (auto & [name, _] : accessor.readDirectory(from)) {
+            copyRecursive(
+                accessor, from + name,
+                sink, to + "/" + name);
+            break;
+        }
+    }
+
+    case SourceAccessor::tMisc:
+        throw Error("file '%1%' has an unsupported type", from);
+
+    default:
+        abort();
+    }
+}
+
+
 struct RestoreSinkSettings : Config
 {
     Setting<bool> preallocateContents{this, false, "preallocate-contents",
diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh
index 6837e2fc4..c22edd390 100644
--- a/src/libutil/fs-sink.hh
+++ b/src/libutil/fs-sink.hh
@@ -3,6 +3,7 @@
 
 #include "types.hh"
 #include "serialise.hh"
+#include "source-accessor.hh"
 
 namespace nix {
 
@@ -11,32 +12,93 @@ namespace nix {
  */
 struct ParseSink
 {
-    virtual void createDirectory(const Path & path) { };
+    virtual void createDirectory(const Path & path) = 0;
 
-    virtual void createRegularFile(const Path & path) { };
-    virtual void closeRegularFile() { };
-    virtual void isExecutable() { };
+    virtual void createRegularFile(const Path & path) = 0;
+    virtual void receiveContents(std::string_view data) = 0;
+    virtual void isExecutable() = 0;
+    virtual void closeRegularFile() = 0;
+
+    virtual void createSymlink(const Path & path, const std::string & target) = 0;
+
+    /**
+     * An optimization. By default, do nothing.
+     */
     virtual void preallocateContents(uint64_t size) { };
-    virtual void receiveContents(std::string_view data) { };
-
-    virtual void createSymlink(const Path & path, const std::string & target) { };
 };
 
+/**
+ * Recusively copy file system objects from the source into the sink.
+ */
+void copyRecursive(
+    SourceAccessor & accessor, const CanonPath & sourcePath,
+    ParseSink & sink, const Path & destPath);
+
+/**
+ * Ignore everything and do nothing
+ */
+struct NullParseSink : ParseSink
+{
+    void createDirectory(const Path & path) override { }
+    void receiveContents(std::string_view data) override { }
+    void createSymlink(const Path & path, const std::string & target) override { }
+    void createRegularFile(const Path & path) override { }
+    void closeRegularFile() override { }
+    void isExecutable() override { }
+};
+
+/**
+ * Write files at the given path
+ */
 struct RestoreSink : ParseSink
 {
     Path dstPath;
-    AutoCloseFD fd;
-
 
     void createDirectory(const Path & path) override;
 
     void createRegularFile(const Path & path) override;
-    void closeRegularFile() override;
-    void isExecutable() override;
-    void preallocateContents(uint64_t size) override;
     void receiveContents(std::string_view data) override;
+    void isExecutable() override;
+    void closeRegularFile() override;
 
     void createSymlink(const Path & path, const std::string & target) override;
+
+    void preallocateContents(uint64_t size) override;
+
+private:
+    AutoCloseFD fd;
+};
+
+/**
+ * Restore a single file at the top level, passing along
+ * `receiveContents` to the underlying `Sink`. For anything but a single
+ * file, set `regular = true` so the caller can fail accordingly.
+ */
+struct RegularFileSink : ParseSink
+{
+    bool regular = true;
+    Sink & sink;
+
+    RegularFileSink(Sink & sink) : sink(sink) { }
+
+    void createDirectory(const Path & path) override
+    {
+        regular = false;
+    }
+
+    void receiveContents(std::string_view data) override
+    {
+        sink(data);
+    }
+
+    void createSymlink(const Path & path, const std::string & target) override
+    {
+        regular = false;
+    }
+
+    void createRegularFile(const Path & path) override { }
+    void closeRegularFile() override { }
+    void isExecutable() override { }
 };
 
 }

From bc4a1695ac71483831ac9ad591c872105794e88f Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Wed, 1 Nov 2023 11:44:16 +0100
Subject: [PATCH 033/294] doc/hacking: Fix clangd for tests

---
 doc/manual/src/contributing/hacking.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md
index 38c144fcc..fe08ceb94 100644
--- a/doc/manual/src/contributing/hacking.md
+++ b/doc/manual/src/contributing/hacking.md
@@ -210,7 +210,7 @@ See [supported compilation environments](#compilation-environments) and instruct
 To use the LSP with your editor, you first need to [set up `clangd`](https://clangd.llvm.org/installation#project-setup) by running:
 
 ```console
-make clean && bear -- make -j$NIX_BUILD_CORES install
+make clean && bear -- make -j$NIX_BUILD_CORES default check install
 ```
 
 Configure your editor to use the `clangd` from the shell, either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration).

From b2ac6fc040223a58f9b923a89798f72b48e310e5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 1 Nov 2023 14:36:40 +0100
Subject: [PATCH 034/294] Remove FSAccessor::Type::tMissing

Instead stat() now returns std::nullopt to denote that the file
doesn't exist.
---
 src/libstore/binary-cache-store.cc |  6 +--
 src/libstore/fs-accessor.hh        | 19 +++++----
 src/libstore/local-fs-store.cc     |  8 ++--
 src/libstore/nar-accessor.cc       | 68 +++++++++++++++---------------
 src/libstore/remote-fs-accessor.cc |  2 +-
 src/libstore/remote-fs-accessor.hh |  2 +-
 src/nix/cat.cc                     | 11 +++--
 src/nix/ls.cc                      | 26 ++++++------
 src/nix/run.cc                     |  2 +-
 src/nix/why-depends.cc             |  7 +--
 10 files changed, 77 insertions(+), 74 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 2a91233ec..06d89c478 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -208,7 +208,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
 
         std::string buildIdDir = "/lib/debug/.build-id";
 
-        if (narAccessor->stat(buildIdDir).type == FSAccessor::tDirectory) {
+        if (auto st = narAccessor->stat(buildIdDir); st && st->type == FSAccessor::tDirectory) {
 
             ThreadPool threadPool(25);
 
@@ -234,14 +234,14 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
             for (auto & s1 : narAccessor->readDirectory(buildIdDir)) {
                 auto dir = buildIdDir + "/" + s1;
 
-                if (narAccessor->stat(dir).type != FSAccessor::tDirectory
+                if (auto st = narAccessor->stat(dir); !st || st->type != FSAccessor::tDirectory
                     || !std::regex_match(s1, regex1))
                     continue;
 
                 for (auto & s2 : narAccessor->readDirectory(dir)) {
                     auto debugPath = dir + "/" + s2;
 
-                    if (narAccessor->stat(debugPath).type != FSAccessor::tRegular
+                    if (auto st = narAccessor->stat(debugPath); !st || st->type != FSAccessor::tRegular
                         || !std::regex_match(s2, regex2))
                         continue;
 
diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh
index 1df19e647..9bae0be74 100644
--- a/src/libstore/fs-accessor.hh
+++ b/src/libstore/fs-accessor.hh
@@ -3,6 +3,8 @@
 
 #include "types.hh"
 
+#include <optional>
+
 namespace nix {
 
 /**
@@ -12,28 +14,29 @@ namespace nix {
 class FSAccessor
 {
 public:
-    enum Type { tMissing, tRegular, tSymlink, tDirectory };
+    enum Type { tRegular, tSymlink, tDirectory };
 
     struct Stat
     {
-        Type type = tMissing;
+        Type type;
         /**
-         * regular files only
+         * For regular files only: the size of the file.
          */
         uint64_t fileSize = 0;
         /**
-         * regular files only
+         * For regular files only: whether this is an executable.
          */
-        bool isExecutable = false; // regular files only
+        bool isExecutable = false;
         /**
-         * regular files only
+         * For regular files only: the position of the contents of this
+         * file in the NAR.
          */
-        uint64_t narOffset = 0; // regular files only
+        uint64_t narOffset = 0;
     };
 
     virtual ~FSAccessor() { }
 
-    virtual Stat stat(const Path & path) = 0;
+    virtual std::optional<Stat> stat(const Path & path) = 0;
 
     virtual StringSet readDirectory(const Path & path) = 0;
 
diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc
index b224fc3e9..bb83a9cd4 100644
--- a/src/libstore/local-fs-store.cc
+++ b/src/libstore/local-fs-store.cc
@@ -27,25 +27,25 @@ struct LocalStoreAccessor : public FSAccessor
         return store->getRealStoreDir() + std::string(path, store->storeDir.size());
     }
 
-    FSAccessor::Stat stat(const Path & path) override
+    std::optional<FSAccessor::Stat> stat(const Path & path) override
     {
         auto realPath = toRealPath(path);
 
         struct stat st;
         if (lstat(realPath.c_str(), &st)) {
-            if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false};
+            if (errno == ENOENT || errno == ENOTDIR) return std::nullopt;
             throw SysError("getting status of '%1%'", path);
         }
 
         if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode))
             throw Error("file '%1%' has unsupported type", path);
 
-        return {
+        return {{
             S_ISREG(st.st_mode) ? Type::tRegular :
             S_ISLNK(st.st_mode) ? Type::tSymlink :
             Type::tDirectory,
             S_ISREG(st.st_mode) ? (uint64_t) st.st_size : 0,
-            S_ISREG(st.st_mode) && st.st_mode & S_IXUSR};
+            S_ISREG(st.st_mode) && st.st_mode & S_IXUSR}};
     }
 
     StringSet readDirectory(const Path & path) override
diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc
index f0dfcb19b..9123bd59d 100644
--- a/src/libstore/nar-accessor.cc
+++ b/src/libstore/nar-accessor.cc
@@ -11,13 +11,7 @@ namespace nix {
 
 struct NarMember
 {
-    FSAccessor::Type type = FSAccessor::Type::tMissing;
-
-    bool isExecutable = false;
-
-    /* If this is a regular file, position of the contents of this
-       file in the NAR. */
-    uint64_t start = 0, size = 0;
+    FSAccessor::Stat stat;
 
     std::string target;
 
@@ -57,7 +51,7 @@ struct NarAccessor : public FSAccessor
                 acc.root = std::move(member);
                 parents.push(&acc.root);
             } else {
-                if (parents.top()->type != FSAccessor::Type::tDirectory)
+                if (parents.top()->stat.type != FSAccessor::Type::tDirectory)
                     throw Error("NAR file missing parent directory of path '%s'", path);
                 auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member));
                 parents.push(&result.first->second);
@@ -79,14 +73,15 @@ struct NarAccessor : public FSAccessor
 
         void isExecutable() override
         {
-            parents.top()->isExecutable = true;
+            parents.top()->stat.isExecutable = true;
         }
 
         void preallocateContents(uint64_t size) override
         {
             assert(size <= std::numeric_limits<uint64_t>::max());
-            parents.top()->size = (uint64_t) size;
-            parents.top()->start = pos;
+            auto & st = parents.top()->stat;
+            st.fileSize = (uint64_t) size;
+            st.narOffset = pos;
         }
 
         void receiveContents(std::string_view data) override
@@ -95,7 +90,9 @@ struct NarAccessor : public FSAccessor
         void createSymlink(const Path & path, const std::string & target) override
         {
             createMember(path,
-                NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
+                NarMember{
+                    .stat = {.type = FSAccessor::Type::tSymlink},
+                    .target = target});
         }
 
         size_t read(char * data, size_t len) override
@@ -130,18 +127,20 @@ struct NarAccessor : public FSAccessor
             std::string type = v["type"];
 
             if (type == "directory") {
-                member.type = FSAccessor::Type::tDirectory;
+                member.stat = {.type = FSAccessor::Type::tDirectory};
                 for (auto i = v["entries"].begin(); i != v["entries"].end(); ++i) {
                     std::string name = i.key();
                     recurse(member.children[name], i.value());
                 }
             } else if (type == "regular") {
-                member.type = FSAccessor::Type::tRegular;
-                member.size = v["size"];
-                member.isExecutable = v.value("executable", false);
-                member.start = v["narOffset"];
+                member.stat = {
+                    .type = FSAccessor::Type::tRegular,
+                    .fileSize = v["size"],
+                    .isExecutable = v.value("executable", false),
+                    .narOffset = v["narOffset"]
+                };
             } else if (type == "symlink") {
-                member.type = FSAccessor::Type::tSymlink;
+                member.stat = {.type = FSAccessor::Type::tSymlink};
                 member.target = v.value("target", "");
             } else return;
         };
@@ -158,7 +157,7 @@ struct NarAccessor : public FSAccessor
         for (auto it = path.begin(); it != end; ) {
             // because it != end, the remaining component is non-empty so we need
             // a directory
-            if (current->type != FSAccessor::Type::tDirectory) return nullptr;
+            if (current->stat.type != FSAccessor::Type::tDirectory) return nullptr;
 
             // skip slash (canonPath above ensures that this is always a slash)
             assert(*it == '/');
@@ -183,19 +182,19 @@ struct NarAccessor : public FSAccessor
         return *result;
     }
 
-    Stat stat(const Path & path) override
+    std::optional<Stat> stat(const Path & path) override
     {
         auto i = find(path);
         if (i == nullptr)
-            return {FSAccessor::Type::tMissing, 0, false};
-        return {i->type, i->size, i->isExecutable, i->start};
+            return std::nullopt;
+        return i->stat;
     }
 
     StringSet readDirectory(const Path & path) override
     {
         auto i = get(path);
 
-        if (i.type != FSAccessor::Type::tDirectory)
+        if (i.stat.type != FSAccessor::Type::tDirectory)
             throw Error("path '%1%' inside NAR file is not a directory", path);
 
         StringSet res;
@@ -208,19 +207,19 @@ struct NarAccessor : public FSAccessor
     std::string readFile(const Path & path, bool requireValidPath = true) override
     {
         auto i = get(path);
-        if (i.type != FSAccessor::Type::tRegular)
+        if (i.stat.type != FSAccessor::Type::tRegular)
             throw Error("path '%1%' inside NAR file is not a regular file", path);
 
-        if (getNarBytes) return getNarBytes(i.start, i.size);
+        if (getNarBytes) return getNarBytes(i.stat.narOffset, i.stat.fileSize);
 
         assert(nar);
-        return std::string(*nar, i.start, i.size);
+        return std::string(*nar, i.stat.narOffset, i.stat.fileSize);
     }
 
     std::string readLink(const Path & path) override
     {
         auto i = get(path);
-        if (i.type != FSAccessor::Type::tSymlink)
+        if (i.stat.type != FSAccessor::Type::tSymlink)
             throw Error("path '%1%' inside NAR file is not a symlink", path);
         return i.target;
     }
@@ -246,17 +245,19 @@ using nlohmann::json;
 json listNar(ref<FSAccessor> accessor, const Path & path, bool recurse)
 {
     auto st = accessor->stat(path);
+    if (!st)
+        throw Error("path '%s' does not exist in NAR", path);
 
     json obj = json::object();
 
-    switch (st.type) {
+    switch (st->type) {
     case FSAccessor::Type::tRegular:
         obj["type"] = "regular";
-        obj["size"] = st.fileSize;
-        if (st.isExecutable)
+        obj["size"] = st->fileSize;
+        if (st->isExecutable)
             obj["executable"] = true;
-        if (st.narOffset)
-            obj["narOffset"] = st.narOffset;
+        if (st->narOffset)
+            obj["narOffset"] = st->narOffset;
         break;
     case FSAccessor::Type::tDirectory:
         obj["type"] = "directory";
@@ -275,9 +276,6 @@ json listNar(ref<FSAccessor> accessor, const Path & path, bool recurse)
         obj["type"] = "symlink";
         obj["target"] = accessor->readLink(path);
         break;
-    case FSAccessor::Type::tMissing:
-    default:
-        throw Error("path '%s' does not exist in NAR", path);
     }
     return obj;
 }
diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc
index fcfb527f5..6c87ebeaa 100644
--- a/src/libstore/remote-fs-accessor.cc
+++ b/src/libstore/remote-fs-accessor.cc
@@ -101,7 +101,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_, boo
     return {addToCache(storePath.hashPart(), std::move(sink.s)), restPath};
 }
 
-FSAccessor::Stat RemoteFSAccessor::stat(const Path & path)
+std::optional<FSAccessor::Stat> RemoteFSAccessor::stat(const Path & path)
 {
     auto res = fetch(path);
     return res.first->stat(res.second);
diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh
index e2673b6f6..5cf759aa0 100644
--- a/src/libstore/remote-fs-accessor.hh
+++ b/src/libstore/remote-fs-accessor.hh
@@ -28,7 +28,7 @@ public:
     RemoteFSAccessor(ref<Store> store,
         const /* FIXME: use std::optional */ Path & cacheDir = "");
 
-    Stat stat(const Path & path) override;
+    std::optional<Stat> stat(const Path & path) override;
 
     StringSet readDirectory(const Path & path) override;
 
diff --git a/src/nix/cat.cc b/src/nix/cat.cc
index 60aa66ce0..b5fe2506f 100644
--- a/src/nix/cat.cc
+++ b/src/nix/cat.cc
@@ -11,13 +11,12 @@ struct MixCat : virtual Args
 
     void cat(ref<FSAccessor> accessor)
     {
-        auto st = accessor->stat(path);
-        if (st.type == FSAccessor::Type::tMissing)
+        if (auto st = accessor->stat(path)) {
+            if (st->type != FSAccessor::Type::tRegular)
+                throw Error("path '%1%' is not a regular file", path);
+            writeFull(STDOUT_FILENO, accessor->readFile(path));
+        } else
             throw Error("path '%1%' does not exist", path);
-        if (st.type != FSAccessor::Type::tRegular)
-            throw Error("path '%1%' is not a regular file", path);
-
-        writeFull(STDOUT_FILENO, accessor->readFile(path));
     }
 };
 
diff --git a/src/nix/ls.cc b/src/nix/ls.cc
index c990a303c..8dc8a47b4 100644
--- a/src/nix/ls.cc
+++ b/src/nix/ls.cc
@@ -46,23 +46,25 @@ struct MixLs : virtual Args, MixJSON
         auto showFile = [&](const Path & curPath, const std::string & relPath) {
             if (verbose) {
                 auto st = accessor->stat(curPath);
+                assert(st);
                 std::string tp =
-                    st.type == FSAccessor::Type::tRegular ?
-                        (st.isExecutable ? "-r-xr-xr-x" : "-r--r--r--") :
-                    st.type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" :
+                    st->type == FSAccessor::Type::tRegular ?
+                        (st->isExecutable ? "-r-xr-xr-x" : "-r--r--r--") :
+                    st->type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" :
                     "dr-xr-xr-x";
-                auto line = fmt("%s %20d %s", tp, st.fileSize, relPath);
-                if (st.type == FSAccessor::Type::tSymlink)
+                auto line = fmt("%s %20d %s", tp, st->fileSize, relPath);
+                if (st->type == FSAccessor::Type::tSymlink)
                     line += " -> " + accessor->readLink(curPath);
                 logger->cout(line);
-                if (recursive && st.type == FSAccessor::Type::tDirectory)
-                    doPath(st, curPath, relPath, false);
+                if (recursive && st->type == FSAccessor::Type::tDirectory)
+                    doPath(*st, curPath, relPath, false);
             } else {
                 logger->cout(relPath);
                 if (recursive) {
                     auto st = accessor->stat(curPath);
-                    if (st.type == FSAccessor::Type::tDirectory)
-                        doPath(st, curPath, relPath, false);
+                    assert(st);
+                    if (st->type == FSAccessor::Type::tDirectory)
+                        doPath(*st, curPath, relPath, false);
                 }
             }
         };
@@ -79,10 +81,10 @@ struct MixLs : virtual Args, MixJSON
         };
 
         auto st = accessor->stat(path);
-        if (st.type == FSAccessor::Type::tMissing)
+        if (!st)
             throw Error("path '%1%' does not exist", path);
-        doPath(st, path,
-            st.type == FSAccessor::Type::tDirectory ? "." : std::string(baseNameOf(path)),
+        doPath(*st, path,
+            st->type == FSAccessor::Type::tDirectory ? "." : std::string(baseNameOf(path)),
             showDirectory);
     }
 
diff --git a/src/nix/run.cc b/src/nix/run.cc
index 1baf299ab..f6c229adc 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -120,7 +120,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment
                 unixPath.push_front(store->printStorePath(path) + "/bin");
 
             auto propPath = store->printStorePath(path) + "/nix-support/propagated-user-env-packages";
-            if (accessor->stat(propPath).type == FSAccessor::tRegular) {
+            if (auto st = accessor->stat(propPath); st && st->type == FSAccessor::tRegular) {
                 for (auto & p : tokenizeString<Paths>(readFile(propPath)))
                     todo.push(store->parseStorePath(p));
             }
diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc
index 055cf6d0d..912ba72fb 100644
--- a/src/nix/why-depends.cc
+++ b/src/nix/why-depends.cc
@@ -214,6 +214,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions
 
             visitPath = [&](const Path & p) {
                 auto st = accessor->stat(p);
+                assert(st);
 
                 auto p2 = p == pathS ? "/" : std::string(p, pathS.size() + 1);
 
@@ -221,13 +222,13 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions
                     return hash == dependencyPathHash ? ANSI_GREEN : ANSI_BLUE;
                 };
 
-                if (st.type == FSAccessor::Type::tDirectory) {
+                if (st->type == FSAccessor::Type::tDirectory) {
                     auto names = accessor->readDirectory(p);
                     for (auto & name : names)
                         visitPath(p + "/" + name);
                 }
 
-                else if (st.type == FSAccessor::Type::tRegular) {
+                else if (st->type == FSAccessor::Type::tRegular) {
                     auto contents = accessor->readFile(p);
 
                     for (auto & hash : hashes) {
@@ -245,7 +246,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions
                     }
                 }
 
-                else if (st.type == FSAccessor::Type::tSymlink) {
+                else if (st->type == FSAccessor::Type::tSymlink) {
                     auto target = accessor->readLink(p);
 
                     for (auto & hash : hashes) {

From 8ffd1695ce31ff81b038fdc995dd8da03b180f03 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 1 Nov 2023 14:43:20 +0100
Subject: [PATCH 035/294] Unify FSAccessor::Type and SourceAccessor::Type

---
 src/libstore/binary-cache-store.cc |  6 +++---
 src/libstore/fs-accessor.hh        |  3 ++-
 src/libstore/nar-accessor.cc       | 30 ++++++++++++++++--------------
 src/nix/run.cc                     |  2 +-
 4 files changed, 22 insertions(+), 19 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 06d89c478..f9abd8cbd 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -208,7 +208,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
 
         std::string buildIdDir = "/lib/debug/.build-id";
 
-        if (auto st = narAccessor->stat(buildIdDir); st && st->type == FSAccessor::tDirectory) {
+        if (auto st = narAccessor->stat(buildIdDir); st && st->type == SourceAccessor::tDirectory) {
 
             ThreadPool threadPool(25);
 
@@ -234,14 +234,14 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
             for (auto & s1 : narAccessor->readDirectory(buildIdDir)) {
                 auto dir = buildIdDir + "/" + s1;
 
-                if (auto st = narAccessor->stat(dir); !st || st->type != FSAccessor::tDirectory
+                if (auto st = narAccessor->stat(dir); !st || st->type != SourceAccessor::tDirectory
                     || !std::regex_match(s1, regex1))
                     continue;
 
                 for (auto & s2 : narAccessor->readDirectory(dir)) {
                     auto debugPath = dir + "/" + s2;
 
-                    if (auto st = narAccessor->stat(debugPath); !st || st->type != FSAccessor::tRegular
+                    if (auto st = narAccessor->stat(debugPath); !st || st->type != SourceAccessor::tRegular
                         || !std::regex_match(s2, regex2))
                         continue;
 
diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh
index 9bae0be74..1e951ec57 100644
--- a/src/libstore/fs-accessor.hh
+++ b/src/libstore/fs-accessor.hh
@@ -2,6 +2,7 @@
 ///@file
 
 #include "types.hh"
+#include "source-accessor.hh"
 
 #include <optional>
 
@@ -14,7 +15,7 @@ namespace nix {
 class FSAccessor
 {
 public:
-    enum Type { tRegular, tSymlink, tDirectory };
+    using Type = SourceAccessor::Type;
 
     struct Stat
     {
diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc
index 9123bd59d..43a78a362 100644
--- a/src/libstore/nar-accessor.cc
+++ b/src/libstore/nar-accessor.cc
@@ -51,7 +51,7 @@ struct NarAccessor : public FSAccessor
                 acc.root = std::move(member);
                 parents.push(&acc.root);
             } else {
-                if (parents.top()->stat.type != FSAccessor::Type::tDirectory)
+                if (parents.top()->stat.type != Type::tDirectory)
                     throw Error("NAR file missing parent directory of path '%s'", path);
                 auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member));
                 parents.push(&result.first->second);
@@ -60,12 +60,12 @@ struct NarAccessor : public FSAccessor
 
         void createDirectory(const Path & path) override
         {
-            createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0});
+            createMember(path, {Type::tDirectory, false, 0, 0});
         }
 
         void createRegularFile(const Path & path) override
         {
-            createMember(path, {FSAccessor::Type::tRegular, false, 0, 0});
+            createMember(path, {Type::tRegular, false, 0, 0});
         }
 
         void closeRegularFile() override
@@ -91,7 +91,7 @@ struct NarAccessor : public FSAccessor
         {
             createMember(path,
                 NarMember{
-                    .stat = {.type = FSAccessor::Type::tSymlink},
+                    .stat = {.type = Type::tSymlink},
                     .target = target});
         }
 
@@ -127,20 +127,20 @@ struct NarAccessor : public FSAccessor
             std::string type = v["type"];
 
             if (type == "directory") {
-                member.stat = {.type = FSAccessor::Type::tDirectory};
+                member.stat = {.type = Type::tDirectory};
                 for (auto i = v["entries"].begin(); i != v["entries"].end(); ++i) {
                     std::string name = i.key();
                     recurse(member.children[name], i.value());
                 }
             } else if (type == "regular") {
                 member.stat = {
-                    .type = FSAccessor::Type::tRegular,
+                    .type = Type::tRegular,
                     .fileSize = v["size"],
                     .isExecutable = v.value("executable", false),
                     .narOffset = v["narOffset"]
                 };
             } else if (type == "symlink") {
-                member.stat = {.type = FSAccessor::Type::tSymlink};
+                member.stat = {.type = Type::tSymlink};
                 member.target = v.value("target", "");
             } else return;
         };
@@ -157,7 +157,7 @@ struct NarAccessor : public FSAccessor
         for (auto it = path.begin(); it != end; ) {
             // because it != end, the remaining component is non-empty so we need
             // a directory
-            if (current->stat.type != FSAccessor::Type::tDirectory) return nullptr;
+            if (current->stat.type != Type::tDirectory) return nullptr;
 
             // skip slash (canonPath above ensures that this is always a slash)
             assert(*it == '/');
@@ -194,7 +194,7 @@ struct NarAccessor : public FSAccessor
     {
         auto i = get(path);
 
-        if (i.stat.type != FSAccessor::Type::tDirectory)
+        if (i.stat.type != Type::tDirectory)
             throw Error("path '%1%' inside NAR file is not a directory", path);
 
         StringSet res;
@@ -207,7 +207,7 @@ struct NarAccessor : public FSAccessor
     std::string readFile(const Path & path, bool requireValidPath = true) override
     {
         auto i = get(path);
-        if (i.stat.type != FSAccessor::Type::tRegular)
+        if (i.stat.type != Type::tRegular)
             throw Error("path '%1%' inside NAR file is not a regular file", path);
 
         if (getNarBytes) return getNarBytes(i.stat.narOffset, i.stat.fileSize);
@@ -219,7 +219,7 @@ struct NarAccessor : public FSAccessor
     std::string readLink(const Path & path) override
     {
         auto i = get(path);
-        if (i.stat.type != FSAccessor::Type::tSymlink)
+        if (i.stat.type != Type::tSymlink)
             throw Error("path '%1%' inside NAR file is not a symlink", path);
         return i.target;
     }
@@ -251,7 +251,7 @@ json listNar(ref<FSAccessor> accessor, const Path & path, bool recurse)
     json obj = json::object();
 
     switch (st->type) {
-    case FSAccessor::Type::tRegular:
+    case SourceAccessor::Type::tRegular:
         obj["type"] = "regular";
         obj["size"] = st->fileSize;
         if (st->isExecutable)
@@ -259,7 +259,7 @@ json listNar(ref<FSAccessor> accessor, const Path & path, bool recurse)
         if (st->narOffset)
             obj["narOffset"] = st->narOffset;
         break;
-    case FSAccessor::Type::tDirectory:
+    case SourceAccessor::Type::tDirectory:
         obj["type"] = "directory";
         {
             obj["entries"] = json::object();
@@ -272,10 +272,12 @@ json listNar(ref<FSAccessor> accessor, const Path & path, bool recurse)
             }
         }
         break;
-    case FSAccessor::Type::tSymlink:
+    case SourceAccessor::Type::tSymlink:
         obj["type"] = "symlink";
         obj["target"] = accessor->readLink(path);
         break;
+    case SourceAccessor::Type::tMisc:
+        assert(false); // cannot happen for NARs
     }
     return obj;
 }
diff --git a/src/nix/run.cc b/src/nix/run.cc
index f6c229adc..07806283c 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -120,7 +120,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment
                 unixPath.push_front(store->printStorePath(path) + "/bin");
 
             auto propPath = store->printStorePath(path) + "/nix-support/propagated-user-env-packages";
-            if (auto st = accessor->stat(propPath); st && st->type == FSAccessor::tRegular) {
+            if (auto st = accessor->stat(propPath); st && st->type == SourceAccessor::tRegular) {
                 for (auto & p : tokenizeString<Paths>(readFile(propPath)))
                     todo.push(store->parseStorePath(p));
             }

From cdb27c1519cd802f477e8fa90beabe1bddc4bac7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 1 Nov 2023 15:26:07 +0100
Subject: [PATCH 036/294] SourceAccessor: Change the main interface from
 lstat() to maybeLstat()

---
 src/libexpr/primops.cc                                 |  6 ++----
 src/libfetchers/fs-input-accessor.cc                   |  4 ++--
 src/libfetchers/memory-input-accessor.cc               |  4 ++--
 src/libutil/posix-source-accessor.cc                   |  8 ++++++--
 src/libutil/posix-source-accessor.hh                   |  2 +-
 src/libutil/source-accessor.cc                         | 10 +++++-----
 src/libutil/source-accessor.hh                         |  4 ++--
 .../lang/eval-fail-bad-string-interpolation-2.err.exp  |  2 +-
 tests/functional/lang/eval-fail-nonexist-path.err.exp  |  2 +-
 9 files changed, 22 insertions(+), 20 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 704e7007b..e3c775d90 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1548,10 +1548,8 @@ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args,
 
     try {
         auto checked = state.checkSourcePath(path);
-        auto exists = checked.pathExists();
-        if (exists && mustBeDir) {
-            exists = checked.lstat().type == InputAccessor::tDirectory;
-        }
+        auto st = checked.maybeLstat();
+        auto exists = st && (!mustBeDir || st->type == SourceAccessor::tDirectory);
         v.mkBool(exists);
     } catch (SysError & e) {
         /* Don't give away info from errors while canonicalising
diff --git a/src/libfetchers/fs-input-accessor.cc b/src/libfetchers/fs-input-accessor.cc
index 7638d2d82..81be64482 100644
--- a/src/libfetchers/fs-input-accessor.cc
+++ b/src/libfetchers/fs-input-accessor.cc
@@ -36,11 +36,11 @@ struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor
         return isAllowed(absPath) && PosixSourceAccessor::pathExists(absPath);
     }
 
-    Stat lstat(const CanonPath & path) override
+    std::optional<Stat> maybeLstat(const CanonPath & path) override
     {
         auto absPath = makeAbsPath(path);
         checkAllowed(absPath);
-        return PosixSourceAccessor::lstat(absPath);
+        return PosixSourceAccessor::maybeLstat(absPath);
     }
 
     DirEntries readDirectory(const CanonPath & path) override
diff --git a/src/libfetchers/memory-input-accessor.cc b/src/libfetchers/memory-input-accessor.cc
index 817d063ba..6468ece41 100644
--- a/src/libfetchers/memory-input-accessor.cc
+++ b/src/libfetchers/memory-input-accessor.cc
@@ -20,12 +20,12 @@ struct MemoryInputAccessorImpl : MemoryInputAccessor
         return i != files.end();
     }
 
-    Stat lstat(const CanonPath & path) override
+    std::optional<Stat> maybeLstat(const CanonPath & path) override
     {
         auto i = files.find(path);
         if (i != files.end())
             return Stat { .type = tRegular, .isExecutable = false };
-        throw Error("file '%s' does not exist", path);
+        return std::nullopt;
     }
 
     DirEntries readDirectory(const CanonPath & path) override
diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc
index 48b4fe626..8a8d64f3f 100644
--- a/src/libutil/posix-source-accessor.cc
+++ b/src/libutil/posix-source-accessor.cc
@@ -44,9 +44,13 @@ bool PosixSourceAccessor::pathExists(const CanonPath & path)
     return nix::pathExists(path.abs());
 }
 
-SourceAccessor::Stat PosixSourceAccessor::lstat(const CanonPath & path)
+std::optional<SourceAccessor::Stat> PosixSourceAccessor::maybeLstat(const CanonPath & path)
 {
-    auto st = nix::lstat(path.abs());
+    struct stat st;
+    if (::lstat(path.c_str(), &st)) {
+        if (errno == ENOENT) return std::nullopt;
+        throw SysError("getting status of '%s'", showPath(path));
+    }
     mtime = std::max(mtime, st.st_mtime);
     return Stat {
         .type =
diff --git a/src/libutil/posix-source-accessor.hh b/src/libutil/posix-source-accessor.hh
index 608f96ee2..cf087d26e 100644
--- a/src/libutil/posix-source-accessor.hh
+++ b/src/libutil/posix-source-accessor.hh
@@ -22,7 +22,7 @@ struct PosixSourceAccessor : SourceAccessor
 
     bool pathExists(const CanonPath & path) override;
 
-    Stat lstat(const CanonPath & path) override;
+    std::optional<Stat> maybeLstat(const CanonPath & path) override;
 
     DirEntries readDirectory(const CanonPath & path) override;
 
diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc
index d168a9667..5b0c7dd34 100644
--- a/src/libutil/source-accessor.cc
+++ b/src/libutil/source-accessor.cc
@@ -42,12 +42,12 @@ Hash SourceAccessor::hashPath(
     return sink.finish().first;
 }
 
-std::optional<SourceAccessor::Stat> SourceAccessor::maybeLstat(const CanonPath & path)
+SourceAccessor::Stat SourceAccessor::lstat(const CanonPath & path)
 {
-    // FIXME: merge these into one operation.
-    if (!pathExists(path))
-        return {};
-    return lstat(path);
+    if (auto st = maybeLstat(path))
+        return *st;
+    else
+        throw Error("path '%s' does not exist", showPath(path));
 }
 
 std::string SourceAccessor::showPath(const CanonPath & path)
diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh
index fd823aa39..80bc02b48 100644
--- a/src/libutil/source-accessor.hh
+++ b/src/libutil/source-accessor.hh
@@ -61,9 +61,9 @@ struct SourceAccessor
         bool isExecutable = false; // regular files only
     };
 
-    virtual Stat lstat(const CanonPath & path) = 0;
+    Stat lstat(const CanonPath & path);
 
-    std::optional<Stat> maybeLstat(const CanonPath & path);
+    virtual std::optional<Stat> maybeLstat(const CanonPath & path) = 0;
 
     typedef std::optional<Type> DirEntry;
 
diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp
index dea119ae8..a287067cd 100644
--- a/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp
+++ b/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp
@@ -1 +1 @@
-error: getting status of '/pwd/lang/fnord': No such file or directory
+error: path '/pwd/lang/fnord' does not exist
diff --git a/tests/functional/lang/eval-fail-nonexist-path.err.exp b/tests/functional/lang/eval-fail-nonexist-path.err.exp
index dea119ae8..a287067cd 100644
--- a/tests/functional/lang/eval-fail-nonexist-path.err.exp
+++ b/tests/functional/lang/eval-fail-nonexist-path.err.exp
@@ -1 +1 @@
-error: getting status of '/pwd/lang/fnord': No such file or directory
+error: path '/pwd/lang/fnord' does not exist

From 53811238790f4bb5f9df74bb25047fe5b734a61f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 1 Nov 2023 15:33:19 +0100
Subject: [PATCH 037/294] Unify DirEntries types

---
 src/libstore/binary-cache-store.cc | 4 ++--
 src/libstore/fs-accessor.hh        | 4 +++-
 src/libstore/local-fs-store.cc     | 6 +++---
 src/libstore/nar-accessor.cc       | 8 ++++----
 src/libstore/remote-fs-accessor.cc | 2 +-
 src/libstore/remote-fs-accessor.hh | 2 +-
 src/nix/ls.cc                      | 2 +-
 src/nix/why-depends.cc             | 2 +-
 8 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index f9abd8cbd..b61868413 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -231,14 +231,14 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
             std::regex regex1("^[0-9a-f]{2}$");
             std::regex regex2("^[0-9a-f]{38}\\.debug$");
 
-            for (auto & s1 : narAccessor->readDirectory(buildIdDir)) {
+            for (auto & [s1, _type] : narAccessor->readDirectory(buildIdDir)) {
                 auto dir = buildIdDir + "/" + s1;
 
                 if (auto st = narAccessor->stat(dir); !st || st->type != SourceAccessor::tDirectory
                     || !std::regex_match(s1, regex1))
                     continue;
 
-                for (auto & s2 : narAccessor->readDirectory(dir)) {
+                for (auto & [s2, _type] : narAccessor->readDirectory(dir)) {
                     auto debugPath = dir + "/" + s2;
 
                     if (auto st = narAccessor->stat(debugPath); !st || st->type != SourceAccessor::tRegular
diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh
index 1e951ec57..f04a92206 100644
--- a/src/libstore/fs-accessor.hh
+++ b/src/libstore/fs-accessor.hh
@@ -39,7 +39,9 @@ public:
 
     virtual std::optional<Stat> stat(const Path & path) = 0;
 
-    virtual StringSet readDirectory(const Path & path) = 0;
+    using DirEntries = SourceAccessor::DirEntries;
+
+    virtual DirEntries readDirectory(const Path & path) = 0;
 
     /**
      * Read a file inside the store.
diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc
index bb83a9cd4..65cbb9e35 100644
--- a/src/libstore/local-fs-store.cc
+++ b/src/libstore/local-fs-store.cc
@@ -48,15 +48,15 @@ struct LocalStoreAccessor : public FSAccessor
             S_ISREG(st.st_mode) && st.st_mode & S_IXUSR}};
     }
 
-    StringSet readDirectory(const Path & path) override
+    DirEntries readDirectory(const Path & path) override
     {
         auto realPath = toRealPath(path);
 
         auto entries = nix::readDirectory(realPath);
 
-        StringSet res;
+        DirEntries res;
         for (auto & entry : entries)
-            res.insert(entry.name);
+            res.insert_or_assign(entry.name, std::nullopt);
 
         return res;
     }
diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc
index 43a78a362..fe857a60e 100644
--- a/src/libstore/nar-accessor.cc
+++ b/src/libstore/nar-accessor.cc
@@ -190,16 +190,16 @@ struct NarAccessor : public FSAccessor
         return i->stat;
     }
 
-    StringSet readDirectory(const Path & path) override
+    DirEntries readDirectory(const Path & path) override
     {
         auto i = get(path);
 
         if (i.stat.type != Type::tDirectory)
             throw Error("path '%1%' inside NAR file is not a directory", path);
 
-        StringSet res;
+        DirEntries res;
         for (auto & child : i.children)
-            res.insert(child.first);
+            res.insert_or_assign(child.first, std::nullopt);
 
         return res;
     }
@@ -264,7 +264,7 @@ json listNar(ref<FSAccessor> accessor, const Path & path, bool recurse)
         {
             obj["entries"] = json::object();
             json &res2 = obj["entries"];
-            for (auto & name : accessor->readDirectory(path)) {
+            for (auto & [name, type] : accessor->readDirectory(path)) {
                 if (recurse) {
                     res2[name] = listNar(accessor, path + "/" + name, true);
                 } else
diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc
index 6c87ebeaa..21419700c 100644
--- a/src/libstore/remote-fs-accessor.cc
+++ b/src/libstore/remote-fs-accessor.cc
@@ -107,7 +107,7 @@ std::optional<FSAccessor::Stat> RemoteFSAccessor::stat(const Path & path)
     return res.first->stat(res.second);
 }
 
-StringSet RemoteFSAccessor::readDirectory(const Path & path)
+SourceAccessor::DirEntries RemoteFSAccessor::readDirectory(const Path & path)
 {
     auto res = fetch(path);
     return res.first->readDirectory(res.second);
diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh
index 5cf759aa0..8de3b7bcd 100644
--- a/src/libstore/remote-fs-accessor.hh
+++ b/src/libstore/remote-fs-accessor.hh
@@ -30,7 +30,7 @@ public:
 
     std::optional<Stat> stat(const Path & path) override;
 
-    StringSet readDirectory(const Path & path) override;
+    DirEntries readDirectory(const Path & path) override;
 
     std::string readFile(const Path & path, bool requireValidPath = true) override;
 
diff --git a/src/nix/ls.cc b/src/nix/ls.cc
index 8dc8a47b4..0ca08cea8 100644
--- a/src/nix/ls.cc
+++ b/src/nix/ls.cc
@@ -74,7 +74,7 @@ struct MixLs : virtual Args, MixJSON
         {
             if (st.type == FSAccessor::Type::tDirectory && !showDirectory) {
                 auto names = accessor->readDirectory(curPath);
-                for (auto & name : names)
+                for (auto & [name, type] : names)
                     showFile(curPath + "/" + name, relPath + "/" + name);
             } else
                 showFile(curPath, relPath);
diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc
index 912ba72fb..04c1a0c1c 100644
--- a/src/nix/why-depends.cc
+++ b/src/nix/why-depends.cc
@@ -224,7 +224,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions
 
                 if (st->type == FSAccessor::Type::tDirectory) {
                     auto names = accessor->readDirectory(p);
-                    for (auto & name : names)
+                    for (auto & [name, type] : names)
                         visitPath(p + "/" + name);
                 }
 

From 50aae0a14c5bbbde5785ead8f46b28333e6248ae Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 1 Nov 2023 15:39:40 +0100
Subject: [PATCH 038/294] FSAccessor: Make the fileSize and narOffset fields
 optional

The narOffset field only applies to NAR accessors. The fileSize field
may be too expensive to compute for certain accessors (e.g. libgit).
---
 src/libstore/fs-accessor.hh  |  4 ++--
 src/libstore/nar-accessor.cc | 11 ++++++-----
 src/nix/ls.cc                |  2 +-
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh
index f04a92206..f6c002a2d 100644
--- a/src/libstore/fs-accessor.hh
+++ b/src/libstore/fs-accessor.hh
@@ -23,7 +23,7 @@ public:
         /**
          * For regular files only: the size of the file.
          */
-        uint64_t fileSize = 0;
+        std::optional<uint64_t> fileSize;
         /**
          * For regular files only: whether this is an executable.
          */
@@ -32,7 +32,7 @@ public:
          * For regular files only: the position of the contents of this
          * file in the NAR.
          */
-        uint64_t narOffset = 0;
+        std::optional<uint64_t> narOffset;
     };
 
     virtual ~FSAccessor() { }
diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc
index fe857a60e..f1be5606e 100644
--- a/src/libstore/nar-accessor.cc
+++ b/src/libstore/nar-accessor.cc
@@ -210,10 +210,10 @@ struct NarAccessor : public FSAccessor
         if (i.stat.type != Type::tRegular)
             throw Error("path '%1%' inside NAR file is not a regular file", path);
 
-        if (getNarBytes) return getNarBytes(i.stat.narOffset, i.stat.fileSize);
+        if (getNarBytes) return getNarBytes(*i.stat.narOffset, *i.stat.fileSize);
 
         assert(nar);
-        return std::string(*nar, i.stat.narOffset, i.stat.fileSize);
+        return std::string(*nar, *i.stat.narOffset, *i.stat.fileSize);
     }
 
     std::string readLink(const Path & path) override
@@ -253,11 +253,12 @@ json listNar(ref<FSAccessor> accessor, const Path & path, bool recurse)
     switch (st->type) {
     case SourceAccessor::Type::tRegular:
         obj["type"] = "regular";
-        obj["size"] = st->fileSize;
+        if (st->fileSize)
+            obj["size"] = *st->fileSize;
         if (st->isExecutable)
             obj["executable"] = true;
-        if (st->narOffset)
-            obj["narOffset"] = st->narOffset;
+        if (st->narOffset && *st->narOffset)
+            obj["narOffset"] = *st->narOffset;
         break;
     case SourceAccessor::Type::tDirectory:
         obj["type"] = "directory";
diff --git a/src/nix/ls.cc b/src/nix/ls.cc
index 0ca08cea8..da978f379 100644
--- a/src/nix/ls.cc
+++ b/src/nix/ls.cc
@@ -52,7 +52,7 @@ struct MixLs : virtual Args, MixJSON
                         (st->isExecutable ? "-r-xr-xr-x" : "-r--r--r--") :
                     st->type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" :
                     "dr-xr-xr-x";
-                auto line = fmt("%s %20d %s", tp, st->fileSize, relPath);
+                auto line = fmt("%s %20d %s", tp, st->fileSize.value_or(0), relPath);
                 if (st->type == FSAccessor::Type::tSymlink)
                     line += " -> " + accessor->readLink(curPath);
                 logger->cout(line);

From 581693bdea3981eb0b106c904c7a1fed7f7582ae Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 1 Nov 2023 16:33:22 +0100
Subject: [PATCH 039/294] fmt(): Handle std::string_view

---
 src/libutil/fmt.hh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh
index 727255b45..ac72e47fb 100644
--- a/src/libutil/fmt.hh
+++ b/src/libutil/fmt.hh
@@ -44,6 +44,11 @@ inline std::string fmt(const std::string & s)
     return s;
 }
 
+inline std::string fmt(std::string_view s)
+{
+    return std::string(s);
+}
+
 inline std::string fmt(const char * s)
 {
     return s;

From 1a902f5fa7d4f268d0fec3e44a48ecc2445b3b6b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 1 Nov 2023 17:09:28 +0100
Subject: [PATCH 040/294] Merge FSAccessor into SourceAccessor

---
 src/libstore/binary-cache-store.cc | 24 +++++------
 src/libstore/binary-cache-store.hh |  2 +-
 src/libstore/derivations.cc        |  1 -
 src/libstore/dummy-store.cc        |  2 +-
 src/libstore/fs-accessor.hh        | 58 ---------------------------
 src/libstore/legacy-ssh-store.cc   |  2 +-
 src/libstore/local-fs-store.cc     | 34 +++++++++-------
 src/libstore/local-fs-store.hh     |  2 +-
 src/libstore/nar-accessor.cc       | 64 ++++++++++++------------------
 src/libstore/nar-accessor.hh       | 11 ++---
 src/libstore/remote-fs-accessor.cc | 28 ++++++-------
 src/libstore/remote-fs-accessor.hh | 21 +++++-----
 src/libstore/remote-store.cc       |  2 +-
 src/libstore/remote-store.hh       |  2 +-
 src/libstore/store-api.cc          |  8 ++--
 src/libstore/store-api.hh          |  4 +-
 src/libstore/uds-remote-store.hh   |  4 +-
 src/libutil/source-accessor.cc     |  5 +++
 src/libutil/source-accessor.hh     | 22 ++++++++--
 src/nix/bundle.cc                  |  1 -
 src/nix/cat.cc                     | 13 +++---
 src/nix/ls.cc                      | 54 +++++++++++--------------
 src/nix/run.cc                     |  8 ++--
 src/nix/why-depends.cc             | 22 +++++-----
 tests/functional/nar-access.sh     |  8 +++-
 25 files changed, 178 insertions(+), 224 deletions(-)
 delete mode 100644 src/libstore/fs-accessor.hh

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index b61868413..dd9e2f3af 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -2,7 +2,7 @@
 #include "binary-cache-store.hh"
 #include "compression.hh"
 #include "derivations.hh"
-#include "fs-accessor.hh"
+#include "source-accessor.hh"
 #include "globals.hh"
 #include "nar-info.hh"
 #include "sync.hh"
@@ -143,7 +143,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
        write the compressed NAR to disk), into a HashSink (to get the
        NAR hash), and into a NarAccessor (to get the NAR listing). */
     HashSink fileHashSink { htSHA256 };
-    std::shared_ptr<FSAccessor> narAccessor;
+    std::shared_ptr<SourceAccessor> narAccessor;
     HashSink narHashSink { htSHA256 };
     {
     FdSink fileSink(fdTemp.get());
@@ -195,7 +195,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     if (writeNARListing) {
         nlohmann::json j = {
             {"version", 1},
-            {"root", listNar(ref<FSAccessor>(narAccessor), "", true)},
+            {"root", listNar(ref<SourceAccessor>(narAccessor), CanonPath::root, true)},
         };
 
         upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json");
@@ -206,9 +206,9 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
        specify the NAR file and member containing the debug info. */
     if (writeDebugInfo) {
 
-        std::string buildIdDir = "/lib/debug/.build-id";
+        CanonPath buildIdDir("lib/debug/.build-id");
 
-        if (auto st = narAccessor->stat(buildIdDir); st && st->type == SourceAccessor::tDirectory) {
+        if (auto st = narAccessor->maybeLstat(buildIdDir); st && st->type == SourceAccessor::tDirectory) {
 
             ThreadPool threadPool(25);
 
@@ -232,16 +232,16 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
             std::regex regex2("^[0-9a-f]{38}\\.debug$");
 
             for (auto & [s1, _type] : narAccessor->readDirectory(buildIdDir)) {
-                auto dir = buildIdDir + "/" + s1;
+                auto dir = buildIdDir + s1;
 
-                if (auto st = narAccessor->stat(dir); !st || st->type != SourceAccessor::tDirectory
+                if (narAccessor->lstat(dir).type != SourceAccessor::tDirectory
                     || !std::regex_match(s1, regex1))
                     continue;
 
                 for (auto & [s2, _type] : narAccessor->readDirectory(dir)) {
-                    auto debugPath = dir + "/" + s2;
+                    auto debugPath = dir + s2;
 
-                    if (auto st = narAccessor->stat(debugPath); !st || st->type != SourceAccessor::tRegular
+                    if ( narAccessor->lstat(debugPath).type != SourceAccessor::tRegular
                         || !std::regex_match(s2, regex2))
                         continue;
 
@@ -250,7 +250,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
                     std::string key = "debuginfo/" + buildId;
                     std::string target = "../" + narInfo->url;
 
-                    threadPool.enqueue(std::bind(doFile, std::string(debugPath, 1), key, target));
+                    threadPool.enqueue(std::bind(doFile, std::string(debugPath.rel()), key, target));
                 }
             }
 
@@ -503,9 +503,9 @@ void BinaryCacheStore::registerDrvOutput(const Realisation& info) {
     upsertFile(filePath, info.toJSON().dump(), "application/json");
 }
 
-ref<FSAccessor> BinaryCacheStore::getFSAccessor()
+ref<SourceAccessor> BinaryCacheStore::getFSAccessor(bool requireValidPath)
 {
-    return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), localNarCache);
+    return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), requireValidPath, localNarCache);
 }
 
 void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSet & sigs)
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index 218a888e3..cea2a571f 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -148,7 +148,7 @@ public:
 
     void narFromPath(const StorePath & path, Sink & sink) override;
 
-    ref<FSAccessor> getFSAccessor() override;
+    ref<SourceAccessor> getFSAccessor(bool requireValidPath) override;
 
     void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
 
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index a5ceb29dc..efdad18e1 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -6,7 +6,6 @@
 #include "split.hh"
 #include "common-protocol.hh"
 #include "common-protocol-impl.hh"
-#include "fs-accessor.hh"
 #include <boost/container/small_vector.hpp>
 #include <nlohmann/json.hpp>
 
diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc
index 74d6ed3b5..821cda399 100644
--- a/src/libstore/dummy-store.cc
+++ b/src/libstore/dummy-store.cc
@@ -72,7 +72,7 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store
         Callback<std::shared_ptr<const Realisation>> callback) noexcept override
     { callback(nullptr); }
 
-    virtual ref<FSAccessor> getFSAccessor() override
+    virtual ref<SourceAccessor> getFSAccessor(bool requireValidPath) override
     { unsupported("getFSAccessor"); }
 };
 
diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh
deleted file mode 100644
index f6c002a2d..000000000
--- a/src/libstore/fs-accessor.hh
+++ /dev/null
@@ -1,58 +0,0 @@
-#pragma once
-///@file
-
-#include "types.hh"
-#include "source-accessor.hh"
-
-#include <optional>
-
-namespace nix {
-
-/**
- * An abstract class for accessing a filesystem-like structure, such
- * as a (possibly remote) Nix store or the contents of a NAR file.
- */
-class FSAccessor
-{
-public:
-    using Type = SourceAccessor::Type;
-
-    struct Stat
-    {
-        Type type;
-        /**
-         * For regular files only: the size of the file.
-         */
-        std::optional<uint64_t> fileSize;
-        /**
-         * For regular files only: whether this is an executable.
-         */
-        bool isExecutable = false;
-        /**
-         * For regular files only: the position of the contents of this
-         * file in the NAR.
-         */
-        std::optional<uint64_t> narOffset;
-    };
-
-    virtual ~FSAccessor() { }
-
-    virtual std::optional<Stat> stat(const Path & path) = 0;
-
-    using DirEntries = SourceAccessor::DirEntries;
-
-    virtual DirEntries readDirectory(const Path & path) = 0;
-
-    /**
-     * Read a file inside the store.
-     *
-     * If `requireValidPath` is set to `true` (the default), the path must be
-     * inside a valid store path, otherwise it just needs to be physically
-     * present (but not necessarily properly registered)
-     */
-    virtual std::string readFile(const Path & path, bool requireValidPath = true) = 0;
-
-    virtual std::string readLink(const Path & path) = 0;
-};
-
-}
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 38fdf118f..731457354 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -363,7 +363,7 @@ public:
     void ensurePath(const StorePath & path) override
     { unsupported("ensurePath"); }
 
-    virtual ref<FSAccessor> getFSAccessor() override
+    virtual ref<SourceAccessor> getFSAccessor(bool requireValidPath) override
     { unsupported("getFSAccessor"); }
 
     /**
diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc
index 65cbb9e35..63497acbd 100644
--- a/src/libstore/local-fs-store.cc
+++ b/src/libstore/local-fs-store.cc
@@ -1,5 +1,5 @@
 #include "archive.hh"
-#include "fs-accessor.hh"
+#include "source-accessor.hh"
 #include "store-api.hh"
 #include "local-fs-store.hh"
 #include "globals.hh"
@@ -13,26 +13,31 @@ LocalFSStore::LocalFSStore(const Params & params)
 {
 }
 
-struct LocalStoreAccessor : public FSAccessor
+struct LocalStoreAccessor : public SourceAccessor
 {
     ref<LocalFSStore> store;
+    bool requireValidPath;
 
-    LocalStoreAccessor(ref<LocalFSStore> store) : store(store) { }
+    LocalStoreAccessor(ref<LocalFSStore> store, bool requireValidPath)
+        : store(store)
+        , requireValidPath(requireValidPath)
+    { }
 
-    Path toRealPath(const Path & path, bool requireValidPath = true)
+    Path toRealPath(const CanonPath & path)
     {
-        auto storePath = store->toStorePath(path).first;
+        auto storePath = store->toStorePath(path.abs()).first;
         if (requireValidPath && !store->isValidPath(storePath))
             throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
-        return store->getRealStoreDir() + std::string(path, store->storeDir.size());
+        return store->getRealStoreDir() + path.abs().substr(store->storeDir.size());
     }
 
-    std::optional<FSAccessor::Stat> stat(const Path & path) override
+    std::optional<Stat> maybeLstat(const CanonPath & path) override
     {
         auto realPath = toRealPath(path);
 
+        // FIXME: use PosixSourceAccessor.
         struct stat st;
-        if (lstat(realPath.c_str(), &st)) {
+        if (::lstat(realPath.c_str(), &st)) {
             if (errno == ENOENT || errno == ENOTDIR) return std::nullopt;
             throw SysError("getting status of '%1%'", path);
         }
@@ -48,7 +53,7 @@ struct LocalStoreAccessor : public FSAccessor
             S_ISREG(st.st_mode) && st.st_mode & S_IXUSR}};
     }
 
-    DirEntries readDirectory(const Path & path) override
+    DirEntries readDirectory(const CanonPath & path) override
     {
         auto realPath = toRealPath(path);
 
@@ -61,21 +66,22 @@ struct LocalStoreAccessor : public FSAccessor
         return res;
     }
 
-    std::string readFile(const Path & path, bool requireValidPath = true) override
+    std::string readFile(const CanonPath & path) override
     {
-        return nix::readFile(toRealPath(path, requireValidPath));
+        return nix::readFile(toRealPath(path));
     }
 
-    std::string readLink(const Path & path) override
+    std::string readLink(const CanonPath & path) override
     {
         return nix::readLink(toRealPath(path));
     }
 };
 
-ref<FSAccessor> LocalFSStore::getFSAccessor()
+ref<SourceAccessor> LocalFSStore::getFSAccessor(bool requireValidPath)
 {
     return make_ref<LocalStoreAccessor>(ref<LocalFSStore>(
-            std::dynamic_pointer_cast<LocalFSStore>(shared_from_this())));
+            std::dynamic_pointer_cast<LocalFSStore>(shared_from_this())),
+        requireValidPath);
 }
 
 void LocalFSStore::narFromPath(const StorePath & path, Sink & sink)
diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh
index d6bda05d1..bf855b67e 100644
--- a/src/libstore/local-fs-store.hh
+++ b/src/libstore/local-fs-store.hh
@@ -43,7 +43,7 @@ public:
     LocalFSStore(const Params & params);
 
     void narFromPath(const StorePath & path, Sink & sink) override;
-    ref<FSAccessor> getFSAccessor() override;
+    ref<SourceAccessor> getFSAccessor(bool requireValidPath) override;
 
     /**
      * Creates symlink from the `gcRoot` to the `storePath` and
diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc
index f1be5606e..02993680f 100644
--- a/src/libstore/nar-accessor.cc
+++ b/src/libstore/nar-accessor.cc
@@ -11,7 +11,7 @@ namespace nix {
 
 struct NarMember
 {
-    FSAccessor::Stat stat;
+    SourceAccessor::Stat stat;
 
     std::string target;
 
@@ -19,7 +19,7 @@ struct NarMember
     std::map<std::string, NarMember> children;
 };
 
-struct NarAccessor : public FSAccessor
+struct NarAccessor : public SourceAccessor
 {
     std::optional<const std::string> nar;
 
@@ -149,48 +149,36 @@ struct NarAccessor : public FSAccessor
         recurse(root, v);
     }
 
-    NarMember * find(const Path & path)
+    NarMember * find(const CanonPath & path)
     {
-        Path canon = path == "" ? "" : canonPath(path);
         NarMember * current = &root;
-        auto end = path.end();
-        for (auto it = path.begin(); it != end; ) {
-            // because it != end, the remaining component is non-empty so we need
-            // a directory
+
+        for (auto & i : path) {
             if (current->stat.type != Type::tDirectory) return nullptr;
-
-            // skip slash (canonPath above ensures that this is always a slash)
-            assert(*it == '/');
-            it += 1;
-
-            // lookup current component
-            auto next = std::find(it, end, '/');
-            auto child = current->children.find(std::string(it, next));
+            auto child = current->children.find(std::string(i));
             if (child == current->children.end()) return nullptr;
             current = &child->second;
-
-            it = next;
         }
 
         return current;
     }
 
-    NarMember & get(const Path & path) {
+    NarMember & get(const CanonPath & path) {
         auto result = find(path);
-        if (result == nullptr)
+        if (!result)
             throw Error("NAR file does not contain path '%1%'", path);
         return *result;
     }
 
-    std::optional<Stat> stat(const Path & path) override
+    std::optional<Stat> maybeLstat(const CanonPath & path) override
     {
         auto i = find(path);
-        if (i == nullptr)
+        if (!i)
             return std::nullopt;
         return i->stat;
     }
 
-    DirEntries readDirectory(const Path & path) override
+    DirEntries readDirectory(const CanonPath & path) override
     {
         auto i = get(path);
 
@@ -204,7 +192,7 @@ struct NarAccessor : public FSAccessor
         return res;
     }
 
-    std::string readFile(const Path & path, bool requireValidPath = true) override
+    std::string readFile(const CanonPath & path) override
     {
         auto i = get(path);
         if (i.stat.type != Type::tRegular)
@@ -216,7 +204,7 @@ struct NarAccessor : public FSAccessor
         return std::string(*nar, *i.stat.narOffset, *i.stat.fileSize);
     }
 
-    std::string readLink(const Path & path) override
+    std::string readLink(const CanonPath & path) override
     {
         auto i = get(path);
         if (i.stat.type != Type::tSymlink)
@@ -225,40 +213,38 @@ struct NarAccessor : public FSAccessor
     }
 };
 
-ref<FSAccessor> makeNarAccessor(std::string && nar)
+ref<SourceAccessor> makeNarAccessor(std::string && nar)
 {
     return make_ref<NarAccessor>(std::move(nar));
 }
 
-ref<FSAccessor> makeNarAccessor(Source & source)
+ref<SourceAccessor> makeNarAccessor(Source & source)
 {
     return make_ref<NarAccessor>(source);
 }
 
-ref<FSAccessor> makeLazyNarAccessor(const std::string & listing,
+ref<SourceAccessor> makeLazyNarAccessor(const std::string & listing,
     GetNarBytes getNarBytes)
 {
     return make_ref<NarAccessor>(listing, getNarBytes);
 }
 
 using nlohmann::json;
-json listNar(ref<FSAccessor> accessor, const Path & path, bool recurse)
+json listNar(ref<SourceAccessor> accessor, const CanonPath & path, bool recurse)
 {
-    auto st = accessor->stat(path);
-    if (!st)
-        throw Error("path '%s' does not exist in NAR", path);
+    auto st = accessor->lstat(path);
 
     json obj = json::object();
 
-    switch (st->type) {
+    switch (st.type) {
     case SourceAccessor::Type::tRegular:
         obj["type"] = "regular";
-        if (st->fileSize)
-            obj["size"] = *st->fileSize;
-        if (st->isExecutable)
+        if (st.fileSize)
+            obj["size"] = *st.fileSize;
+        if (st.isExecutable)
             obj["executable"] = true;
-        if (st->narOffset && *st->narOffset)
-            obj["narOffset"] = *st->narOffset;
+        if (st.narOffset && *st.narOffset)
+            obj["narOffset"] = *st.narOffset;
         break;
     case SourceAccessor::Type::tDirectory:
         obj["type"] = "directory";
@@ -267,7 +253,7 @@ json listNar(ref<FSAccessor> accessor, const Path & path, bool recurse)
             json &res2 = obj["entries"];
             for (auto & [name, type] : accessor->readDirectory(path)) {
                 if (recurse) {
-                    res2[name] = listNar(accessor, path + "/" + name, true);
+                    res2[name] = listNar(accessor, path + name, true);
                 } else
                     res2[name] = json::object();
             }
diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh
index 5e19bd3c7..433774524 100644
--- a/src/libstore/nar-accessor.hh
+++ b/src/libstore/nar-accessor.hh
@@ -1,10 +1,11 @@
 #pragma once
 ///@file
 
+#include "source-accessor.hh"
+
 #include <functional>
 
 #include <nlohmann/json_fwd.hpp>
-#include "fs-accessor.hh"
 
 namespace nix {
 
@@ -14,9 +15,9 @@ struct Source;
  * Return an object that provides access to the contents of a NAR
  * file.
  */
-ref<FSAccessor> makeNarAccessor(std::string && nar);
+ref<SourceAccessor> makeNarAccessor(std::string && nar);
 
-ref<FSAccessor> makeNarAccessor(Source & source);
+ref<SourceAccessor> makeNarAccessor(Source & source);
 
 /**
  * Create a NAR accessor from a NAR listing (in the format produced by
@@ -26,7 +27,7 @@ ref<FSAccessor> makeNarAccessor(Source & source);
  */
 typedef std::function<std::string(uint64_t, uint64_t)> GetNarBytes;
 
-ref<FSAccessor> makeLazyNarAccessor(
+ref<SourceAccessor> makeLazyNarAccessor(
     const std::string & listing,
     GetNarBytes getNarBytes);
 
@@ -34,6 +35,6 @@ ref<FSAccessor> makeLazyNarAccessor(
  * Write a JSON representation of the contents of a NAR (except file
  * contents).
  */
-nlohmann::json listNar(ref<FSAccessor> accessor, const Path & path, bool recurse);
+nlohmann::json listNar(ref<SourceAccessor> accessor, const CanonPath & path, bool recurse);
 
 }
diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc
index 21419700c..03e57a565 100644
--- a/src/libstore/remote-fs-accessor.cc
+++ b/src/libstore/remote-fs-accessor.cc
@@ -8,8 +8,9 @@
 
 namespace nix {
 
-RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path & cacheDir)
+RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, bool requireValidPath, const Path & cacheDir)
     : store(store)
+    , requireValidPath(requireValidPath)
     , cacheDir(cacheDir)
 {
     if (cacheDir != "")
@@ -22,7 +23,7 @@ Path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::strin
     return fmt("%s/%s.%s", cacheDir, hashPart, ext);
 }
 
-ref<FSAccessor> RemoteFSAccessor::addToCache(std::string_view hashPart, std::string && nar)
+ref<SourceAccessor> RemoteFSAccessor::addToCache(std::string_view hashPart, std::string && nar)
 {
     if (cacheDir != "") {
         try {
@@ -38,7 +39,7 @@ ref<FSAccessor> RemoteFSAccessor::addToCache(std::string_view hashPart, std::str
 
     if (cacheDir != "") {
         try {
-            nlohmann::json j = listNar(narAccessor, "", true);
+            nlohmann::json j = listNar(narAccessor, CanonPath::root, true);
             writeFile(makeCacheFile(hashPart, "ls"), j.dump());
         } catch (...) {
             ignoreException();
@@ -48,11 +49,10 @@ ref<FSAccessor> RemoteFSAccessor::addToCache(std::string_view hashPart, std::str
     return narAccessor;
 }
 
-std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_, bool requireValidPath)
+std::pair<ref<SourceAccessor>, CanonPath> RemoteFSAccessor::fetch(const CanonPath & path)
 {
-    auto path = canonPath(path_);
-
-    auto [storePath, restPath] = store->toStorePath(path);
+    auto [storePath, restPath_] = store->toStorePath(path.abs());
+    auto restPath = CanonPath(restPath_);
 
     if (requireValidPath && !store->isValidPath(storePath))
         throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
@@ -63,7 +63,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_, boo
     std::string listing;
     Path cacheFile;
 
-    if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) {
+    if (cacheDir != "" && nix::pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) {
 
         try {
             listing = nix::readFile(makeCacheFile(storePath.hashPart(), "ls"));
@@ -101,25 +101,25 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_, boo
     return {addToCache(storePath.hashPart(), std::move(sink.s)), restPath};
 }
 
-std::optional<FSAccessor::Stat> RemoteFSAccessor::stat(const Path & path)
+std::optional<SourceAccessor::Stat> RemoteFSAccessor::maybeLstat(const CanonPath & path)
 {
     auto res = fetch(path);
-    return res.first->stat(res.second);
+    return res.first->maybeLstat(res.second);
 }
 
-SourceAccessor::DirEntries RemoteFSAccessor::readDirectory(const Path & path)
+SourceAccessor::DirEntries RemoteFSAccessor::readDirectory(const CanonPath & path)
 {
     auto res = fetch(path);
     return res.first->readDirectory(res.second);
 }
 
-std::string RemoteFSAccessor::readFile(const Path & path, bool requireValidPath)
+std::string RemoteFSAccessor::readFile(const CanonPath & path)
 {
-    auto res = fetch(path, requireValidPath);
+    auto res = fetch(path);
     return res.first->readFile(res.second);
 }
 
-std::string RemoteFSAccessor::readLink(const Path & path)
+std::string RemoteFSAccessor::readLink(const CanonPath & path)
 {
     auto res = fetch(path);
     return res.first->readLink(res.second);
diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh
index 8de3b7bcd..d09762a53 100644
--- a/src/libstore/remote-fs-accessor.hh
+++ b/src/libstore/remote-fs-accessor.hh
@@ -1,40 +1,43 @@
 #pragma once
 ///@file
 
-#include "fs-accessor.hh"
+#include "source-accessor.hh"
 #include "ref.hh"
 #include "store-api.hh"
 
 namespace nix {
 
-class RemoteFSAccessor : public FSAccessor
+class RemoteFSAccessor : public SourceAccessor
 {
     ref<Store> store;
 
-    std::map<std::string, ref<FSAccessor>> nars;
+    std::map<std::string, ref<SourceAccessor>> nars;
+
+    bool requireValidPath;
 
     Path cacheDir;
 
-    std::pair<ref<FSAccessor>, Path> fetch(const Path & path_, bool requireValidPath = true);
+    std::pair<ref<SourceAccessor>, CanonPath> fetch(const CanonPath & path);
 
     friend class BinaryCacheStore;
 
     Path makeCacheFile(std::string_view hashPart, const std::string & ext);
 
-    ref<FSAccessor> addToCache(std::string_view hashPart, std::string && nar);
+    ref<SourceAccessor> addToCache(std::string_view hashPart, std::string && nar);
 
 public:
 
     RemoteFSAccessor(ref<Store> store,
+        bool requireValidPath = true,
         const /* FIXME: use std::optional */ Path & cacheDir = "");
 
-    std::optional<Stat> stat(const Path & path) override;
+    std::optional<Stat> maybeLstat(const CanonPath & path) override;
 
-    DirEntries readDirectory(const Path & path) override;
+    DirEntries readDirectory(const CanonPath & path) override;
 
-    std::string readFile(const Path & path, bool requireValidPath = true) override;
+    std::string readFile(const CanonPath & path) override;
 
-    std::string readLink(const Path & path) override;
+    std::string readLink(const CanonPath & path) override;
 };
 
 }
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 7bdc25433..f16949f42 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -970,7 +970,7 @@ void RemoteStore::narFromPath(const StorePath & path, Sink & sink)
     copyNAR(conn->from, sink);
 }
 
-ref<FSAccessor> RemoteStore::getFSAccessor()
+ref<SourceAccessor> RemoteStore::getFSAccessor(bool requireValidPath)
 {
     return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()));
 }
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index f0985fdc1..1cc11af86 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -185,7 +185,7 @@ protected:
 
     friend struct ConnectionHandle;
 
-    virtual ref<FSAccessor> getFSAccessor() override;
+    virtual ref<SourceAccessor> getFSAccessor(bool requireValidPath) override;
 
     virtual void narFromPath(const StorePath & path, Sink & sink) override;
 
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 0399120d1..665b5fed7 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -1,5 +1,5 @@
 #include "crypto.hh"
-#include "fs-accessor.hh"
+#include "source-accessor.hh"
 #include "globals.hh"
 #include "derivations.hh"
 #include "store-api.hh"
@@ -1338,12 +1338,12 @@ Derivation Store::derivationFromPath(const StorePath & drvPath)
     return readDerivation(drvPath);
 }
 
-Derivation readDerivationCommon(Store& store, const StorePath& drvPath, bool requireValidPath)
+static Derivation readDerivationCommon(Store & store, const StorePath & drvPath, bool requireValidPath)
 {
-    auto accessor = store.getFSAccessor();
+    auto accessor = store.getFSAccessor(requireValidPath);
     try {
         return parseDerivation(store,
-            accessor->readFile(store.printStorePath(drvPath), requireValidPath),
+            accessor->readFile(CanonPath(store.printStorePath(drvPath))),
             Derivation::nameFromPath(drvPath));
     } catch (FormatError & e) {
         throw Error("error parsing derivation '%s': %s", store.printStorePath(drvPath), e.msg());
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index e123fccc5..6aa317e3d 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -70,7 +70,7 @@ MakeError(InvalidStoreURI, Error);
 
 struct BasicDerivation;
 struct Derivation;
-class FSAccessor;
+struct SourceAccessor;
 class NarInfoDiskCache;
 class Store;
 
@@ -703,7 +703,7 @@ public:
     /**
      * @return An object to access files in the Nix store.
      */
-    virtual ref<FSAccessor> getFSAccessor() = 0;
+    virtual ref<SourceAccessor> getFSAccessor(bool requireValidPath = true) = 0;
 
     /**
      * Repair the contents of the given path by redownloading it using
diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh
index cdb28a001..a5ac9080a 100644
--- a/src/libstore/uds-remote-store.hh
+++ b/src/libstore/uds-remote-store.hh
@@ -35,8 +35,8 @@ public:
     static std::set<std::string> uriSchemes()
     { return {"unix"}; }
 
-    ref<FSAccessor> getFSAccessor() override
-    { return LocalFSStore::getFSAccessor(); }
+    ref<SourceAccessor> getFSAccessor(bool requireValidPath) override
+    { return LocalFSStore::getFSAccessor(requireValidPath); }
 
     void narFromPath(const StorePath & path, Sink & sink) override
     { LocalFSStore::narFromPath(path, sink); }
diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc
index 5b0c7dd34..e2114e18f 100644
--- a/src/libutil/source-accessor.cc
+++ b/src/libutil/source-accessor.cc
@@ -10,6 +10,11 @@ SourceAccessor::SourceAccessor()
 {
 }
 
+bool SourceAccessor::pathExists(const CanonPath & path)
+{
+    return maybeLstat(path).has_value();
+}
+
 std::string SourceAccessor::readFile(const CanonPath & path)
 {
     StringSink sink;
diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh
index 80bc02b48..1a4e80361 100644
--- a/src/libutil/source-accessor.hh
+++ b/src/libutil/source-accessor.hh
@@ -40,7 +40,7 @@ struct SourceAccessor
         Sink & sink,
         std::function<void(uint64_t)> sizeCallback = [](uint64_t size){});
 
-    virtual bool pathExists(const CanonPath & path) = 0;
+    virtual bool pathExists(const CanonPath & path);
 
     enum Type {
       tRegular, tSymlink, tDirectory,
@@ -57,8 +57,24 @@ struct SourceAccessor
     struct Stat
     {
         Type type = tMisc;
-        //uint64_t fileSize = 0; // regular files only
-        bool isExecutable = false; // regular files only
+
+        /**
+         * For regular files only: the size of the file. Not all
+         * accessors return this since it may be too expensive to
+         * compute.
+         */
+        std::optional<uint64_t> fileSize;
+
+        /**
+         * For regular files only: whether this is an executable.
+         */
+        bool isExecutable = false;
+
+        /**
+         * For regular files only: the position of the contents of this
+         * file in the NAR. Only returned by NAR accessors.
+         */
+        std::optional<uint64_t> narOffset;
     };
 
     Stat lstat(const CanonPath & path);
diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc
index 504e35c81..54cc6a17f 100644
--- a/src/nix/bundle.cc
+++ b/src/nix/bundle.cc
@@ -4,7 +4,6 @@
 #include "shared.hh"
 #include "store-api.hh"
 #include "local-fs-store.hh"
-#include "fs-accessor.hh"
 #include "eval-inline.hh"
 
 using namespace nix;
diff --git a/src/nix/cat.cc b/src/nix/cat.cc
index b5fe2506f..6e5a736f2 100644
--- a/src/nix/cat.cc
+++ b/src/nix/cat.cc
@@ -1,6 +1,5 @@
 #include "command.hh"
 #include "store-api.hh"
-#include "fs-accessor.hh"
 #include "nar-accessor.hh"
 
 using namespace nix;
@@ -9,14 +8,12 @@ struct MixCat : virtual Args
 {
     std::string path;
 
-    void cat(ref<FSAccessor> accessor)
+    void cat(ref<SourceAccessor> accessor)
     {
-        if (auto st = accessor->stat(path)) {
-            if (st->type != FSAccessor::Type::tRegular)
-                throw Error("path '%1%' is not a regular file", path);
-            writeFull(STDOUT_FILENO, accessor->readFile(path));
-        } else
-            throw Error("path '%1%' does not exist", path);
+        auto st = accessor->lstat(CanonPath(path));
+        if (st.type != SourceAccessor::Type::tRegular)
+            throw Error("path '%1%' is not a regular file", path);
+        writeFull(STDOUT_FILENO, accessor->readFile(CanonPath(path)));
     }
 };
 
diff --git a/src/nix/ls.cc b/src/nix/ls.cc
index da978f379..231456c9c 100644
--- a/src/nix/ls.cc
+++ b/src/nix/ls.cc
@@ -1,6 +1,5 @@
 #include "command.hh"
 #include "store-api.hh"
-#include "fs-accessor.hh"
 #include "nar-accessor.hh"
 #include "common-args.hh"
 #include <nlohmann/json.hpp>
@@ -39,63 +38,58 @@ struct MixLs : virtual Args, MixJSON
         });
     }
 
-    void listText(ref<FSAccessor> accessor)
+    void listText(ref<SourceAccessor> accessor)
     {
-        std::function<void(const FSAccessor::Stat &, const Path &, const std::string &, bool)> doPath;
+        std::function<void(const SourceAccessor::Stat &, const CanonPath &, std::string_view, bool)> doPath;
 
-        auto showFile = [&](const Path & curPath, const std::string & relPath) {
+        auto showFile = [&](const CanonPath & curPath, std::string_view relPath) {
             if (verbose) {
-                auto st = accessor->stat(curPath);
-                assert(st);
+                auto st = accessor->lstat(curPath);
                 std::string tp =
-                    st->type == FSAccessor::Type::tRegular ?
-                        (st->isExecutable ? "-r-xr-xr-x" : "-r--r--r--") :
-                    st->type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" :
+                    st.type == SourceAccessor::Type::tRegular ?
+                        (st.isExecutable ? "-r-xr-xr-x" : "-r--r--r--") :
+                    st.type == SourceAccessor::Type::tSymlink ? "lrwxrwxrwx" :
                     "dr-xr-xr-x";
-                auto line = fmt("%s %20d %s", tp, st->fileSize.value_or(0), relPath);
-                if (st->type == FSAccessor::Type::tSymlink)
+                auto line = fmt("%s %20d %s", tp, st.fileSize.value_or(0), relPath);
+                if (st.type == SourceAccessor::Type::tSymlink)
                     line += " -> " + accessor->readLink(curPath);
                 logger->cout(line);
-                if (recursive && st->type == FSAccessor::Type::tDirectory)
-                    doPath(*st, curPath, relPath, false);
+                if (recursive && st.type == SourceAccessor::Type::tDirectory)
+                    doPath(st, curPath, relPath, false);
             } else {
                 logger->cout(relPath);
                 if (recursive) {
-                    auto st = accessor->stat(curPath);
-                    assert(st);
-                    if (st->type == FSAccessor::Type::tDirectory)
-                        doPath(*st, curPath, relPath, false);
+                    auto st = accessor->lstat(curPath);
+                    if (st.type == SourceAccessor::Type::tDirectory)
+                        doPath(st, curPath, relPath, false);
                 }
             }
         };
 
-        doPath = [&](const FSAccessor::Stat & st, const Path & curPath,
-            const std::string & relPath, bool showDirectory)
+        doPath = [&](const SourceAccessor::Stat & st, const CanonPath & curPath,
+            std::string_view relPath, bool showDirectory)
         {
-            if (st.type == FSAccessor::Type::tDirectory && !showDirectory) {
+            if (st.type == SourceAccessor::Type::tDirectory && !showDirectory) {
                 auto names = accessor->readDirectory(curPath);
                 for (auto & [name, type] : names)
-                    showFile(curPath + "/" + name, relPath + "/" + name);
+                    showFile(curPath + name, relPath + "/" + name);
             } else
                 showFile(curPath, relPath);
         };
 
-        auto st = accessor->stat(path);
-        if (!st)
-            throw Error("path '%1%' does not exist", path);
-        doPath(*st, path,
-            st->type == FSAccessor::Type::tDirectory ? "." : std::string(baseNameOf(path)),
+        auto path2 = CanonPath(path);
+        auto st = accessor->lstat(path2);
+        doPath(st, path2,
+            st.type == SourceAccessor::Type::tDirectory ? "." : path2.baseName().value_or(""),
             showDirectory);
     }
 
-    void list(ref<FSAccessor> accessor)
+    void list(ref<SourceAccessor> accessor)
     {
-        if (path == "/") path = "";
-
         if (json) {
             if (showDirectory)
                 throw UsageError("'--directory' is useless with '--json'");
-            logger->cout("%s", listNar(accessor, path, recursive));
+            logger->cout("%s", listNar(accessor, CanonPath(path), recursive));
         } else
             listText(accessor);
     }
diff --git a/src/nix/run.cc b/src/nix/run.cc
index 07806283c..1465e8cde 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -6,7 +6,7 @@
 #include "derivations.hh"
 #include "local-store.hh"
 #include "finally.hh"
-#include "fs-accessor.hh"
+#include "source-accessor.hh"
 #include "progress-bar.hh"
 #include "eval.hh"
 #include "build/personality.hh"
@@ -119,9 +119,9 @@ struct CmdShell : InstallablesCommand, MixEnvironment
             if (true)
                 unixPath.push_front(store->printStorePath(path) + "/bin");
 
-            auto propPath = store->printStorePath(path) + "/nix-support/propagated-user-env-packages";
-            if (auto st = accessor->stat(propPath); st && st->type == SourceAccessor::tRegular) {
-                for (auto & p : tokenizeString<Paths>(readFile(propPath)))
+            auto propPath = CanonPath(store->printStorePath(path)) + "nix-support" + "propagated-user-env-packages";
+            if (auto st = accessor->maybeLstat(propPath); st && st->type == SourceAccessor::tRegular) {
+                for (auto & p : tokenizeString<Paths>(accessor->readFile(propPath)))
                     todo.push(store->parseStorePath(p));
             }
         }
diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc
index 04c1a0c1c..aecf65922 100644
--- a/src/nix/why-depends.cc
+++ b/src/nix/why-depends.cc
@@ -1,7 +1,7 @@
 #include "command.hh"
 #include "store-api.hh"
 #include "progress-bar.hh"
-#include "fs-accessor.hh"
+#include "source-accessor.hh"
 #include "shared.hh"
 
 #include <queue>
@@ -175,7 +175,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions
         struct BailOut { };
 
         printNode = [&](Node & node, const std::string & firstPad, const std::string & tailPad) {
-            auto pathS = store->printStorePath(node.path);
+            CanonPath pathS(store->printStorePath(node.path));
 
             assert(node.dist != inf);
             if (precise) {
@@ -183,7 +183,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions
                     firstPad,
                     node.visited ? "\e[38;5;244m" : "",
                     firstPad != "" ? "→ " : "",
-                    pathS);
+                    pathS.abs());
             }
 
             if (node.path == dependencyPath && !all
@@ -210,25 +210,25 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions
                contain the reference. */
             std::map<std::string, Strings> hits;
 
-            std::function<void(const Path &)> visitPath;
+            std::function<void(const CanonPath &)> visitPath;
 
-            visitPath = [&](const Path & p) {
-                auto st = accessor->stat(p);
+            visitPath = [&](const CanonPath & p) {
+                auto st = accessor->maybeLstat(p);
                 assert(st);
 
-                auto p2 = p == pathS ? "/" : std::string(p, pathS.size() + 1);
+                auto p2 = p == pathS ? "/" : p.abs().substr(pathS.abs().size() + 1);
 
                 auto getColour = [&](const std::string & hash) {
                     return hash == dependencyPathHash ? ANSI_GREEN : ANSI_BLUE;
                 };
 
-                if (st->type == FSAccessor::Type::tDirectory) {
+                if (st->type == SourceAccessor::Type::tDirectory) {
                     auto names = accessor->readDirectory(p);
                     for (auto & [name, type] : names)
-                        visitPath(p + "/" + name);
+                        visitPath(p + name);
                 }
 
-                else if (st->type == FSAccessor::Type::tRegular) {
+                else if (st->type == SourceAccessor::Type::tRegular) {
                     auto contents = accessor->readFile(p);
 
                     for (auto & hash : hashes) {
@@ -246,7 +246,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions
                     }
                 }
 
-                else if (st->type == FSAccessor::Type::tSymlink) {
+                else if (st->type == SourceAccessor::Type::tSymlink) {
                     auto target = accessor->readLink(p);
 
                     for (auto & hash : hashes) {
diff --git a/tests/functional/nar-access.sh b/tests/functional/nar-access.sh
index d487d58d2..13d23c342 100644
--- a/tests/functional/nar-access.sh
+++ b/tests/functional/nar-access.sh
@@ -25,6 +25,12 @@ diff -u baz.cat-nar $storePath/foo/baz
 nix store cat $storePath/foo/baz > baz.cat-nar
 diff -u baz.cat-nar $storePath/foo/baz
 
+# Check that 'nix store cat' fails on invalid store paths.
+invalidPath="$(dirname $storePath)/99999999999999999999999999999999-foo"
+mv $storePath $invalidPath
+(! nix store cat $invalidPath/foo/baz)
+mv $invalidPath $storePath
+
 # Test --json.
 diff -u \
     <(nix nar ls --json $narFile / | jq -S) \
@@ -46,7 +52,7 @@ diff -u \
     <(echo '{"type":"regular","size":0}' | jq -S)
 
 # Test missing files.
-expect 1 nix store ls --json -R $storePath/xyzzy 2>&1 | grep 'does not exist in NAR'
+expect 1 nix store ls --json -R $storePath/xyzzy 2>&1 | grep 'does not exist'
 expect 1 nix store ls $storePath/xyzzy 2>&1 | grep 'does not exist'
 
 # Test failure to dump.

From 2f5c1a27dc71275c1d4c96cff42beffed0d4d2f7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 1 Nov 2023 17:22:25 +0100
Subject: [PATCH 041/294] LocalStoreAccessor: Reuse PosixSourceAccessor

---
 src/libstore/local-fs-store.cc       | 48 ++++++++--------------------
 src/libutil/posix-source-accessor.cc |  3 +-
 2 files changed, 15 insertions(+), 36 deletions(-)

diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc
index 63497acbd..953f3a264 100644
--- a/src/libstore/local-fs-store.cc
+++ b/src/libstore/local-fs-store.cc
@@ -1,5 +1,5 @@
 #include "archive.hh"
-#include "source-accessor.hh"
+#include "posix-source-accessor.hh"
 #include "store-api.hh"
 #include "local-fs-store.hh"
 #include "globals.hh"
@@ -13,7 +13,7 @@ LocalFSStore::LocalFSStore(const Params & params)
 {
 }
 
-struct LocalStoreAccessor : public SourceAccessor
+struct LocalStoreAccessor : PosixSourceAccessor
 {
     ref<LocalFSStore> store;
     bool requireValidPath;
@@ -23,57 +23,35 @@ struct LocalStoreAccessor : public SourceAccessor
         , requireValidPath(requireValidPath)
     { }
 
-    Path toRealPath(const CanonPath & path)
+    CanonPath toRealPath(const CanonPath & path)
     {
-        auto storePath = store->toStorePath(path.abs()).first;
+        auto [storePath, rest] = store->toStorePath(path.abs());
         if (requireValidPath && !store->isValidPath(storePath))
             throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
-        return store->getRealStoreDir() + path.abs().substr(store->storeDir.size());
+        return CanonPath(store->getRealStoreDir()) + storePath.to_string() + CanonPath(rest);
     }
 
     std::optional<Stat> maybeLstat(const CanonPath & path) override
     {
-        auto realPath = toRealPath(path);
-
-        // FIXME: use PosixSourceAccessor.
-        struct stat st;
-        if (::lstat(realPath.c_str(), &st)) {
-            if (errno == ENOENT || errno == ENOTDIR) return std::nullopt;
-            throw SysError("getting status of '%1%'", path);
-        }
-
-        if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode))
-            throw Error("file '%1%' has unsupported type", path);
-
-        return {{
-            S_ISREG(st.st_mode) ? Type::tRegular :
-            S_ISLNK(st.st_mode) ? Type::tSymlink :
-            Type::tDirectory,
-            S_ISREG(st.st_mode) ? (uint64_t) st.st_size : 0,
-            S_ISREG(st.st_mode) && st.st_mode & S_IXUSR}};
+        return PosixSourceAccessor::maybeLstat(toRealPath(path));
     }
 
     DirEntries readDirectory(const CanonPath & path) override
     {
-        auto realPath = toRealPath(path);
-
-        auto entries = nix::readDirectory(realPath);
-
-        DirEntries res;
-        for (auto & entry : entries)
-            res.insert_or_assign(entry.name, std::nullopt);
-
-        return res;
+        return PosixSourceAccessor::readDirectory(toRealPath(path));
     }
 
-    std::string readFile(const CanonPath & path) override
+    void readFile(
+        const CanonPath & path,
+        Sink & sink,
+        std::function<void(uint64_t)> sizeCallback) override
     {
-        return nix::readFile(toRealPath(path));
+        return PosixSourceAccessor::readFile(toRealPath(path), sink, sizeCallback);
     }
 
     std::string readLink(const CanonPath & path) override
     {
-        return nix::readLink(toRealPath(path));
+        return PosixSourceAccessor::readLink(toRealPath(path));
     }
 };
 
diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc
index 8a8d64f3f..d5e32d989 100644
--- a/src/libutil/posix-source-accessor.cc
+++ b/src/libutil/posix-source-accessor.cc
@@ -58,7 +58,8 @@ std::optional<SourceAccessor::Stat> PosixSourceAccessor::maybeLstat(const CanonP
             S_ISDIR(st.st_mode) ? tDirectory :
             S_ISLNK(st.st_mode) ? tSymlink :
             tMisc,
-        .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR
+        .fileSize = S_ISREG(st.st_mode) ? std::optional<uint64_t>(st.st_size) : std::nullopt,
+        .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR,
     };
 }
 

From eab92927388bca29027a98199184ebb5e4e3c03a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= <zbyszek@in.waw.pl>
Date: Wed, 1 Nov 2023 18:10:06 +0100
Subject: [PATCH 042/294] fix: gcc complains about if which doesn't guard the
 indented statement

---
 src/libstore/build/local-derivation-goal.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 738e7051e..dcb7dc6bc 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -1563,10 +1563,11 @@ void LocalDerivationGoal::addDependency(const StorePath & path)
             Path source = worker.store.Store::toRealPath(path);
             Path target = chrootRootDir + worker.store.printStorePath(path);
 
-            if (pathExists(target))
+            if (pathExists(target)) {
                 // There is a similar debug message in doBind, so only run it in this block to not have double messages.
                 debug("bind-mounting %s -> %s", target, source);
                 throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path));
+            }
 
             /* Bind-mount the path into the sandbox. This requires
                entering its mount namespace, which is not possible

From e47984ce0b37cb8e00b66e85703c1ff72de80a73 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 1 Nov 2023 20:19:08 +0100
Subject: [PATCH 043/294] Fix whitespace

Co-authored-by: John Ericson <git@JohnEricson.me>
---
 src/libstore/binary-cache-store.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index dd9e2f3af..6a52c4c51 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -241,7 +241,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
                 for (auto & [s2, _type] : narAccessor->readDirectory(dir)) {
                     auto debugPath = dir + s2;
 
-                    if ( narAccessor->lstat(debugPath).type != SourceAccessor::tRegular
+                    if (narAccessor->lstat(debugPath).type != SourceAccessor::tRegular
                         || !std::regex_match(s2, regex2))
                         continue;
 

From d7710a40be1a871859d331e9a50cc7f31797d792 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 1 Nov 2023 20:05:23 -0400
Subject: [PATCH 044/294] flake: Temporarily get Nixpkgs ahead of Hydra
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nixpkgs':
    'github:NixOS/nixpkgs/31ed632c692e6a36cfc18083b88ece892f863ed4' (2023-09-21)
  → 'github:NixOS/nixpkgs/9eb24edd6a0027fed010ccfe300a9734d029983c' (2023-11-01)
---
 flake.lock | 8 ++++----
 flake.nix  | 4 +++-
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/flake.lock b/flake.lock
index 56df9c3fb..991cef1ee 100644
--- a/flake.lock
+++ b/flake.lock
@@ -34,16 +34,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1695283060,
-        "narHash": "sha256-CJz71xhCLlRkdFUSQEL0pIAAfcnWFXMzd9vXhPrnrEg=",
+        "lastModified": 1698876495,
+        "narHash": "sha256-nsQo2/mkDUFeAjuu92p0dEqhRvHHiENhkKVIV1y0/Oo=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "31ed632c692e6a36cfc18083b88ece892f863ed4",
+        "rev": "9eb24edd6a0027fed010ccfe300a9734d029983c",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-23.05-small",
+        "ref": "release-23.05",
         "repo": "nixpkgs",
         "type": "github"
       }
diff --git a/flake.nix b/flake.nix
index 398ba10a0..7cc4ed7fe 100644
--- a/flake.nix
+++ b/flake.nix
@@ -1,7 +1,9 @@
 {
   description = "The purely functional package manager";
 
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small";
+  # FIXME go back to nixos-23.05-small once
+  # https://github.com/NixOS/nixpkgs/pull/264875 is included.
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-23.05";
   inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
   inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
   inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; };

From 4ba8b182be350a04caf5b7efff6b804d789570ad Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Wed, 25 Oct 2023 04:50:43 +0200
Subject: [PATCH 045/294] document store objects in terms of their constituent
 parts

this also rephrases the introductory sentence to be more general, in order to
avoid the same word being repeated in short succession.
---
 doc/manual/src/SUMMARY.md.in                |  1 +
 doc/manual/src/architecture/architecture.md |  2 +-
 doc/manual/src/glossary.md                  |  2 +-
 doc/manual/src/store/index.md               |  5 +++--
 doc/manual/src/store/store-object.md        | 10 ++++++++++
 5 files changed, 16 insertions(+), 4 deletions(-)
 create mode 100644 doc/manual/src/store/store-object.md

diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in
index 2fe77d2c6..c728f5296 100644
--- a/doc/manual/src/SUMMARY.md.in
+++ b/doc/manual/src/SUMMARY.md.in
@@ -18,6 +18,7 @@
   - [Uninstalling Nix](installation/uninstall.md)
 - [Nix Store](store/index.md)
   - [File System Object](store/file-system-object.md)
+  - [Store Object](store/store-object.md)
 - [Nix Language](language/index.md)
   - [Data Types](language/values.md)
   - [Language Constructs](language/constructs.md)
diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md
index 6e832e1f9..79429508f 100644
--- a/doc/manual/src/architecture/architecture.md
+++ b/doc/manual/src/architecture/architecture.md
@@ -63,7 +63,7 @@ The command line interface and Nix expressions are what users deal with most.
 > The Nix language itself does not have a notion of *packages* or *configurations*.
 > As far as we are concerned here, the inputs and results of a build plan are just data.
 
-Underlying the command line interface and the Nix language evaluator is the [Nix store](../glossary.md#gloss-store), a mechanism to keep track of build plans, data, and references between them.
+Underlying the command line interface and the Nix language evaluator is the [Nix store](../store/index.md), a mechanism to keep track of build plans, data, and references between them.
 It can also execute build plans to produce new data, which are made available to the operating system as files.
 
 A build plan itself is a series of *build tasks*, together with their build inputs.
diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md
index ad3cc147b..b6d8a433a 100644
--- a/doc/manual/src/glossary.md
+++ b/doc/manual/src/glossary.md
@@ -59,7 +59,7 @@
 - [store]{#gloss-store}
 
   A collection of store objects, with operations to manipulate that collection.
-  See [Nix Store] for details.
+  See [Nix store](./store/index.md) for details.
 
   There are many types of stores.
   See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md) for a complete list.
diff --git a/doc/manual/src/store/index.md b/doc/manual/src/store/index.md
index 316e04179..8a5305062 100644
--- a/doc/manual/src/store/index.md
+++ b/doc/manual/src/store/index.md
@@ -1,4 +1,5 @@
 # Nix Store
 
-The *Nix store* is an abstraction used by Nix to store immutable filesystem artifacts (such as software packages) that can have dependencies (*references*) between them.
-There are multiple implementations of the Nix store, such as the actual filesystem (`/nix/store`) and binary caches.
+The *Nix store* is an abstraction to store immutable file system data (such as software packages) that can have dependencies on other such data.
+
+There are multiple implementations of Nix stores with different capabilities, such as the actual filesystem (`/nix/store`) or binary caches.
diff --git a/doc/manual/src/store/store-object.md b/doc/manual/src/store/store-object.md
new file mode 100644
index 000000000..0b2b84ea5
--- /dev/null
+++ b/doc/manual/src/store/store-object.md
@@ -0,0 +1,10 @@
+## Store Object
+
+A Nix store is a collection of *store objects* with *references* between them.
+A store object consists of
+
+  - A [file system object](./file-system-object.md) as data
+  - A set of [store paths](@docroot@/glossary.md#gloss-store-path) as references to other store objects
+
+Store objects are [immutable](https://en.wikipedia.org/wiki/Immutable_object):
+Once created, they do not change until they are deleted.

From d7b7a79f3ef865ebe5f61962a7c2737cdb5d6445 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Mon, 23 Oct 2023 01:39:26 +0200
Subject: [PATCH 046/294] document store paths

update the glossary to point to the new page.

since this is a cross-cutting concern, it warrants its own section in
the manual.

Co-authored-by: John Ericson <git@JohnEricson.me>
---
 doc/manual/src/SUMMARY.md.in         |  1 +
 doc/manual/src/glossary.md           |  9 ++--
 doc/manual/src/store/store-object.md |  2 +-
 doc/manual/src/store/store-path.md   | 69 ++++++++++++++++++++++++++++
 4 files changed, 77 insertions(+), 4 deletions(-)
 create mode 100644 doc/manual/src/store/store-path.md

diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in
index c728f5296..794f78a07 100644
--- a/doc/manual/src/SUMMARY.md.in
+++ b/doc/manual/src/SUMMARY.md.in
@@ -19,6 +19,7 @@
 - [Nix Store](store/index.md)
   - [File System Object](store/file-system-object.md)
   - [Store Object](store/store-object.md)
+  - [Store Path](store/store-path.md)
 - [Nix Language](language/index.md)
   - [Data Types](language/values.md)
   - [Language Constructs](language/constructs.md)
diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md
index b6d8a433a..07891175a 100644
--- a/doc/manual/src/glossary.md
+++ b/doc/manual/src/glossary.md
@@ -86,10 +86,13 @@
 
 - [store path]{#gloss-store-path}
 
-  The location of a [store object] in the file system, i.e., an
-  immediate child of the Nix store directory.
+  The location of a [store object](@docroot@/store/index.md#store-object) in the file system, i.e., an immediate child of the Nix store directory.
 
-  Example: `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1`
+  > **Example**
+  >
+  > `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1`
+
+  See [Store Path](@docroot@/store/store-path.md) for details.
 
   [store path]: #gloss-store-path
 
diff --git a/doc/manual/src/store/store-object.md b/doc/manual/src/store/store-object.md
index 0b2b84ea5..caf5657d1 100644
--- a/doc/manual/src/store/store-object.md
+++ b/doc/manual/src/store/store-object.md
@@ -4,7 +4,7 @@ A Nix store is a collection of *store objects* with *references* between them.
 A store object consists of
 
   - A [file system object](./file-system-object.md) as data
-  - A set of [store paths](@docroot@/glossary.md#gloss-store-path) as references to other store objects
+  - A set of [store paths](./store-path.md) as references to other store objects
 
 Store objects are [immutable](https://en.wikipedia.org/wiki/Immutable_object):
 Once created, they do not change until they are deleted.
diff --git a/doc/manual/src/store/store-path.md b/doc/manual/src/store/store-path.md
new file mode 100644
index 000000000..b5ad0c654
--- /dev/null
+++ b/doc/manual/src/store/store-path.md
@@ -0,0 +1,69 @@
+# Store Path
+
+Nix implements references to [store objects](./index.md#store-object) as *store paths*.
+
+Think of a store path as an [opaque], [unique identifier]:
+The only way to obtain store path is by adding or building store objects.
+A store path will always reference exactly one store object.
+
+[opaque]: https://en.m.wikipedia.org/wiki/Opaque_data_type
+[unique identifier]: https://en.m.wikipedia.org/wiki/Unique_identifier
+
+Store paths are pairs of
+
+- A 20-byte digest for identification
+- A symbolic name for people to read
+
+> **Example**
+>
+> - Digest: `b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z`
+> - Name:   `firefox-33.1`
+
+To make store objects accessible to operating system processes, stores have to expose store objects through the file system.
+
+A store path is rendered to a file system path as the concatenation of
+
+- [Store directory](#store-directory) (typically `/nix/store`)
+- Path separator (`/`)
+- Digest rendered in a custom variant of [Base32](https://en.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters)
+- Hyphen (`-`)
+- Name
+
+> **Example**
+>
+> ```
+>   /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1
+>   |--------| |------------------------------| |----------|
+> store directory            digest                 name
+> ```
+
+## Store Directory
+
+Every [Nix store](./index.md) has a store directory.
+
+Not every store can be accessed through the file system.
+But if the store has a file system representation, the store directory contains the store’s [file system objects], which can be addressed by [store paths](#store-path).
+
+[file system objects]: ./file-system-object.md
+
+This means a store path is not just derived from the referenced store object itself, but depends on the store the store object is in.
+
+> **Note**
+>
+> The store directory defaults to `/nix/store`, but is in principle arbitrary.
+
+It is important which store a given store object belongs to:
+Files in the store object can contain store paths, and processes may read these paths.
+Nix can only guarantee referential integrity if store paths do not cross store boundaries.
+
+Therefore one can only copy store objects to a different store if
+
+- The source and target stores' directories match
+
+  or
+
+- The store object in question has no references, that is, contains no store paths
+
+One cannot copy a store object to a store with a different store directory.
+Instead, it has to be rebuilt, together with all its dependencies.
+It is in general not enough to replace the store directory string in file contents, as this may render executables unusable by invalidating their internal offsets or checksums.

From 55ed09c4f251d87e5aa23c7fb931e87cea63c68d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= <zbyszek@in.waw.pl>
Date: Thu, 2 Nov 2023 09:22:00 +0100
Subject: [PATCH 047/294] Remove stray executable permissions on source files

Noticed because of a warning during an rpm build:
*** WARNING: ./usr/src/debug/nix-2.18.1-1.fc40.x86_64/src/nix-copy-closure/nix-copy-closure.cc is executable but has no shebang, removing executable bit
*** WARNING: ./usr/src/debug/nix-2.18.1-1.fc40.x86_64/src/nix-channel/nix-channel.cc is executable but has no shebang, removing executable bit
---
 src/nix-channel/nix-channel.cc           | 0
 src/nix-copy-closure/nix-copy-closure.cc | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 mode change 100755 => 100644 src/nix-channel/nix-channel.cc
 mode change 100755 => 100644 src/nix-copy-closure/nix-copy-closure.cc

diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
old mode 100755
new mode 100644
diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc
old mode 100755
new mode 100644

From d26c317b14bc3f0ce82d5a91acc63e62a8836dee Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 2 Nov 2023 13:40:54 +0100
Subject: [PATCH 048/294] Use expect

Co-authored-by: John Ericson <git@JohnEricson.me>
---
 tests/functional/nar-access.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/functional/nar-access.sh b/tests/functional/nar-access.sh
index 13d23c342..218b521fb 100644
--- a/tests/functional/nar-access.sh
+++ b/tests/functional/nar-access.sh
@@ -28,7 +28,7 @@ diff -u baz.cat-nar $storePath/foo/baz
 # Check that 'nix store cat' fails on invalid store paths.
 invalidPath="$(dirname $storePath)/99999999999999999999999999999999-foo"
 mv $storePath $invalidPath
-(! nix store cat $invalidPath/foo/baz)
+expect 1 nix store cat $invalidPath/foo/baz
 mv $invalidPath $storePath
 
 # Test --json.

From b107431816fcbf364aeae6942cc9d1e709635a44 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 1 Nov 2023 11:15:21 -0400
Subject: [PATCH 049/294] Systematize characterization tests a bit more

Deduplicating code moreover enforcing the pattern means:

 - It is easier to write new characterization tests because less boilerplate

 - It is harder to mess up new tests because there are fewer places to
   make mistakes.

Co-authored-by: Jacek Galowicz <jacek@galowicz.de>
---
 src/libstore/tests/characterization.hh |  28 ------
 src/libstore/tests/common-protocol.cc  |  48 ++++------
 src/libstore/tests/derivation.cc       | 116 ++++++++-----------------
 src/libstore/tests/libstore.hh         |   2 +-
 src/libstore/tests/protocol.hh         |  61 +++++--------
 src/libutil/tests/characterization.hh  | 111 +++++++++++++++++++++++
 6 files changed, 183 insertions(+), 183 deletions(-)
 delete mode 100644 src/libstore/tests/characterization.hh
 create mode 100644 src/libutil/tests/characterization.hh

diff --git a/src/libstore/tests/characterization.hh b/src/libstore/tests/characterization.hh
deleted file mode 100644
index 46bf4b2e5..000000000
--- a/src/libstore/tests/characterization.hh
+++ /dev/null
@@ -1,28 +0,0 @@
-#pragma once
-///@file
-
-namespace nix {
-
-/**
- * The path to the `unit-test-data` directory. See the contributing
- * guide in the manual for further details.
- */
-static Path getUnitTestData() {
-    return getEnv("_NIX_TEST_UNIT_DATA").value();
-}
-
-/**
- * Whether we should update "golden masters" instead of running tests
- * against them. See the contributing guide in the manual for further
- * details.
- */
-static bool testAccept() {
-    return getEnv("_NIX_TEST_ACCEPT") == "1";
-}
-
-constexpr std::string_view cannotReadGoldenMaster =
-    "Cannot read golden master because another test is also updating it";
-
-constexpr std::string_view updatingGoldenMaster =
-    "Updating golden master";
-}
diff --git a/src/libstore/tests/common-protocol.cc b/src/libstore/tests/common-protocol.cc
index b3f4977d2..c09ac6a3e 100644
--- a/src/libstore/tests/common-protocol.cc
+++ b/src/libstore/tests/common-protocol.cc
@@ -20,16 +20,9 @@ public:
      * Golden test for `T` reading
      */
     template<typename T>
-    void readTest(PathView testStem, T value)
+    void readProtoTest(PathView testStem, const T & expected)
     {
-        if (testAccept())
-        {
-            GTEST_SKIP() << cannotReadGoldenMaster;
-        }
-        else
-        {
-            auto encoded = readFile(goldenMaster(testStem));
-
+        CharacterizationTest::readTest(testStem, [&](const auto & encoded) {
             T got = ({
                 StringSource from { encoded };
                 CommonProto::Serialise<T>::read(
@@ -37,44 +30,33 @@ public:
                     CommonProto::ReadConn { .from = from });
             });
 
-            ASSERT_EQ(got, value);
-        }
+            ASSERT_EQ(got, expected);
+        });
     }
 
     /**
      * Golden test for `T` write
      */
     template<typename T>
-    void writeTest(PathView testStem, const T & value)
+    void writeProtoTest(PathView testStem, const T & decoded)
     {
-        auto file = goldenMaster(testStem);
-
-        StringSink to;
-        CommonProto::write(
-            *store,
-            CommonProto::WriteConn { .to = to },
-            value);
-
-        if (testAccept())
-        {
-            createDirs(dirOf(file));
-            writeFile(file, to.s);
-            GTEST_SKIP() << updatingGoldenMaster;
-        }
-        else
-        {
-            auto expected = readFile(file);
-            ASSERT_EQ(to.s, expected);
-        }
+        CharacterizationTest::writeTest(testStem, [&]() -> std::string {
+            StringSink to;
+            CommonProto::Serialise<T>::write(
+                *store,
+                CommonProto::WriteConn { .to = to },
+                decoded);
+            return to.s;
+        });
     }
 };
 
 #define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
     TEST_F(CommonProtoTest, NAME ## _read) { \
-        readTest(STEM, VALUE); \
+        readProtoTest(STEM, VALUE); \
     } \
     TEST_F(CommonProtoTest, NAME ## _write) { \
-        writeTest(STEM, VALUE); \
+        writeProtoTest(STEM, VALUE); \
     }
 
 CHARACTERIZATION_TEST(
diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc
index ca0cdff71..29d5693db 100644
--- a/src/libstore/tests/derivation.cc
+++ b/src/libstore/tests/derivation.cc
@@ -11,20 +11,20 @@ namespace nix {
 
 using nlohmann::json;
 
-class DerivationTest : public LibStoreTest
+class DerivationTest : public CharacterizationTest, public LibStoreTest
 {
+    Path unitTestData = getUnitTestData() + "/libstore/derivation";
+
 public:
+    Path goldenMaster(std::string_view testStem) const override {
+        return unitTestData + "/" + testStem;
+    }
+
     /**
      * We set these in tests rather than the regular globals so we don't have
      * to worry about race conditions if the tests run concurrently.
      */
     ExperimentalFeatureSettings mockXpSettings;
-
-    Path unitTestData = getUnitTestData() + "/libstore/derivation";
-
-    Path goldenMaster(std::string_view testStem) {
-        return unitTestData + "/" + testStem;
-    }
 };
 
 class CaDerivationTest : public DerivationTest
@@ -73,14 +73,8 @@ TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) {
 
 #define TEST_JSON(FIXTURE, NAME, VAL, DRV_NAME, OUTPUT_NAME)              \
     TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _from_json) {            \
-        if (testAccept())                                                 \
-        {                                                                 \
-            GTEST_SKIP() << cannotReadGoldenMaster;                       \
-        }                                                                 \
-        else                                                              \
-        {                                                                 \
-            auto encoded = json::parse(                                   \
-                readFile(goldenMaster("output-" #NAME ".json")));         \
+        readTest("output-" #NAME ".json", [&](const auto & encoded_) {    \
+            auto encoded = json::parse(encoded_);                         \
             DerivationOutput got = DerivationOutput::fromJSON(            \
                 *store,                                                   \
                 DRV_NAME,                                                 \
@@ -89,28 +83,20 @@ TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) {
                 mockXpSettings);                                          \
             DerivationOutput expected { VAL };                            \
             ASSERT_EQ(got, expected);                                     \
-        }                                                                 \
+        });                                                               \
     }                                                                     \
                                                                           \
     TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) {              \
-        auto file = goldenMaster("output-" #NAME ".json");                \
-                                                                          \
-        json got = DerivationOutput { VAL }.toJSON(                       \
-            *store,                                                       \
-            DRV_NAME,                                                     \
-            OUTPUT_NAME);                                                 \
-                                                                          \
-        if (testAccept())                                                 \
-        {                                                                 \
-            createDirs(dirOf(file));                                      \
-            writeFile(file, got.dump(2) + "\n");                          \
-            GTEST_SKIP() << updatingGoldenMaster;                         \
-        }                                                                 \
-        else                                                              \
-        {                                                                 \
-            auto expected = json::parse(readFile(file));                  \
-            ASSERT_EQ(got, expected);                                     \
-        }                                                                 \
+        writeTest<json>("output-" #NAME ".json", [&]() -> json {          \
+            return DerivationOutput { (VAL) }.toJSON(                     \
+                *store,                                                   \
+                (DRV_NAME),                                               \
+                (OUTPUT_NAME));                                           \
+        }, [](const auto & file) {                                        \
+            return json::parse(readFile(file));                           \
+        }, [](const auto & file, const auto & got) {                      \
+            return writeFile(file, got.dump(2) + "\n");                   \
+        });                                                               \
     }
 
 TEST_JSON(DerivationTest, inputAddressed,
@@ -167,50 +153,30 @@ TEST_JSON(ImpureDerivationTest, impure,
 
 #define TEST_JSON(FIXTURE, NAME, VAL)                                     \
     TEST_F(FIXTURE, Derivation_ ## NAME ## _from_json) {                  \
-        if (testAccept())                                                 \
-        {                                                                 \
-            GTEST_SKIP() << cannotReadGoldenMaster;                       \
-        }                                                                 \
-        else                                                              \
-        {                                                                 \
-            auto encoded = json::parse(                                   \
-                readFile(goldenMaster( #NAME ".json")));                  \
+        readTest(#NAME ".json", [&](const auto & encoded_) {              \
+            auto encoded = json::parse(encoded_);                         \
             Derivation expected { VAL };                                  \
             Derivation got = Derivation::fromJSON(                        \
                 *store,                                                   \
                 encoded,                                                  \
                 mockXpSettings);                                          \
             ASSERT_EQ(got, expected);                                     \
-        }                                                                 \
+        });                                                               \
     }                                                                     \
                                                                           \
     TEST_F(FIXTURE, Derivation_ ## NAME ## _to_json) {                    \
-        auto file = goldenMaster( #NAME ".json");                         \
-                                                                          \
-        json got = Derivation { VAL }.toJSON(*store);                     \
-                                                                          \
-        if (testAccept())                                                 \
-        {                                                                 \
-            createDirs(dirOf(file));                                      \
-            writeFile(file, got.dump(2) + "\n");                          \
-            GTEST_SKIP() << updatingGoldenMaster;                         \
-        }                                                                 \
-        else                                                              \
-        {                                                                 \
-            auto expected = json::parse(readFile(file));                  \
-            ASSERT_EQ(got, expected);                                     \
-        }                                                                 \
+        writeTest<json>(#NAME ".json", [&]() -> json {                    \
+            return Derivation { VAL }.toJSON(*store);                     \
+        }, [](const auto & file) {                                        \
+            return json::parse(readFile(file));                           \
+        }, [](const auto & file, const auto & got) {                      \
+            return writeFile(file, got.dump(2) + "\n");                   \
+        });                                                               \
     }
 
 #define TEST_ATERM(FIXTURE, NAME, VAL, DRV_NAME)                          \
     TEST_F(FIXTURE, Derivation_ ## NAME ## _from_aterm) {                 \
-        if (testAccept())                                                 \
-        {                                                                 \
-            GTEST_SKIP() << cannotReadGoldenMaster;                       \
-        }                                                                 \
-        else                                                              \
-        {                                                                 \
-            auto encoded = readFile(goldenMaster( #NAME ".drv"));         \
+        readTest(#NAME ".drv", [&](auto encoded) {                        \
             Derivation expected { VAL };                                  \
             auto got = parseDerivation(                                   \
                 *store,                                                   \
@@ -219,25 +185,13 @@ TEST_JSON(ImpureDerivationTest, impure,
                 mockXpSettings);                                          \
             ASSERT_EQ(got.toJSON(*store), expected.toJSON(*store)) ;      \
             ASSERT_EQ(got, expected);                                     \
-        }                                                                 \
+        });                                                               \
     }                                                                     \
                                                                           \
     TEST_F(FIXTURE, Derivation_ ## NAME ## _to_aterm) {                   \
-        auto file = goldenMaster( #NAME ".drv");                          \
-                                                                          \
-        auto got = (VAL).unparse(*store, false);                          \
-                                                                          \
-        if (testAccept())                                                 \
-        {                                                                 \
-            createDirs(dirOf(file));                                      \
-            writeFile(file, got);                                         \
-            GTEST_SKIP() << updatingGoldenMaster;                         \
-        }                                                                 \
-        else                                                              \
-        {                                                                 \
-            auto expected = readFile(file);                               \
-            ASSERT_EQ(got, expected);                                     \
-        }                                                                 \
+        writeTest(#NAME ".drv", [&]() -> std::string {                    \
+            return (VAL).unparse(*store, false);                          \
+        });                                                               \
     }
 
 Derivation makeSimpleDrv(const Store & store) {
diff --git a/src/libstore/tests/libstore.hh b/src/libstore/tests/libstore.hh
index ef93457b5..78b162b95 100644
--- a/src/libstore/tests/libstore.hh
+++ b/src/libstore/tests/libstore.hh
@@ -8,7 +8,7 @@
 
 namespace nix {
 
-class LibStoreTest : public ::testing::Test {
+class LibStoreTest : public virtual ::testing::Test {
     public:
         static void SetUpTestSuite() {
             initLibStore();
diff --git a/src/libstore/tests/protocol.hh b/src/libstore/tests/protocol.hh
index 7fdd3e11c..0378b3e1f 100644
--- a/src/libstore/tests/protocol.hh
+++ b/src/libstore/tests/protocol.hh
@@ -7,12 +7,11 @@
 namespace nix {
 
 template<class Proto, const char * protocolDir>
-class ProtoTest : public LibStoreTest
+class ProtoTest : public CharacterizationTest, public LibStoreTest
 {
-protected:
     Path unitTestData = getUnitTestData() + "/libstore/" + protocolDir;
 
-    Path goldenMaster(std::string_view testStem) {
+    Path goldenMaster(std::string_view testStem) const override {
         return unitTestData + "/" + testStem + ".bin";
     }
 };
@@ -25,18 +24,11 @@ public:
      * Golden test for `T` reading
      */
     template<typename T>
-    void readTest(PathView testStem, typename Proto::Version version, T value)
+    void readProtoTest(PathView testStem, typename Proto::Version version, T expected)
     {
-        if (testAccept())
-        {
-            GTEST_SKIP() << cannotReadGoldenMaster;
-        }
-        else
-        {
-            auto expected = readFile(ProtoTest<Proto, protocolDir>::goldenMaster(testStem));
-
+        CharacterizationTest::readTest(testStem, [&](const auto & encoded) {
             T got = ({
-                StringSource from { expected };
+                StringSource from { encoded };
                 Proto::template Serialise<T>::read(
                     *LibStoreTest::store,
                     typename Proto::ReadConn {
@@ -45,47 +37,36 @@ public:
                     });
             });
 
-            ASSERT_EQ(got, value);
-        }
+            ASSERT_EQ(got, expected);
+        });
     }
 
     /**
      * Golden test for `T` write
      */
     template<typename T>
-    void writeTest(PathView testStem, typename Proto::Version version, const T & value)
+    void writeProtoTest(PathView testStem, typename Proto::Version version, const T & decoded)
     {
-        auto file = ProtoTest<Proto, protocolDir>::goldenMaster(testStem);
-
-        StringSink to;
-        Proto::write(
-            *LibStoreTest::store,
-            typename Proto::WriteConn {
-                .to = to,
-                .version = version,
-            },
-            value);
-
-        if (testAccept())
-        {
-            createDirs(dirOf(file));
-            writeFile(file, to.s);
-            GTEST_SKIP() << updatingGoldenMaster;
-        }
-        else
-        {
-            auto expected = readFile(file);
-            ASSERT_EQ(to.s, expected);
-        }
+        CharacterizationTest::writeTest(testStem, [&]() {
+            StringSink to;
+            Proto::template Serialise<T>::write(
+                *LibStoreTest::store,
+                typename Proto::WriteConn {
+                    .to = to,
+                    .version = version,
+                },
+                decoded);
+            return std::move(to.s);
+        });
     }
 };
 
 #define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
     TEST_F(FIXTURE, NAME ## _read) { \
-        readTest(STEM, VERSION, VALUE); \
+        readProtoTest(STEM, VERSION, VALUE); \
     } \
     TEST_F(FIXTURE, NAME ## _write) { \
-        writeTest(STEM, VERSION, VALUE); \
+        writeProtoTest(STEM, VERSION, VALUE); \
     }
 
 }
diff --git a/src/libutil/tests/characterization.hh b/src/libutil/tests/characterization.hh
new file mode 100644
index 000000000..10c8b4f7e
--- /dev/null
+++ b/src/libutil/tests/characterization.hh
@@ -0,0 +1,111 @@
+#pragma once
+///@file
+
+#include <gtest/gtest.h>
+
+#include "types.hh"
+
+namespace nix {
+
+/**
+ * The path to the `unit-test-data` directory. See the contributing
+ * guide in the manual for further details.
+ */
+static Path getUnitTestData() {
+    return getEnv("_NIX_TEST_UNIT_DATA").value();
+}
+
+/**
+ * Whether we should update "golden masters" instead of running tests
+ * against them. See the contributing guide in the manual for further
+ * details.
+ */
+static bool testAccept() {
+    return getEnv("_NIX_TEST_ACCEPT") == "1";
+}
+
+/**
+ * Mixin class for writing characterization tests
+ */
+class CharacterizationTest : public virtual ::testing::Test
+{
+protected:
+    /**
+     * While the "golden master" for this characterization test is
+     * located. It should not be shared with any other test.
+     */
+    virtual Path goldenMaster(PathView testStem) const = 0;
+
+public:
+    /**
+     * Golden test for reading
+     *
+     * @param test hook that takes the contents of the file and does the
+     * actual work
+     */
+    void readTest(PathView testStem, auto && test)
+    {
+        auto file = goldenMaster(testStem);
+
+        if (testAccept())
+        {
+            GTEST_SKIP()
+                << "Cannot read golden master "
+                << file
+                << "because another test is also updating it";
+        }
+        else
+        {
+            test(readFile(file));
+        }
+    }
+
+    /**
+     * Golden test for writing
+     *
+     * @param test hook that produces contents of the file and does the
+     * actual work
+     */
+    template<typename T>
+    void writeTest(
+        PathView testStem,
+        std::invocable<> auto && test,
+        std::invocable<const Path &> auto && readFile2,
+        std::invocable<const Path &, const T &> auto && writeFile2)
+    {
+        auto file = goldenMaster(testStem);
+
+        T got = test();
+
+        if (testAccept())
+        {
+            createDirs(dirOf(file));
+            writeFile2(file, got);
+            GTEST_SKIP()
+                << "Updating golden master "
+                << file;
+        }
+        else
+        {
+            T expected = readFile2(file);
+            ASSERT_EQ(got, expected);
+        }
+    }
+
+    /**
+     * Specialize to `std::string`
+     */
+    void writeTest(PathView testStem, auto && test)
+    {
+        writeTest<std::string>(
+            testStem, test,
+            [](const Path & f) -> std::string {
+                return readFile(f);
+            },
+            [](const Path & f, const std::string & c) {
+                return writeFile(f, c);
+            });
+    }
+};
+
+}

From d15c3a33e680228c9deaa6d0898d4680cdc8dbc3 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 1 Nov 2023 16:11:20 -0400
Subject: [PATCH 050/294] Don't use `std::invocable` C++ concept yet

It s not supported on all platforms yet. Can revert this once it is.
---
 src/libstore/tests/derivation.cc      |  4 ++--
 src/libutil/tests/characterization.hh | 12 ++++--------
 2 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc
index 29d5693db..7becfa5ab 100644
--- a/src/libstore/tests/derivation.cc
+++ b/src/libstore/tests/derivation.cc
@@ -87,7 +87,7 @@ TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) {
     }                                                                     \
                                                                           \
     TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) {              \
-        writeTest<json>("output-" #NAME ".json", [&]() -> json {          \
+        writeTest("output-" #NAME ".json", [&]() -> json {                \
             return DerivationOutput { (VAL) }.toJSON(                     \
                 *store,                                                   \
                 (DRV_NAME),                                               \
@@ -165,7 +165,7 @@ TEST_JSON(ImpureDerivationTest, impure,
     }                                                                     \
                                                                           \
     TEST_F(FIXTURE, Derivation_ ## NAME ## _to_json) {                    \
-        writeTest<json>(#NAME ".json", [&]() -> json {                    \
+        writeTest(#NAME ".json", [&]() -> json {                          \
             return Derivation { VAL }.toJSON(*store);                     \
         }, [](const auto & file) {                                        \
             return json::parse(readFile(file));                           \
diff --git a/src/libutil/tests/characterization.hh b/src/libutil/tests/characterization.hh
index 10c8b4f7e..6698c5239 100644
--- a/src/libutil/tests/characterization.hh
+++ b/src/libutil/tests/characterization.hh
@@ -66,16 +66,12 @@ public:
      * @param test hook that produces contents of the file and does the
      * actual work
      */
-    template<typename T>
     void writeTest(
-        PathView testStem,
-        std::invocable<> auto && test,
-        std::invocable<const Path &> auto && readFile2,
-        std::invocable<const Path &, const T &> auto && writeFile2)
+        PathView testStem, auto && test, auto && readFile2, auto && writeFile2)
     {
         auto file = goldenMaster(testStem);
 
-        T got = test();
+        auto got = test();
 
         if (testAccept())
         {
@@ -87,7 +83,7 @@ public:
         }
         else
         {
-            T expected = readFile2(file);
+            decltype(got) expected = readFile2(file);
             ASSERT_EQ(got, expected);
         }
     }
@@ -97,7 +93,7 @@ public:
      */
     void writeTest(PathView testStem, auto && test)
     {
-        writeTest<std::string>(
+        writeTest(
             testStem, test,
             [](const Path & f) -> std::string {
                 return readFile(f);

From e5908212e25f2cb7a36ec176a1c7fcb2d522088b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 3 Nov 2023 11:03:58 +0100
Subject: [PATCH 051/294] Fix nar-access test on macOS

---
 tests/functional/nar-access.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/functional/nar-access.sh b/tests/functional/nar-access.sh
index 218b521fb..87981e7d9 100644
--- a/tests/functional/nar-access.sh
+++ b/tests/functional/nar-access.sh
@@ -27,9 +27,8 @@ diff -u baz.cat-nar $storePath/foo/baz
 
 # Check that 'nix store cat' fails on invalid store paths.
 invalidPath="$(dirname $storePath)/99999999999999999999999999999999-foo"
-mv $storePath $invalidPath
+cp -r $storePath $invalidPath
 expect 1 nix store cat $invalidPath/foo/baz
-mv $invalidPath $storePath
 
 # Test --json.
 diff -u \

From 55dd1244d280d768bfebb8ca2ec93e061d7aa4eb Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 3 Nov 2023 11:39:50 +0100
Subject: [PATCH 052/294] parseDerivation(): Fix warning about uninitialized
 'version' variable

---
 src/libstore/derivations.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index efdad18e1..1fecd1c97 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -352,7 +352,7 @@ Derivation parseDerivation(
         expect(str, "erive(");
         version = DerivationATermVersion::Traditional;
         break;
-    case 'r':
+    case 'r': {
         expect(str, "rvWithVersion(");
         auto versionS = parseString(str);
         if (versionS == "xp-dyn-drv") {
@@ -365,6 +365,9 @@ Derivation parseDerivation(
         expect(str, ",");
         break;
     }
+    default:
+        throw Error("derivation does not start with 'Derive' or 'DrvWithVersion'");
+    }
 
     /* Parse the list of outputs. */
     expect(str, "[");

From b0455e9931fbcd996b1b240a4513132c36cf852c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 3 Nov 2023 11:58:47 +0100
Subject: [PATCH 053/294] Fix uninitialized variable warnings on i686-linux

https://hydra.nixos.org/build/239849607
---
 src/libcmd/command.cc      | 4 ++--
 src/libcmd/installables.cc | 2 +-
 src/libstore/store-api.cc  | 2 +-
 src/nix-build/nix-build.cc | 2 +-
 src/nix-env/nix-env.cc     | 4 ++--
 5 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc
index a88ba8134..de9f546fc 100644
--- a/src/libcmd/command.cc
+++ b/src/libcmd/command.cc
@@ -175,7 +175,7 @@ void BuiltPathsCommand::run(ref<Store> store, Installables && installables)
             throw UsageError("'--all' does not expect arguments");
         // XXX: Only uses opaque paths, ignores all the realisations
         for (auto & p : store->queryAllValidPaths())
-            paths.push_back(BuiltPath::Opaque{p});
+            paths.emplace_back(BuiltPath::Opaque{p});
     } else {
         paths = Installable::toBuiltPaths(getEvalStore(), store, realiseMode, operateOn, installables);
         if (recursive) {
@@ -188,7 +188,7 @@ void BuiltPathsCommand::run(ref<Store> store, Installables && installables)
             }
             store->computeFSClosure(pathsRoots, pathsClosure);
             for (auto & path : pathsClosure)
-                paths.push_back(BuiltPath::Opaque{path});
+                paths.emplace_back(BuiltPath::Opaque{path});
         }
     }
 
diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index 3aff601e0..bc0b8a988 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -663,7 +663,7 @@ BuiltPaths Installable::toBuiltPaths(
 
         BuiltPaths res;
         for (auto & drvPath : Installable::toDerivations(store, installables, true))
-            res.push_back(BuiltPath::Opaque{drvPath});
+            res.emplace_back(BuiltPath::Opaque{drvPath});
         return res;
     }
 }
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index ac96e8bb1..646b0ec7d 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -819,7 +819,7 @@ void Store::substitutePaths(const StorePathSet & paths)
     std::vector<DerivedPath> paths2;
     for (auto & path : paths)
         if (!path.isDerivation())
-            paths2.push_back(DerivedPath::Opaque{path});
+            paths2.emplace_back(DerivedPath::Opaque{path});
     uint64_t downloadSize, narSize;
     StorePathSet willBuild, willSubstitute, unknown;
     queryMissing(paths2,
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index e62c4f6b1..60bc08146 100644
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -449,7 +449,7 @@ static void main_nix_build(int argc, char * * argv)
             }
         }
         for (const auto & src : drv.inputSrcs) {
-            pathsToBuild.push_back(DerivedPath::Opaque{src});
+            pathsToBuild.emplace_back(DerivedPath::Opaque{src});
             pathsToCopy.insert(src);
         }
 
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 01742daa8..25068f801 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -481,12 +481,12 @@ static void printMissing(EvalState & state, DrvInfos & elems)
     std::vector<DerivedPath> targets;
     for (auto & i : elems)
         if (auto drvPath = i.queryDrvPath())
-            targets.push_back(DerivedPath::Built{
+            targets.emplace_back(DerivedPath::Built{
                 .drvPath = makeConstantStorePathRef(*drvPath),
                 .outputs = OutputsSpec::All { },
             });
         else
-            targets.push_back(DerivedPath::Opaque{
+            targets.emplace_back(DerivedPath::Opaque{
                 .path = i.queryOutPath(),
             });
 

From 60b363936d2fd53ac8741d35ba30ff1e4c405a9f Mon Sep 17 00:00:00 2001
From: r-vdp <ramses@well-founded.dev>
Date: Tue, 31 Oct 2023 17:32:09 +0100
Subject: [PATCH 054/294] libstore/ssh-ng: Fix phase reporting in log files.

When doing local builds, we get phase reporting lines in the log file,
they look like '@nix {"action":"setPhase","phase":"unpackPhase"}'.
With the ssh-ng protocol, we do have access to these messages, but since we
are only including messages of type resBuildLogLine in the logs, the phase
information does not end up in the log file.

The phase reporting could probably be improved altoghether (it looks like it
is kind of accidental that these JSON messages for phase reporting show up
but others don't, just because they are actually emitted by nixpkgs' stdenv),
but as a first step I propose to make ssh-ng behave in the same way as local builds do.
---
 src/libstore/build/derivation-goal.cc |  23 +++++-
 tests/nixos/default.nix               |   2 +
 tests/nixos/remote-builds-ssh-ng.nix  | 108 ++++++++++++++++++++++++++
 3 files changed, 130 insertions(+), 3 deletions(-)
 create mode 100644 tests/nixos/remote-builds-ssh-ng.nix

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 360c6b70b..0cfa9a148 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -1317,9 +1317,26 @@ void DerivationGoal::handleChildOutput(int fd, std::string_view data)
                     auto s = handleJSONLogMessage(*json, worker.act, hook->activities, true);
                     // ensure that logs from a builder using `ssh-ng://` as protocol
                     // are also available to `nix log`.
-                    if (s && !isWrittenToLog && logSink && (*json)["type"] == resBuildLogLine) {
-                        auto f = (*json)["fields"];
-                        (*logSink)((f.size() > 0 ? f.at(0).get<std::string>() : "") + "\n");
+                    if (s && !isWrittenToLog && logSink) {
+                        const auto type = (*json)["type"];
+                        const auto fields = (*json)["fields"];
+                        if (type == resBuildLogLine) {
+                            (*logSink)((fields.size() > 0 ? fields[0].get<std::string>() : "") + "\n");
+                        } else if (type == resSetPhase && ! fields.is_null()) {
+                            const auto phase = fields[0];
+                            if (! phase.is_null()) {
+                                // nixpkgs' stdenv produces lines in the log to signal
+                                // phase changes.
+                                // We want to get the same lines in case of remote builds.
+                                // The format is:
+                                //   @nix { "action": "setPhase", "phase": "$curPhase" }
+                                const auto logLine = nlohmann::json::object({
+                                    {"action", "setPhase"},
+                                    {"phase", phase}
+                                });
+                                (*logSink)("@nix " + logLine.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace) + "\n");
+                            }
+                        }
                     }
                 }
                 currentHookLine.clear();
diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix
index b391d7ef2..4459aa664 100644
--- a/tests/nixos/default.nix
+++ b/tests/nixos/default.nix
@@ -21,6 +21,8 @@ in
 
   remoteBuilds = runNixOSTestFor "x86_64-linux" ./remote-builds.nix;
 
+  remoteBuildsSshNg = runNixOSTestFor "x86_64-linux" ./remote-builds-ssh-ng.nix;
+
   nix-copy-closure = runNixOSTestFor "x86_64-linux" ./nix-copy-closure.nix;
 
   nix-copy = runNixOSTestFor "x86_64-linux" ./nix-copy.nix;
diff --git a/tests/nixos/remote-builds-ssh-ng.nix b/tests/nixos/remote-builds-ssh-ng.nix
new file mode 100644
index 000000000..b59dde9bf
--- /dev/null
+++ b/tests/nixos/remote-builds-ssh-ng.nix
@@ -0,0 +1,108 @@
+{ config, lib, hostPkgs, ... }:
+
+let
+  pkgs = config.nodes.client.nixpkgs.pkgs;
+
+  # Trivial Nix expression to build remotely.
+  expr = config: nr: pkgs.writeText "expr.nix"
+    ''
+      let utils = builtins.storePath ${config.system.build.extraUtils}; in
+      derivation {
+        name = "hello-${toString nr}";
+        system = "i686-linux";
+        PATH = "''${utils}/bin";
+        builder = "''${utils}/bin/sh";
+        args = [ "-c" "${
+          lib.concatStringsSep "; " [
+            ''if [[ -n $NIX_LOG_FD ]]''
+            ''then echo '@nix {\"action\":\"setPhase\",\"phase\":\"buildPhase\"}' >&''$NIX_LOG_FD''
+            "fi"
+            "echo Hello"
+            "mkdir $out"
+            "cat /proc/sys/kernel/hostname > $out/host"
+          ]
+        }" ];
+        outputs = [ "out" ];
+      }
+    '';
+in
+
+{
+  name = "remote-builds-ssh-ng";
+
+  nodes =
+    { builder =
+      { config, pkgs, ... }:
+      { services.openssh.enable = true;
+        virtualisation.writableStore = true;
+        nix.settings.sandbox = true;
+        nix.settings.substituters = lib.mkForce [ ];
+      };
+
+      client =
+        { config, lib, pkgs, ... }:
+        { nix.settings.max-jobs = 0; # force remote building
+          nix.distributedBuilds = true;
+          nix.buildMachines =
+            [ { hostName = "builder";
+                sshUser = "root";
+                sshKey = "/root/.ssh/id_ed25519";
+                system = "i686-linux";
+                maxJobs = 1;
+                protocol = "ssh-ng";
+              }
+            ];
+          virtualisation.writableStore = true;
+          virtualisation.additionalPaths = [ config.system.build.extraUtils ];
+          nix.settings.substituters = lib.mkForce [ ];
+          programs.ssh.extraConfig = "ConnectTimeout 30";
+        };
+    };
+
+  testScript = { nodes }: ''
+    # fmt: off
+    import subprocess
+
+    start_all()
+
+    # Create an SSH key on the client.
+    subprocess.run([
+      "${hostPkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", ""
+    ], capture_output=True, check=True)
+    client.succeed("mkdir -p -m 700 /root/.ssh")
+    client.copy_from_host("key", "/root/.ssh/id_ed25519")
+    client.succeed("chmod 600 /root/.ssh/id_ed25519")
+
+    # Install the SSH key on the builder.
+    client.wait_for_unit("network.target")
+    builder.succeed("mkdir -p -m 700 /root/.ssh")
+    builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys")
+    builder.wait_for_unit("sshd")
+    client.succeed(f"ssh -o StrictHostKeyChecking=no {builder.name} 'echo hello world'")
+
+    # Perform a build
+    out = client.succeed("nix-build ${expr nodes.client.config 1} 2> build-output")
+
+    # Verify that the build was done on the builder
+    builder.succeed(f"test -e {out.strip()}")
+
+    # Print the build log, prefix the log lines to avoid nix intercepting lines starting with @nix
+    buildOutput = client.succeed("sed -e 's/^/build-output:/' build-output")
+    print(buildOutput)
+
+    # Make sure that we get the expected build output
+    client.succeed("grep -qF Hello build-output")
+
+    # We don't want phase reporting in the build output
+    client.fail("grep -qF '@nix' build-output")
+
+    # Get the log file
+    client.succeed(f"nix-store --read-log {out.strip()} > log-output")
+    # Prefix the log lines to avoid nix intercepting lines starting with @nix
+    logOutput = client.succeed("sed -e 's/^/log-file:/' log-output")
+    print(logOutput)
+
+    # Check that we get phase reporting in the log file
+    client.succeed("grep -q '@nix {\"action\":\"setPhase\",\"phase\":\"buildPhase\"}' log-output")
+  '';
+}

From 6df32889a51510dff44c776fa312b7ba61ab8edf Mon Sep 17 00:00:00 2001
From: BootRhetoric <110117466+BootRhetoric@users.noreply.github.com>
Date: Fri, 20 Oct 2023 21:16:56 +0200
Subject: [PATCH 055/294] Add git commit verification input attributes

This implements the git input attributes `verifyCommit`, `keytype`,
`publicKey` and `publicKeys` as experimental feature
`verified-fetches`. `publicKeys` should be a json string.
This representation was chosen because all attributes must be of type bool,
int or string so they can be included in flake uris (see definition of
fetchers::Attr).
---
 src/libfetchers/fetchers.cc          |   5 ++
 src/libfetchers/fetchers.hh          |   9 +++
 src/libfetchers/git.cc               | 104 +++++++++++++++++++++++++--
 src/libutil/experimental-features.cc |  11 ++-
 src/libutil/experimental-features.hh |   1 +
 5 files changed, 124 insertions(+), 6 deletions(-)

diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index 92692d23a..895515327 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -360,4 +360,9 @@ std::optional<ExperimentalFeature> InputScheme::experimentalFeature() const
     return {};
 }
 
+std::string publicKeys_to_string(const std::vector<PublicKey>& publicKeys)
+{
+    return ((nlohmann::json) publicKeys).dump();
+}
+
 }
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index 7d768bac1..a056c8939 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -182,4 +182,13 @@ void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);
 
 nlohmann::json dumpRegisterInputSchemeInfo();
 
+struct PublicKey
+{
+    std::string type = "ssh-ed25519";
+    std::string key;
+};
+NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(PublicKey, type, key)
+
+std::string publicKeys_to_string(const std::vector<PublicKey>&);
+
 }
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index d625fe01e..51e551879 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -143,6 +143,69 @@ struct WorkdirInfo
     bool hasHead = false;
 };
 
+std::vector<PublicKey> getPublicKeys(const Attrs & attrs) {
+    std::vector<PublicKey> publicKeys;
+    if (attrs.contains("publicKeys")) {
+        nlohmann::json publicKeysJson = nlohmann::json::parse(getStrAttr(attrs, "publicKeys"));
+        ensureType(publicKeysJson, nlohmann::json::value_t::array);
+        publicKeys = publicKeysJson.get<std::vector<PublicKey>>();
+    }
+    else {
+        publicKeys = {};
+    }
+    if (attrs.contains("publicKey"))
+        publicKeys.push_back(PublicKey{maybeGetStrAttr(attrs, "keytype").value_or("ssh-ed25519"),getStrAttr(attrs, "publicKey")});
+    return publicKeys;
+}
+
+void doCommitVerification(const Path repoDir, const Path gitDir, const std::string rev, const std::vector<PublicKey>& publicKeys) {
+    // Create ad-hoc allowedSignersFile and populate it with publicKeys
+    auto allowedSignersFile = createTempFile().second;
+    std::string allowedSigners;
+    for (const PublicKey& k : publicKeys) {
+        if (k.type != "ssh-dsa"
+            && k.type != "ssh-ecdsa"
+            && k.type != "ssh-ecdsa-sk"
+            && k.type != "ssh-ed25519"
+            && k.type != "ssh-ed25519-sk"
+            && k.type != "ssh-rsa")
+            warn("Unknow keytype: %s\n"
+                 "Please use one of\n"
+                 "- ssh-dsa\n"
+                 "- ssh-ecdsa\n"
+                 "- ssh-ecdsa-sk\n"
+                 "- ssh-ed25519\n"
+                 "- ssh-ed25519-sk\n"
+                 "- ssh-rsa", k.type);
+        allowedSigners += "* " + k.type + " " + k.key + "\n";
+    }
+    writeFile(allowedSignersFile, allowedSigners);
+
+    // Run verification command
+    auto [status, output] = runProgram(RunOptions {
+            .program = "git",
+            .args = {"-c", "gpg.ssh.allowedSignersFile=" + allowedSignersFile, "-C", repoDir,
+                     "--git-dir", gitDir, "verify-commit", rev},
+            .mergeStderrToStdout = true,
+    });
+
+    /* Evaluate result through status code and checking if public key fingerprints appear on stderr
+     * This is neccessary because the git command might also succeed due to the commit being signed by gpg keys
+     * that are present in the users key agent. */
+    std::string re = R"(Good "git" signature for \* with .* key SHA256:[)";
+    for (const PublicKey& k : publicKeys){
+        // Calculate sha256 fingerprint from public key and escape the regex symbol '+' to match the key literally
+        auto fingerprint = trim(hashString(htSHA256, base64Decode(k.key)).to_string(nix::HashFormat::Base64, false), "=");
+        auto escaped_fingerprint = std::regex_replace(fingerprint, std::regex("\\+"), "\\+" );
+        re += "(" + escaped_fingerprint + ")";
+    }
+    re += "]";
+    if (status == 0 && std::regex_search(output, std::regex(re)))
+        printTalkative("Commit signature verification on commit %s succeeded", rev);
+    else
+        throw Error("Commit signature verification on commit %s failed: \n%s", rev, output);
+}
+
 // Returns whether a git workdir is clean and has commits.
 WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir)
 {
@@ -272,9 +335,9 @@ struct GitInputScheme : InputScheme
         attrs.emplace("type", "git");
 
         for (auto & [name, value] : url.query) {
-            if (name == "rev" || name == "ref")
+            if (name == "rev" || name == "ref" || name == "keytype" || name == "publicKey" || name == "publicKeys")
                 attrs.emplace(name, value);
-            else if (name == "shallow" || name == "submodules" || name == "allRefs")
+            else if (name == "shallow" || name == "submodules" || name == "allRefs" || name == "verifyCommit")
                 attrs.emplace(name, Explicit<bool> { value == "1" });
             else
                 url2.query.emplace(name, value);
@@ -306,14 +369,26 @@ struct GitInputScheme : InputScheme
             "name",
             "dirtyRev",
             "dirtyShortRev",
+            "verifyCommit",
+            "keytype",
+            "publicKey",
+            "publicKeys",
         };
     }
 
     std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
     {
+        for (auto & [name, _] : attrs)
+            if (name == "verifyCommit"
+                || name == "keytype"
+                || name == "publicKey"
+                || name == "publicKeys")
+                experimentalFeatureSettings.require(Xp::VerifiedFetches);
+
         maybeGetBoolAttr(attrs, "shallow");
         maybeGetBoolAttr(attrs, "submodules");
         maybeGetBoolAttr(attrs, "allRefs");
+        maybeGetBoolAttr(attrs, "verifyCommit");
 
         if (auto ref = maybeGetStrAttr(attrs, "ref")) {
             if (std::regex_search(*ref, badGitRefRegex))
@@ -336,6 +411,15 @@ struct GitInputScheme : InputScheme
         if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref);
         if (maybeGetBoolAttr(input.attrs, "shallow").value_or(false))
             url.query.insert_or_assign("shallow", "1");
+        if (maybeGetBoolAttr(input.attrs, "verifyCommit").value_or(false))
+            url.query.insert_or_assign("verifyCommit", "1");
+        auto publicKeys = getPublicKeys(input.attrs);
+        if (publicKeys.size() == 1) {
+            url.query.insert_or_assign("keytype", publicKeys.at(0).type);
+            url.query.insert_or_assign("publicKey", publicKeys.at(0).key);
+        }
+        else if (publicKeys.size() > 1)
+            url.query.insert_or_assign("publicKeys", publicKeys_to_string(publicKeys));
         return url;
     }
 
@@ -425,6 +509,8 @@ struct GitInputScheme : InputScheme
         bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
         bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
         bool allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false);
+        std::vector<PublicKey> publicKeys = getPublicKeys(input.attrs);
+        bool verifyCommit = maybeGetBoolAttr(input.attrs, "verifyCommit").value_or(!publicKeys.empty());
 
         std::string cacheType = "git";
         if (shallow) cacheType += "-shallow";
@@ -445,6 +531,8 @@ struct GitInputScheme : InputScheme
                 {"type", cacheType},
                 {"name", name},
                 {"rev", input.getRev()->gitRev()},
+                {"verifyCommit", verifyCommit},
+                {"publicKeys", publicKeys_to_string(publicKeys)},
             });
         };
 
@@ -467,12 +555,15 @@ struct GitInputScheme : InputScheme
         auto [isLocal, actualUrl_] = getActualUrl(input);
         auto actualUrl = actualUrl_; // work around clang bug
 
-        /* If this is a local directory and no ref or revision is given,
+        /* If this is a local directory, no ref or revision is given and no signature verification is needed,
            allow fetching directly from a dirty workdir. */
         if (!input.getRef() && !input.getRev() && isLocal) {
             auto workdirInfo = getWorkdirInfo(input, actualUrl);
             if (!workdirInfo.clean) {
-                return fetchFromWorkdir(store, input, actualUrl, workdirInfo);
+                if (verifyCommit)
+                    throw Error("Can't fetch from a dirty workdir with commit signature verification enabled.");
+                else
+                    return fetchFromWorkdir(store, input, actualUrl, workdirInfo);
             }
         }
 
@@ -480,6 +571,8 @@ struct GitInputScheme : InputScheme
             {"type", cacheType},
             {"name", name},
             {"url", actualUrl},
+            {"verifyCommit", verifyCommit},
+            {"publicKeys", publicKeys_to_string(publicKeys)},
         });
 
         Path repoDir;
@@ -637,6 +730,9 @@ struct GitInputScheme : InputScheme
             );
         }
 
+        if (verifyCommit)
+            doCommitVerification(repoDir, gitDir, input.getRev()->gitRev(), publicKeys);
+
         if (submodules) {
             Path tmpGitDir = createTempDir();
             AutoDelete delTmpGitDir(tmpGitDir, true);
diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc
index 74af9aae0..47edca3a5 100644
--- a/src/libutil/experimental-features.cc
+++ b/src/libutil/experimental-features.cc
@@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails
     std::string_view description;
 };
 
-constexpr std::array<ExperimentalFeatureDetails, 15> xpFeatureDetails = {{
+constexpr std::array<ExperimentalFeatureDetails, 16> xpFeatureDetails = {{
     {
         .tag = Xp::CaDerivations,
         .name = "ca-derivations",
@@ -227,7 +227,14 @@ constexpr std::array<ExperimentalFeatureDetails, 15> xpFeatureDetails = {{
         .description = R"(
             Allow the use of the [impure-env](@docroot@/command-ref/conf-file.md#conf-impure-env) setting.
         )",
-    }
+    },
+    {
+        .tag = Xp::VerifiedFetches,
+        .name = "verified-fetches",
+        .description = R"(
+            Enables verification of git commit signatures through the [`fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) built-in.
+        )",
+    },
 }};
 
 static_assert(
diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh
index e02f8353e..f005cc9ee 100644
--- a/src/libutil/experimental-features.hh
+++ b/src/libutil/experimental-features.hh
@@ -32,6 +32,7 @@ enum struct ExperimentalFeature
     ParseTomlTimestamps,
     ReadOnlyLocalStore,
     ConfigurableImpureEnv,
+    VerifiedFetches,
 };
 
 /**

From 098f0615c9401414a76e66653fbf4c9dd30d55a7 Mon Sep 17 00:00:00 2001
From: BootRhetoric <110117466+BootRhetoric@users.noreply.github.com>
Date: Fri, 20 Oct 2023 21:17:14 +0200
Subject: [PATCH 056/294] fetchGit and flake: add publicKeys list input

This adds publicKeys as an optional fetcher input attribute to flakes
and builtins.fetchGit to provide a nix interface for the json-encoded
`publicKeys` attribute of the git fetcher.

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
---
 doc/manual/src/release-notes/rl-next.md |  3 +-
 src/libexpr/flake/flake.cc              | 10 ++++-
 src/libexpr/primops/fetchTree.cc        | 56 +++++++++++++++++++++++++
 src/libfetchers/git.cc                  | 14 +++----
 4 files changed, 73 insertions(+), 10 deletions(-)

diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
index 3cfb53998..8cd69f8fd 100644
--- a/doc/manual/src/release-notes/rl-next.md
+++ b/doc/manual/src/release-notes/rl-next.md
@@ -16,7 +16,6 @@
 
 - `builtins.fetchTree` is now marked as stable.
 
-
 - The interface for creating and updating lock files has been overhauled:
 
   - [`nix flake lock`](@docroot@/command-ref/new-cli/nix3-flake-lock.md) only creates lock files and adds missing inputs now.
@@ -29,3 +28,5 @@
 
   - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables.
     They are superceded by `nix flake update`.
+  
+- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches).
diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index 70ae7b584..ded132695 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -8,6 +8,7 @@
 #include "fetchers.hh"
 #include "finally.hh"
 #include "fetch-settings.hh"
+#include "value-to-json.hh"
 
 namespace nix {
 
@@ -140,8 +141,13 @@ static FlakeInput parseFlakeInput(EvalState & state,
                         attrs.emplace(state.symbols[attr.name], (long unsigned int)attr.value->integer);
                         break;
                     default:
-                        throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected",
-                            state.symbols[attr.name], showType(*attr.value));
+                        if (attr.name == state.symbols.create("publicKeys")) {
+                            experimentalFeatureSettings.require(Xp::VerifiedFetches);
+                            NixStringContext emptyContext = {};
+                            attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, emptyContext).dump());
+                        } else
+                            throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected",
+                                state.symbols[attr.name], showType(*attr.value));
                 }
                 #pragma GCC diagnostic pop
             }
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 767f559be..3717b9022 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -7,6 +7,7 @@
 #include "registry.hh"
 #include "tarball.hh"
 #include "url.hh"
+#include "value-to-json.hh"
 
 #include <ctime>
 #include <iomanip>
@@ -125,6 +126,10 @@ static void fetchTree(
                 attrs.emplace(state.symbols[attr.name], Explicit<bool>{attr.value->boolean});
             else if (attr.value->type() == nInt)
                 attrs.emplace(state.symbols[attr.name], uint64_t(attr.value->integer));
+            else if (state.symbols[attr.name] == "publicKeys") {
+                experimentalFeatureSettings.require(Xp::VerifiedFetches);
+                attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, context).dump());
+            }
             else
                 state.debugThrowLastTrace(TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected",
                     state.symbols[attr.name], showType(*attr.value)));
@@ -427,6 +432,42 @@ static RegisterPrimOp primop_fetchGit({
         With this argument being true, it's possible to load a `rev` from *any* `ref`
         (by default only `rev`s from the specified `ref` are supported).
 
+      - `verifyCommit` (default: `true` if `publicKey` or `publicKeys` are provided, otherwise `false`)
+
+        Whether to check `rev` for a signature matching `publicKey` or `publicKeys`.
+        If `verifyCommit` is enabled, then `fetchGit` cannot use a local repository with uncommitted changes.
+        Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches).
+
+      - `publicKey`
+
+        The public key against which `rev` is verified if `verifyCommit` is enabled.
+        Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches).
+
+      - `keytype` (default: `"ssh-ed25519"`)
+
+        The key type of `publicKey`.
+        Possible values:
+        - `"ssh-dsa"`
+        - `"ssh-ecdsa"`
+        - `"ssh-ecdsa-sk"`
+        - `"ssh-ed25519"`
+        - `"ssh-ed25519-sk"`
+        - `"ssh-rsa"`
+        Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches).
+
+      - `publicKeys`
+
+        The public keys against which `rev` is verified if `verifyCommit` is enabled.
+        Must be given as a list of attribute sets with the following form:
+        ```nix
+        {
+          key = "<public key>";
+          type = "<key type>"; # optional, default: "ssh-ed25519"
+        }
+        ```
+        Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches).
+
+
       Here are some examples of how to use `fetchGit`.
 
         - To fetch a private repository over SSH:
@@ -501,6 +542,21 @@ static RegisterPrimOp primop_fetchGit({
           }
           ```
 
+        - To verify the commit signature:
+
+          ```nix
+          builtins.fetchGit {
+            url = "ssh://git@github.com/nixos/nix.git";
+            verifyCommit = true;
+            publicKeys = [
+                {
+                  type = "ssh-ed25519";
+                  key = "AAAAC3NzaC1lZDI1NTE5AAAAIArPKULJOid8eS6XETwUjO48/HKBWl7FTCK0Z//fplDi";
+                }
+            ];
+          }
+          ```
+
           Nix will refetch the branch according to the [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl) setting.
 
           This behavior is disabled in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval).
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 51e551879..72fba0582 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -169,14 +169,14 @@ void doCommitVerification(const Path repoDir, const Path gitDir, const std::stri
             && k.type != "ssh-ed25519"
             && k.type != "ssh-ed25519-sk"
             && k.type != "ssh-rsa")
-            warn("Unknow keytype: %s\n"
+            warn("Unknown keytype: %s\n"
                  "Please use one of\n"
                  "- ssh-dsa\n"
-                 "- ssh-ecdsa\n"
-                 "- ssh-ecdsa-sk\n"
-                 "- ssh-ed25519\n"
-                 "- ssh-ed25519-sk\n"
-                 "- ssh-rsa", k.type);
+                 "  ssh-ecdsa\n"
+                 "  ssh-ecdsa-sk\n"
+                 "  ssh-ed25519\n"
+                 "  ssh-ed25519-sk\n"
+                 "  ssh-rsa", k.type);
         allowedSigners += "* " + k.type + " " + k.key + "\n";
     }
     writeFile(allowedSignersFile, allowedSigners);
@@ -201,7 +201,7 @@ void doCommitVerification(const Path repoDir, const Path gitDir, const std::stri
     }
     re += "]";
     if (status == 0 && std::regex_search(output, std::regex(re)))
-        printTalkative("Commit signature verification on commit %s succeeded", rev);
+        printTalkative("Signature verification on commit %s succeeded", rev);
     else
         throw Error("Commit signature verification on commit %s failed: \n%s", rev, output);
 }

From 271932782dd3d44e0e238bd3234ca1e97996cfea Mon Sep 17 00:00:00 2001
From: BootRhetoric <110117466+BootRhetoric@users.noreply.github.com>
Date: Fri, 20 Oct 2023 21:18:01 +0200
Subject: [PATCH 057/294] fetchGit and flake: add commit signature verification
 tests

This adds simple tests of the commit signature verification mechanism of
fetchGit and its flake input wrapper.
OpenSSH is added to the build dependencies since it's needed to create
a key when testing the functionality. It is neither a built- nor a
runtime dependency.
---
 flake.nix                                |  1 +
 tests/functional/fetchGitVerification.sh | 76 ++++++++++++++++++++++++
 tests/functional/local.mk                |  1 +
 3 files changed, 78 insertions(+)
 create mode 100644 tests/functional/fetchGitVerification.sh

diff --git a/flake.nix b/flake.nix
index 7cc4ed7fe..51d818423 100644
--- a/flake.nix
+++ b/flake.nix
@@ -185,6 +185,7 @@
             buildPackages.git
             buildPackages.mercurial # FIXME: remove? only needed for tests
             buildPackages.jq # Also for custom mdBook preprocessor.
+            buildPackages.openssh # only needed for tests (ssh-keygen)
           ]
           ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)];
 
diff --git a/tests/functional/fetchGitVerification.sh b/tests/functional/fetchGitVerification.sh
new file mode 100644
index 000000000..4d9209498
--- /dev/null
+++ b/tests/functional/fetchGitVerification.sh
@@ -0,0 +1,76 @@
+source common.sh
+
+requireGit
+[[ $(type -p ssh-keygen) ]] || skipTest "ssh-keygen not installed" # require ssh-keygen
+
+enableFeatures "verified-fetches"
+
+clearStore
+
+repo="$TEST_ROOT/git"
+
+# generate signing keys
+keysDir=$TEST_ROOT/.ssh
+mkdir -p "$keysDir"
+ssh-keygen -f "$keysDir/testkey1" -t ed25519 -P "" -C "test key 1"
+key1File="$keysDir/testkey1.pub"
+publicKey1=$(awk '{print $2}' "$key1File")
+ssh-keygen -f "$keysDir/testkey2" -t rsa -P "" -C "test key 2"
+key2File="$keysDir/testkey2.pub"
+publicKey2=$(awk '{print $2}' "$key2File")
+
+git init $repo
+git -C $repo config user.email "foobar@example.com"
+git -C $repo config user.name "Foobar"
+git -C $repo config gpg.format ssh
+
+echo 'hello' > $repo/text
+git -C $repo add text
+git -C $repo -c "user.signingkey=$key1File" commit -S -m 'initial commit'
+
+out=$(nix eval --impure --raw --expr "builtins.fetchGit { url = \"file://$repo\"; keytype = \"ssh-rsa\"; publicKey = \"$publicKey2\"; }" 2>&1) || status=$?
+[[ $status == 1 ]]
+[[ $out =~ 'No principal matched.' ]]
+[[ $(nix eval --impure --raw --expr "builtins.readFile (builtins.fetchGit { url = \"file://$repo\"; publicKey = \"$publicKey1\"; } + \"/text\")") = 'hello' ]]
+
+echo 'hello world' > $repo/text
+git -C $repo add text
+git -C $repo -c "user.signingkey=$key2File" commit -S -m 'second commit'
+
+[[ $(nix eval --impure --raw --expr "builtins.readFile (builtins.fetchGit { url = \"file://$repo\"; publicKeys = [{key = \"$publicKey1\";} {type = \"ssh-rsa\"; key = \"$publicKey2\";}]; } + \"/text\")") = 'hello world' ]]
+
+# Flake input test
+flakeDir="$TEST_ROOT/flake"
+mkdir -p "$flakeDir"
+cat > "$flakeDir/flake.nix" <<EOF
+{
+  inputs.test = {
+    type = "git";
+    url = "file://$repo";
+    flake = false;
+    publicKeys = [
+      { type = "ssh-rsa"; key = "$publicKey2"; }
+    ];
+  };
+
+  outputs = { test, ... }: { test = test.outPath; };
+}
+EOF
+nix build --out-link "$flakeDir/result" "$flakeDir#test"
+[[ $(cat "$flakeDir/result/text") = 'hello world' ]]
+
+cat > "$flakeDir/flake.nix" <<EOF
+{
+  inputs.test = {
+    type = "git";
+    url = "file://$repo";
+    flake = false;
+    publicKey= "$publicKey1";
+  };
+
+  outputs = { test, ... }: { test = test.outPath; };
+}
+EOF
+out=$(nix build "$flakeDir#test" 2>&1) || status=$?
+[[ $status == 1 ]]
+[[ $out =~ 'No principal matched.' ]]
\ No newline at end of file
diff --git a/tests/functional/local.mk b/tests/functional/local.mk
index 3679349f8..fe0d0c4ed 100644
--- a/tests/functional/local.mk
+++ b/tests/functional/local.mk
@@ -55,6 +55,7 @@ nix_tests = \
   secure-drv-outputs.sh \
   restricted.sh \
   fetchGitSubmodules.sh \
+  fetchGitVerification.sh \
   flakes/search-root.sh \
   readfile-context.sh \
   nix-channel.sh \

From 9b880e3e29c7a485b0e21495f2d089c5151589cc Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 2 Nov 2023 19:39:09 -0400
Subject: [PATCH 058/294] Factor out `MemorySourceAccessor`, implement missing
 features

The new `MemorySourceAccessor` rather than being a slightly lossy flat
map is a complete in-memory model of file system objects.

Co-authored-by: Eelco Dolstra <edolstra@gmail.com>
---
 src/libfetchers/input-accessor.hh        |   2 +-
 src/libfetchers/memory-input-accessor.cc |  44 ++------
 src/libutil/memory-source-accessor.cc    | 124 +++++++++++++++++++++++
 src/libutil/memory-source-accessor.hh    |  74 ++++++++++++++
 4 files changed, 205 insertions(+), 39 deletions(-)
 create mode 100644 src/libutil/memory-source-accessor.cc
 create mode 100644 src/libutil/memory-source-accessor.hh

diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh
index 5dc05a363..68fdf07a7 100644
--- a/src/libfetchers/input-accessor.hh
+++ b/src/libfetchers/input-accessor.hh
@@ -14,7 +14,7 @@ struct SourcePath;
 class StorePath;
 class Store;
 
-struct InputAccessor : SourceAccessor, std::enable_shared_from_this<InputAccessor>
+struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this<InputAccessor>
 {
     /**
      * Return the maximum last-modified time of the files in this
diff --git a/src/libfetchers/memory-input-accessor.cc b/src/libfetchers/memory-input-accessor.cc
index 6468ece41..057f3e37f 100644
--- a/src/libfetchers/memory-input-accessor.cc
+++ b/src/libfetchers/memory-input-accessor.cc
@@ -1,48 +1,16 @@
 #include "memory-input-accessor.hh"
+#include "memory-source-accessor.hh"
 
 namespace nix {
 
-struct MemoryInputAccessorImpl : MemoryInputAccessor
+struct MemoryInputAccessorImpl : MemoryInputAccessor, MemorySourceAccessor
 {
-    std::map<CanonPath, std::string> files;
-
-    std::string readFile(const CanonPath & path) override
-    {
-        auto i = files.find(path);
-        if (i == files.end())
-            throw Error("file '%s' does not exist", path);
-        return i->second;
-    }
-
-    bool pathExists(const CanonPath & path) override
-    {
-        auto i = files.find(path);
-        return i != files.end();
-    }
-
-    std::optional<Stat> maybeLstat(const CanonPath & path) override
-    {
-        auto i = files.find(path);
-        if (i != files.end())
-            return Stat { .type = tRegular, .isExecutable = false };
-        return std::nullopt;
-    }
-
-    DirEntries readDirectory(const CanonPath & path) override
-    {
-        return {};
-    }
-
-    std::string readLink(const CanonPath & path) override
-    {
-        throw UnimplementedError("MemoryInputAccessor::readLink");
-    }
-
     SourcePath addFile(CanonPath path, std::string && contents) override
     {
-        files.emplace(path, std::move(contents));
-
-        return {ref(shared_from_this()), std::move(path)};
+        return {
+            ref(shared_from_this()),
+            MemorySourceAccessor::addFile(path, std::move(contents))
+        };
     }
 };
 
diff --git a/src/libutil/memory-source-accessor.cc b/src/libutil/memory-source-accessor.cc
new file mode 100644
index 000000000..f34f6c091
--- /dev/null
+++ b/src/libutil/memory-source-accessor.cc
@@ -0,0 +1,124 @@
+#include "memory-source-accessor.hh"
+
+namespace nix {
+
+MemorySourceAccessor::File *
+MemorySourceAccessor::open(const CanonPath & path, std::optional<File> create)
+{
+    File * cur = &root;
+
+    bool newF = false;
+
+    for (std::string_view name : path)
+    {
+        auto * curDirP = std::get_if<File::Directory>(&cur->raw);
+        if (!curDirP)
+            return nullptr;
+        auto & curDir = *curDirP;
+
+        auto i = curDir.contents.find(name);
+        if (i == curDir.contents.end()) {
+            if (!create)
+                return nullptr;
+            else {
+                newF = true;
+                i = curDir.contents.insert(i, {
+                    std::string { name },
+                    File::Directory {},
+                });
+            }
+        }
+        cur = &i->second;
+    }
+
+    if (newF && create) *cur = std::move(*create);
+
+    return cur;
+}
+
+std::string MemorySourceAccessor::readFile(const CanonPath & path)
+{
+    auto * f = open(path, std::nullopt);
+    if (!f)
+        throw Error("file '%s' does not exist", path);
+    if (auto * r = std::get_if<File::Regular>(&f->raw))
+        return r->contents;
+    else
+        throw Error("file '%s' is not a regular file", path);
+}
+
+bool MemorySourceAccessor::pathExists(const CanonPath & path)
+{
+    return open(path, std::nullopt);
+}
+
+MemorySourceAccessor::Stat MemorySourceAccessor::File::lstat() const
+{
+    return std::visit(overloaded {
+        [](const Regular & r) {
+            return Stat {
+                .type = tRegular,
+                .fileSize = r.contents.size(),
+                .isExecutable = r.executable,
+            };
+        },
+        [](const Directory &) {
+            return Stat {
+                .type = tDirectory,
+            };
+        },
+        [](const Symlink &) {
+            return Stat {
+                .type = tSymlink,
+            };
+        },
+    }, this->raw);
+}
+
+std::optional<MemorySourceAccessor::Stat>
+MemorySourceAccessor::maybeLstat(const CanonPath & path)
+{
+    const auto * f = open(path, std::nullopt);
+    return f ? std::optional { f->lstat() } : std::nullopt;
+}
+
+MemorySourceAccessor::DirEntries MemorySourceAccessor::readDirectory(const CanonPath & path)
+{
+    auto * f = open(path, std::nullopt);
+    if (!f)
+        throw Error("file '%s' does not exist", path);
+    if (auto * d = std::get_if<File::Directory>(&f->raw)) {
+        DirEntries res;
+        for (auto & [name, file] : d->contents)
+            res.insert_or_assign(name, file.lstat().type);
+        return res;
+    } else
+        throw Error("file '%s' is not a directory", path);
+    return {};
+}
+
+std::string MemorySourceAccessor::readLink(const CanonPath & path)
+{
+    auto * f = open(path, std::nullopt);
+    if (!f)
+        throw Error("file '%s' does not exist", path);
+    if (auto * s = std::get_if<File::Symlink>(&f->raw))
+        return s->target;
+    else
+        throw Error("file '%s' is not a symbolic link", path);
+}
+
+CanonPath MemorySourceAccessor::addFile(CanonPath path, std::string && contents)
+{
+    auto * f = open(path, File { File::Regular {} });
+    if (!f)
+        throw Error("file '%s' cannot be made because some parent file is not a directory", path);
+    if (auto * r = std::get_if<File::Regular>(&f->raw))
+        r->contents = std::move(contents);
+    else
+        throw Error("file '%s' is not a regular file", path);
+
+    return path;
+}
+
+}
diff --git a/src/libutil/memory-source-accessor.hh b/src/libutil/memory-source-accessor.hh
new file mode 100644
index 000000000..014fa8098
--- /dev/null
+++ b/src/libutil/memory-source-accessor.hh
@@ -0,0 +1,74 @@
+#include "source-accessor.hh"
+#include "variant-wrapper.hh"
+
+namespace nix {
+
+/**
+ * An source accessor for an in-memory file system.
+ */
+struct MemorySourceAccessor : virtual SourceAccessor
+{
+    /**
+     * In addition to being part of the implementation of
+     * `MemorySourceAccessor`, this has a side benefit of nicely
+     * defining what a "file system object" is in Nix.
+     */
+    struct File {
+        struct Regular {
+            bool executable = false;
+            std::string contents;
+
+            GENERATE_CMP(Regular, me->executable, me->contents);
+        };
+
+        struct Directory {
+            using Name = std::string;
+
+            std::map<Name, File, std::less<>> contents;
+
+            GENERATE_CMP(Directory, me->contents);
+        };
+
+        struct Symlink {
+            std::string target;
+
+            GENERATE_CMP(Symlink, me->target);
+        };
+
+        using Raw = std::variant<Regular, Directory, Symlink>;
+        Raw raw;
+
+        MAKE_WRAPPER_CONSTRUCTOR(File);
+
+        GENERATE_CMP(File, me->raw);
+
+        Stat lstat() const;
+    };
+
+    File root { File::Directory {} };
+
+    GENERATE_CMP(MemorySourceAccessor, me->root);
+
+    std::string readFile(const CanonPath & path) override;
+    bool pathExists(const CanonPath & path) override;
+    std::optional<Stat> maybeLstat(const CanonPath & path) override;
+    DirEntries readDirectory(const CanonPath & path) override;
+    std::string readLink(const CanonPath & path) override;
+
+    /**
+     * @param create If present, create this file and any parent directories
+     * that are needed.
+     *
+     * Return null if
+     *
+     * - `create = false`: File does not exist.
+     *
+     * - `create = true`: some parent file was not a dir, so couldn't
+     *   look/create inside.
+     */
+    File * open(const CanonPath & path, std::optional<File> create);
+
+    CanonPath addFile(CanonPath path, std::string && contents);
+};
+
+}

From e97ac09abeab44fa3d10eb539f0b3d51f8575798 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 6 Jan 2023 18:06:03 -0500
Subject: [PATCH 059/294] Factor out `StoreDirConfig`

More progress on #5729.
---
 src/libstore/path.cc             |  14 ++--
 src/libstore/store-api.cc        |  24 +++---
 src/libstore/store-api.hh        | 103 +------------------------
 src/libstore/store-dir-config.hh | 126 +++++++++++++++++++++++++++++++
 4 files changed, 148 insertions(+), 119 deletions(-)
 create mode 100644 src/libstore/store-dir-config.hh

diff --git a/src/libstore/path.cc b/src/libstore/path.cc
index ec3e53232..69f6d7356 100644
--- a/src/libstore/path.cc
+++ b/src/libstore/path.cc
@@ -1,4 +1,4 @@
-#include "store-api.hh"
+#include "store-dir-config.hh"
 
 #include <sodium.h>
 
@@ -54,7 +54,7 @@ StorePath StorePath::random(std::string_view name)
     return StorePath(hash, name);
 }
 
-StorePath Store::parseStorePath(std::string_view path) const
+StorePath StoreDirConfig::parseStorePath(std::string_view path) const
 {
     auto p = canonPath(std::string(path));
     if (dirOf(p) != storeDir)
@@ -62,7 +62,7 @@ StorePath Store::parseStorePath(std::string_view path) const
     return StorePath(baseNameOf(p));
 }
 
-std::optional<StorePath> Store::maybeParseStorePath(std::string_view path) const
+std::optional<StorePath> StoreDirConfig::maybeParseStorePath(std::string_view path) const
 {
     try {
         return parseStorePath(path);
@@ -71,24 +71,24 @@ std::optional<StorePath> Store::maybeParseStorePath(std::string_view path) const
     }
 }
 
-bool Store::isStorePath(std::string_view path) const
+bool StoreDirConfig::isStorePath(std::string_view path) const
 {
     return (bool) maybeParseStorePath(path);
 }
 
-StorePathSet Store::parseStorePathSet(const PathSet & paths) const
+StorePathSet StoreDirConfig::parseStorePathSet(const PathSet & paths) const
 {
     StorePathSet res;
     for (auto & i : paths) res.insert(parseStorePath(i));
     return res;
 }
 
-std::string Store::printStorePath(const StorePath & path) const
+std::string StoreDirConfig::printStorePath(const StorePath & path) const
 {
     return (storeDir + "/").append(path.to_string());
 }
 
-PathSet Store::printStorePathSet(const StorePathSet & paths) const
+PathSet StoreDirConfig::printStorePathSet(const StorePathSet & paths) const
 {
     PathSet res;
     for (auto & i : paths) res.insert(printStorePath(i));
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 646b0ec7d..a681bb6cf 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -23,13 +23,13 @@ using json = nlohmann::json;
 namespace nix {
 
 
-bool Store::isInStore(PathView path) const
+bool StoreDirConfig::isInStore(PathView path) const
 {
     return isInDir(path, storeDir);
 }
 
 
-std::pair<StorePath, Path> Store::toStorePath(PathView path) const
+std::pair<StorePath, Path> StoreDirConfig::toStorePath(PathView path) const
 {
     if (!isInStore(path))
         throw Error("path '%1%' is not in the Nix store", path);
@@ -143,7 +143,7 @@ StorePath Store::followLinksToStorePath(std::string_view path) const
 */
 
 
-StorePath Store::makeStorePath(std::string_view type,
+StorePath StoreDirConfig::makeStorePath(std::string_view type,
     std::string_view hash, std::string_view name) const
 {
     /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
@@ -154,14 +154,14 @@ StorePath Store::makeStorePath(std::string_view type,
 }
 
 
-StorePath Store::makeStorePath(std::string_view type,
+StorePath StoreDirConfig::makeStorePath(std::string_view type,
     const Hash & hash, std::string_view name) const
 {
     return makeStorePath(type, hash.to_string(HashFormat::Base16, true), name);
 }
 
 
-StorePath Store::makeOutputPath(std::string_view id,
+StorePath StoreDirConfig::makeOutputPath(std::string_view id,
     const Hash & hash, std::string_view name) const
 {
     return makeStorePath("output:" + std::string { id }, hash, outputPathName(name, id));
@@ -172,7 +172,7 @@ StorePath Store::makeOutputPath(std::string_view id,
    hacky, but we can't put them in, say, <s2> (per the grammar above)
    since that would be ambiguous. */
 static std::string makeType(
-    const Store & store,
+    const StoreDirConfig & store,
     std::string && type,
     const StoreReferences & references)
 {
@@ -185,7 +185,7 @@ static std::string makeType(
 }
 
 
-StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const
+StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const
 {
     if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) {
         return makeStorePath(makeType(*this, "source", info.references), info.hash, name);
@@ -201,7 +201,7 @@ StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInf
 }
 
 
-StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const
+StorePath StoreDirConfig::makeTextPath(std::string_view name, const TextInfo & info) const
 {
     assert(info.hash.type == htSHA256);
     return makeStorePath(
@@ -214,7 +214,7 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons
 }
 
 
-StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const
+StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const
 {
     // New template
     return std::visit(overloaded {
@@ -228,7 +228,7 @@ StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentA
 }
 
 
-std::pair<StorePath, Hash> Store::computeStorePathFromDump(
+std::pair<StorePath, Hash> StoreDirConfig::computeStorePathFromDump(
     Source & dump,
     std::string_view name,
     FileIngestionMethod method,
@@ -247,7 +247,7 @@ std::pair<StorePath, Hash> Store::computeStorePathFromDump(
 }
 
 
-StorePath Store::computeStorePathForText(
+StorePath StoreDirConfig::computeStorePathForText(
     std::string_view name,
     std::string_view s,
     const StorePathSet & references) const
@@ -1315,7 +1315,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istre
 }
 
 
-std::string Store::showPaths(const StorePathSet & paths)
+std::string StoreDirConfig::showPaths(const StorePathSet & paths)
 {
     std::string s;
     for (auto & i : paths) {
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 6aa317e3d..bee5ec16c 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -14,6 +14,7 @@
 #include "config.hh"
 #include "path-info.hh"
 #include "repair-flag.hh"
+#include "store-dir-config.hh"
 
 #include <nlohmann/json_fwd.hpp>
 #include <atomic>
@@ -64,7 +65,6 @@ MakeError(InvalidPath, Error);
 MakeError(Unsupported, Error);
 MakeError(SubstituteGone, Error);
 MakeError(SubstituterDisabled, Error);
-MakeError(BadStorePath, Error);
 
 MakeError(InvalidStoreURI, Error);
 
@@ -97,11 +97,11 @@ struct KeyedBuildResult;
 
 typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;
 
-struct StoreConfig : public Config
+struct StoreConfig : public StoreDirConfig
 {
     typedef std::map<std::string, std::string> Params;
 
-    using Config::Config;
+    using StoreDirConfig::StoreDirConfig;
 
     StoreConfig() = delete;
 
@@ -131,15 +131,6 @@ struct StoreConfig : public Config
         return std::nullopt;
     }
 
-    const PathSetting storeDir_{this, settings.nixStore,
-        "store",
-        R"(
-          Logical location of the Nix store, usually
-          `/nix/store`. Note that you can only copy store paths
-          between stores if they have the same `store` setting.
-        )"};
-    const Path storeDir = storeDir_;
-
     const Setting<int> pathInfoCacheSize{this, 65536, "path-info-cache-size",
         "Size of the in-memory store path metadata cache."};
 
@@ -224,45 +215,6 @@ public:
 
     virtual std::string getUri() = 0;
 
-    StorePath parseStorePath(std::string_view path) const;
-
-    std::optional<StorePath> maybeParseStorePath(std::string_view path) const;
-
-    std::string printStorePath(const StorePath & path) const;
-
-    /**
-     * Deprecated
-     *
-     * \todo remove
-     */
-    StorePathSet parseStorePathSet(const PathSet & paths) const;
-
-    PathSet printStorePathSet(const StorePathSet & path) const;
-
-    /**
-     * Display a set of paths in human-readable form (i.e., between quotes
-     * and separated by commas).
-     */
-    std::string showPaths(const StorePathSet & paths);
-
-    /**
-     * @return true if ‘path’ is in the Nix store (but not the Nix
-     * store itself).
-     */
-    bool isInStore(PathView path) const;
-
-    /**
-     * @return true if ‘path’ is a store path, i.e. a direct child of the
-     * Nix store.
-     */
-    bool isStorePath(std::string_view path) const;
-
-    /**
-     * Split a path like /nix/store/<hash>-<name>/<bla> into
-     * /nix/store/<hash>-<name> and /<bla>.
-     */
-    std::pair<StorePath, Path> toStorePath(PathView path) const;
-
     /**
      * Follow symlinks until we end up with a path in the Nix store.
      */
@@ -274,55 +226,6 @@ public:
      */
     StorePath followLinksToStorePath(std::string_view path) const;
 
-    /**
-     * Constructs a unique store path name.
-     */
-    StorePath makeStorePath(std::string_view type,
-        std::string_view hash, std::string_view name) const;
-    StorePath makeStorePath(std::string_view type,
-        const Hash & hash, std::string_view name) const;
-
-    StorePath makeOutputPath(std::string_view id,
-        const Hash & hash, std::string_view name) const;
-
-    StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const;
-
-    StorePath makeTextPath(std::string_view name, const TextInfo & info) const;
-
-    StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const;
-
-    /**
-     * Read-only variant of addToStoreFromDump(). It returns the store
-     * path to which a NAR or flat file would be written.
-     */
-    std::pair<StorePath, Hash> computeStorePathFromDump(
-        Source & dump,
-        std::string_view name,
-        FileIngestionMethod method = FileIngestionMethod::Recursive,
-        HashType hashAlgo = htSHA256,
-        const StorePathSet & references = {}) const;
-
-    /**
-     * Preparatory part of addTextToStore().
-     *
-     * !!! Computation of the path should take the references given to
-     * addTextToStore() into account, otherwise we have a (relatively
-     * minor) security hole: a caller can register a source file with
-     * bogus references.  If there are too many references, the path may
-     * not be garbage collected when it has to be (not really a problem,
-     * the caller could create a root anyway), or it may be garbage
-     * collected when it shouldn't be (more serious).
-     *
-     * Hashing the references would solve this (bogus references would
-     * simply yield a different store path, so other users wouldn't be
-     * affected), but it has some backwards compatibility issues (the
-     * hashing scheme changes), so I'm not doing that for now.
-     */
-    StorePath computeStorePathForText(
-        std::string_view name,
-        std::string_view s,
-        const StorePathSet & references) const;
-
     /**
      * Check whether a path is valid.
      */
diff --git a/src/libstore/store-dir-config.hh b/src/libstore/store-dir-config.hh
new file mode 100644
index 000000000..53843d663
--- /dev/null
+++ b/src/libstore/store-dir-config.hh
@@ -0,0 +1,126 @@
+#pragma once
+
+#include "path.hh"
+#include "hash.hh"
+#include "content-address.hh"
+#include "globals.hh"
+#include "config.hh"
+
+#include <map>
+#include <string>
+#include <variant>
+
+
+namespace nix {
+
+MakeError(BadStorePath, Error);
+
+struct StoreDirConfig : public Config
+{
+    using Config::Config;
+
+    StoreDirConfig() = delete;
+
+    virtual ~StoreDirConfig() = default;
+
+    const PathSetting storeDir_{this, settings.nixStore,
+        "store",
+        R"(
+          Logical location of the Nix store, usually
+          `/nix/store`. Note that you can only copy store paths
+          between stores if they have the same `store` setting.
+        )"};
+    const Path storeDir = storeDir_;
+
+    // pure methods
+
+    StorePath parseStorePath(std::string_view path) const;
+
+    std::optional<StorePath> maybeParseStorePath(std::string_view path) const;
+
+    std::string printStorePath(const StorePath & path) const;
+
+    /**
+     * Deprecated
+     *
+     * \todo remove
+     */
+    StorePathSet parseStorePathSet(const PathSet & paths) const;
+
+    PathSet printStorePathSet(const StorePathSet & path) const;
+
+    /**
+     * Display a set of paths in human-readable form (i.e., between quotes
+     * and separated by commas).
+     */
+    std::string showPaths(const StorePathSet & paths);
+
+    /**
+     * @return true if ‘path’ is in the Nix store (but not the Nix
+     * store itself).
+     */
+    bool isInStore(PathView path) const;
+
+    /**
+     * @return true if ‘path’ is a store path, i.e. a direct child of the
+     * Nix store.
+     */
+    bool isStorePath(std::string_view path) const;
+
+    /**
+     * Split a path like /nix/store/<hash>-<name>/<bla> into
+     * /nix/store/<hash>-<name> and /<bla>.
+     */
+    std::pair<StorePath, Path> toStorePath(PathView path) const;
+
+    /**
+     * Constructs a unique store path name.
+     */
+    StorePath makeStorePath(std::string_view type,
+        std::string_view hash, std::string_view name) const;
+    StorePath makeStorePath(std::string_view type,
+        const Hash & hash, std::string_view name) const;
+
+    StorePath makeOutputPath(std::string_view id,
+        const Hash & hash, std::string_view name) const;
+
+    StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const;
+
+    StorePath makeTextPath(std::string_view name, const TextInfo & info) const;
+
+    StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const;
+
+    /**
+     * Read-only variant of addToStoreFromDump(). It returns the store
+     * path to which a NAR or flat file would be written.
+     */
+    std::pair<StorePath, Hash> computeStorePathFromDump(
+        Source & dump,
+        std::string_view name,
+        FileIngestionMethod method = FileIngestionMethod::Recursive,
+        HashType hashAlgo = htSHA256,
+        const StorePathSet & references = {}) const;
+
+    /**
+     * Preparatory part of addTextToStore().
+     *
+     * !!! Computation of the path should take the references given to
+     * addTextToStore() into account, otherwise we have a (relatively
+     * minor) security hole: a caller can register a source file with
+     * bogus references.  If there are too many references, the path may
+     * not be garbage collected when it has to be (not really a problem,
+     * the caller could create a root anyway), or it may be garbage
+     * collected when it shouldn't be (more serious).
+     *
+     * Hashing the references would solve this (bogus references would
+     * simply yield a different store path, so other users wouldn't be
+     * affected), but it has some backwards compatibility issues (the
+     * hashing scheme changes), so I'm not doing that for now.
+     */
+    StorePath computeStorePathForText(
+        std::string_view name,
+        std::string_view s,
+        const StorePathSet & references) const;
+};
+
+}

From dde1d863388617b3a63db808c125f274c86a3222 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 18 Mar 2022 15:35:45 +0000
Subject: [PATCH 060/294] Restrict some code to `StoreDirConfig`

 - part of eval cache
 - part of derivations
 - derived path
 - store path with outputs
 - serializers
---
 perl/lib/Nix/Store.xs                         |  1 +
 src/libcmd/built-path.cc                      |  8 ++--
 src/libcmd/built-path.hh                      | 18 ++++-----
 src/libexpr/eval-cache.cc                     |  6 +--
 src/libexpr/primops/fetchClosure.cc           |  1 +
 src/libexpr/value.hh                          |  1 -
 src/libstore/builtins/buildenv.cc             |  1 +
 src/libstore/builtins/buildenv.hh             |  1 -
 src/libstore/common-protocol-impl.hh          |  4 +-
 src/libstore/common-protocol.cc               | 28 ++++++-------
 src/libstore/common-protocol.hh               |  8 ++--
 src/libstore/derivations.cc                   | 32 +++++++--------
 src/libstore/derivations.hh                   | 26 +++++++------
 src/libstore/derived-path.cc                  | 39 ++++++++++---------
 src/libstore/derived-path.hh                  | 37 ++++++++++--------
 .../length-prefixed-protocol-helper.hh        | 22 +++++------
 src/libstore/local-store.cc                   |  1 +
 src/libstore/misc.cc                          |  1 +
 src/libstore/path-with-outputs.cc             |  4 +-
 src/libstore/path-with-outputs.hh             | 10 +++--
 src/libstore/serve-protocol-impl.hh           |  8 ++--
 src/libstore/serve-protocol.cc                |  4 +-
 src/libstore/serve-protocol.hh                | 12 +++---
 src/libstore/store-api.cc                     |  2 +
 src/libstore/store-api.hh                     | 11 ++++--
 src/libstore/worker-protocol-impl.hh          |  8 ++--
 src/libstore/worker-protocol.cc               | 24 ++++++------
 src/libstore/worker-protocol.hh               | 12 +++---
 src/nix-build/nix-build.cc                    |  1 +
 src/nix-copy-closure/nix-copy-closure.cc      |  1 +
 30 files changed, 175 insertions(+), 157 deletions(-)

diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index 08f812b31..210d50b6e 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -9,6 +9,7 @@
 #undef do_close
 
 #include "derivations.hh"
+#include "realisation.hh"
 #include "globals.hh"
 #include "store-api.hh"
 #include "util.hh"
diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc
index 9a2dce806..8e2efc7c3 100644
--- a/src/libcmd/built-path.cc
+++ b/src/libcmd/built-path.cc
@@ -80,7 +80,7 @@ SingleDerivedPath SingleBuiltPath::discardOutputPath() const
     );
 }
 
-nlohmann::json BuiltPath::Built::toJSON(const Store & store) const
+nlohmann::json BuiltPath::Built::toJSON(const StoreDirConfig & store) const
 {
     nlohmann::json res;
     res["drvPath"] = drvPath->toJSON(store);
@@ -90,7 +90,7 @@ nlohmann::json BuiltPath::Built::toJSON(const Store & store) const
     return res;
 }
 
-nlohmann::json SingleBuiltPath::Built::toJSON(const Store & store) const
+nlohmann::json SingleBuiltPath::Built::toJSON(const StoreDirConfig & store) const
 {
     nlohmann::json res;
     res["drvPath"] = drvPath->toJSON(store);
@@ -100,14 +100,14 @@ nlohmann::json SingleBuiltPath::Built::toJSON(const Store & store) const
     return res;
 }
 
-nlohmann::json SingleBuiltPath::toJSON(const Store & store) const
+nlohmann::json SingleBuiltPath::toJSON(const StoreDirConfig & store) const
 {
     return std::visit([&](const auto & buildable) {
         return buildable.toJSON(store);
     }, raw());
 }
 
-nlohmann::json BuiltPath::toJSON(const Store & store) const
+nlohmann::json BuiltPath::toJSON(const StoreDirConfig & store) const
 {
     return std::visit([&](const auto & buildable) {
         return buildable.toJSON(store);
diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh
index e677bc810..51918f96c 100644
--- a/src/libcmd/built-path.hh
+++ b/src/libcmd/built-path.hh
@@ -11,9 +11,9 @@ struct SingleBuiltPathBuilt {
 
     SingleDerivedPathBuilt discardOutputPath() const;
 
-    std::string to_string(const Store & store) const;
-    static SingleBuiltPathBuilt parse(const Store & store, std::string_view, std::string_view);
-    nlohmann::json toJSON(const Store & store) const;
+    std::string to_string(const StoreDirConfig & store) const;
+    static SingleBuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view);
+    nlohmann::json toJSON(const StoreDirConfig & store) const;
 
     DECLARE_CMP(SingleBuiltPathBuilt);
 };
@@ -38,8 +38,8 @@ struct SingleBuiltPath : _SingleBuiltPathRaw {
 
     SingleDerivedPath discardOutputPath() const;
 
-    static SingleBuiltPath parse(const Store & store, std::string_view);
-    nlohmann::json toJSON(const Store & store) const;
+    static SingleBuiltPath parse(const StoreDirConfig & store, std::string_view);
+    nlohmann::json toJSON(const StoreDirConfig & store) const;
 };
 
 static inline ref<SingleBuiltPath> staticDrv(StorePath drvPath)
@@ -56,9 +56,9 @@ struct BuiltPathBuilt {
     ref<SingleBuiltPath> drvPath;
     std::map<std::string, StorePath> outputs;
 
-    std::string to_string(const Store & store) const;
-    static BuiltPathBuilt parse(const Store & store, std::string_view, std::string_view);
-    nlohmann::json toJSON(const Store & store) const;
+    std::string to_string(const StoreDirConfig & store) const;
+    static BuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view);
+    nlohmann::json toJSON(const StoreDirConfig & store) const;
 
     DECLARE_CMP(BuiltPathBuilt);
 };
@@ -86,7 +86,7 @@ struct BuiltPath : _BuiltPathRaw {
     StorePathSet outPaths() const;
     RealisedPath::Set toRealisedPaths(Store & store) const;
 
-    nlohmann::json toJSON(const Store & store) const;
+    nlohmann::json toJSON(const StoreDirConfig & store) const;
 };
 
 typedef std::vector<BuiltPath> BuiltPaths;
diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc
index 10fc799a9..824f94ba1 100644
--- a/src/libexpr/eval-cache.cc
+++ b/src/libexpr/eval-cache.cc
@@ -21,7 +21,7 @@ struct AttrDb
 {
     std::atomic_bool failed{false};
 
-    const Store & cfg;
+    const StoreDirConfig & cfg;
 
     struct State
     {
@@ -38,7 +38,7 @@ struct AttrDb
     SymbolTable & symbols;
 
     AttrDb(
-        const Store & cfg,
+        const StoreDirConfig & cfg,
         const Hash & fingerprint,
         SymbolTable & symbols)
         : cfg(cfg)
@@ -322,7 +322,7 @@ struct AttrDb
 };
 
 static std::shared_ptr<AttrDb> makeAttrDb(
-    const Store & cfg,
+    const StoreDirConfig & cfg,
     const Hash & fingerprint,
     SymbolTable & symbols)
 {
diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc
index b86ef6b93..27147a5d1 100644
--- a/src/libexpr/primops/fetchClosure.cc
+++ b/src/libexpr/primops/fetchClosure.cc
@@ -1,5 +1,6 @@
 #include "primops.hh"
 #include "store-api.hh"
+#include "realisation.hh"
 #include "make-content-addressed.hh"
 #include "url.hh"
 
diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 622e613ea..20f268a3e 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -66,7 +66,6 @@ class Symbol;
 class PosIdx;
 struct Pos;
 class StorePath;
-class Store;
 class EvalState;
 class XMLWriter;
 
diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc
index c8911d153..9283251ac 100644
--- a/src/libstore/builtins/buildenv.cc
+++ b/src/libstore/builtins/buildenv.cc
@@ -1,4 +1,5 @@
 #include "buildenv.hh"
+#include "derivations.hh"
 
 #include <sys/stat.h>
 #include <sys/types.h>
diff --git a/src/libstore/builtins/buildenv.hh b/src/libstore/builtins/buildenv.hh
index 0923c2adb..8bebd390d 100644
--- a/src/libstore/builtins/buildenv.hh
+++ b/src/libstore/builtins/buildenv.hh
@@ -1,7 +1,6 @@
 #pragma once
 ///@file
 
-#include "derivations.hh"
 #include "store-api.hh"
 
 namespace nix {
diff --git a/src/libstore/common-protocol-impl.hh b/src/libstore/common-protocol-impl.hh
index 079c182b8..360882c02 100644
--- a/src/libstore/common-protocol-impl.hh
+++ b/src/libstore/common-protocol-impl.hh
@@ -16,11 +16,11 @@ namespace nix {
 /* protocol-agnostic templates */
 
 #define COMMON_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \
-    TEMPLATE T CommonProto::Serialise< T >::read(const Store & store, CommonProto::ReadConn conn) \
+    TEMPLATE T CommonProto::Serialise< T >::read(const StoreDirConfig & store, CommonProto::ReadConn conn) \
     { \
         return LengthPrefixedProtoHelper<CommonProto, T >::read(store, conn); \
     } \
-    TEMPLATE void CommonProto::Serialise< T >::write(const Store & store, CommonProto::WriteConn conn, const T & t) \
+    TEMPLATE void CommonProto::Serialise< T >::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const T & t) \
     { \
         LengthPrefixedProtoHelper<CommonProto, T >::write(store, conn, t); \
     }
diff --git a/src/libstore/common-protocol.cc b/src/libstore/common-protocol.cc
index f906814bc..c14a6cfcd 100644
--- a/src/libstore/common-protocol.cc
+++ b/src/libstore/common-protocol.cc
@@ -14,40 +14,40 @@ namespace nix {
 
 /* protocol-agnostic definitions */
 
-std::string CommonProto::Serialise<std::string>::read(const Store & store, CommonProto::ReadConn conn)
+std::string CommonProto::Serialise<std::string>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
 {
     return readString(conn.from);
 }
 
-void CommonProto::Serialise<std::string>::write(const Store & store, CommonProto::WriteConn conn, const std::string & str)
+void CommonProto::Serialise<std::string>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const std::string & str)
 {
     conn.to << str;
 }
 
 
-StorePath CommonProto::Serialise<StorePath>::read(const Store & store, CommonProto::ReadConn conn)
+StorePath CommonProto::Serialise<StorePath>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
 {
     return store.parseStorePath(readString(conn.from));
 }
 
-void CommonProto::Serialise<StorePath>::write(const Store & store, CommonProto::WriteConn conn, const StorePath & storePath)
+void CommonProto::Serialise<StorePath>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const StorePath & storePath)
 {
     conn.to << store.printStorePath(storePath);
 }
 
 
-ContentAddress CommonProto::Serialise<ContentAddress>::read(const Store & store, CommonProto::ReadConn conn)
+ContentAddress CommonProto::Serialise<ContentAddress>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
 {
     return ContentAddress::parse(readString(conn.from));
 }
 
-void CommonProto::Serialise<ContentAddress>::write(const Store & store, CommonProto::WriteConn conn, const ContentAddress & ca)
+void CommonProto::Serialise<ContentAddress>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const ContentAddress & ca)
 {
     conn.to << renderContentAddress(ca);
 }
 
 
-Realisation CommonProto::Serialise<Realisation>::read(const Store & store, CommonProto::ReadConn conn)
+Realisation CommonProto::Serialise<Realisation>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
 {
     std::string rawInput = readString(conn.from);
     return Realisation::fromJSON(
@@ -56,41 +56,41 @@ Realisation CommonProto::Serialise<Realisation>::read(const Store & store, Commo
     );
 }
 
-void CommonProto::Serialise<Realisation>::write(const Store & store, CommonProto::WriteConn conn, const Realisation & realisation)
+void CommonProto::Serialise<Realisation>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const Realisation & realisation)
 {
     conn.to << realisation.toJSON().dump();
 }
 
 
-DrvOutput CommonProto::Serialise<DrvOutput>::read(const Store & store, CommonProto::ReadConn conn)
+DrvOutput CommonProto::Serialise<DrvOutput>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
 {
     return DrvOutput::parse(readString(conn.from));
 }
 
-void CommonProto::Serialise<DrvOutput>::write(const Store & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput)
+void CommonProto::Serialise<DrvOutput>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput)
 {
     conn.to << drvOutput.to_string();
 }
 
 
-std::optional<StorePath> CommonProto::Serialise<std::optional<StorePath>>::read(const Store & store, CommonProto::ReadConn conn)
+std::optional<StorePath> CommonProto::Serialise<std::optional<StorePath>>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
 {
     auto s = readString(conn.from);
     return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
 }
 
-void CommonProto::Serialise<std::optional<StorePath>>::write(const Store & store, CommonProto::WriteConn conn, const std::optional<StorePath> & storePathOpt)
+void CommonProto::Serialise<std::optional<StorePath>>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const std::optional<StorePath> & storePathOpt)
 {
     conn.to << (storePathOpt ? store.printStorePath(*storePathOpt) : "");
 }
 
 
-std::optional<ContentAddress> CommonProto::Serialise<std::optional<ContentAddress>>::read(const Store & store, CommonProto::ReadConn conn)
+std::optional<ContentAddress> CommonProto::Serialise<std::optional<ContentAddress>>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
 {
     return ContentAddress::parseOpt(readString(conn.from));
 }
 
-void CommonProto::Serialise<std::optional<ContentAddress>>::write(const Store & store, CommonProto::WriteConn conn, const std::optional<ContentAddress> & caOpt)
+void CommonProto::Serialise<std::optional<ContentAddress>>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const std::optional<ContentAddress> & caOpt)
 {
     conn.to << (caOpt ? renderContentAddress(*caOpt) : "");
 }
diff --git a/src/libstore/common-protocol.hh b/src/libstore/common-protocol.hh
index f3f28972a..a878e84c9 100644
--- a/src/libstore/common-protocol.hh
+++ b/src/libstore/common-protocol.hh
@@ -5,7 +5,7 @@
 
 namespace nix {
 
-class Store;
+struct StoreDirConfig;
 struct Source;
 
 // items being serialized
@@ -48,7 +48,7 @@ struct CommonProto
      * infer the type instead of having to write it down explicitly.
      */
     template<typename T>
-    static void write(const Store & store, WriteConn conn, const T & t)
+    static void write(const StoreDirConfig & store, WriteConn conn, const T & t)
     {
         CommonProto::Serialise<T>::write(store, conn, t);
     }
@@ -57,8 +57,8 @@ struct CommonProto
 #define DECLARE_COMMON_SERIALISER(T) \
     struct CommonProto::Serialise< T > \
     { \
-        static T read(const Store & store, CommonProto::ReadConn conn); \
-        static void write(const Store & store, CommonProto::WriteConn conn, const T & str); \
+        static T read(const StoreDirConfig & store, CommonProto::ReadConn conn); \
+        static void write(const StoreDirConfig & store, CommonProto::WriteConn conn, const T & str); \
     }
 
 template<>
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 1fecd1c97..239232c8e 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -11,7 +11,7 @@
 
 namespace nix {
 
-std::optional<StorePath> DerivationOutput::path(const Store & store, std::string_view drvName, OutputNameView outputName) const
+std::optional<StorePath> DerivationOutput::path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const
 {
     return std::visit(overloaded {
         [](const DerivationOutput::InputAddressed & doi) -> std::optional<StorePath> {
@@ -35,7 +35,7 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
 }
 
 
-StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, OutputNameView outputName) const
+StorePath DerivationOutput::CAFixed::path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const
 {
     return store.makeFixedOutputPathFromCA(
         outputPathName(drvName, outputName),
@@ -215,7 +215,7 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
 
 
 static DerivationOutput parseDerivationOutput(
-    const Store & store,
+    const StoreDirConfig & store,
     std::string_view pathS, std::string_view hashAlgo, std::string_view hashS,
     const ExperimentalFeatureSettings & xpSettings)
 {
@@ -262,7 +262,7 @@ static DerivationOutput parseDerivationOutput(
 }
 
 static DerivationOutput parseDerivationOutput(
-    const Store & store, std::istringstream & str,
+    const StoreDirConfig & store, std::istringstream & str,
     const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings)
 {
     expect(str, ","); const auto pathS = parseString(str);
@@ -291,7 +291,7 @@ enum struct DerivationATermVersion {
 };
 
 static DerivedPathMap<StringSet>::ChildNode parseDerivedPathMapNode(
-    const Store & store,
+    const StoreDirConfig & store,
     std::istringstream & str,
     DerivationATermVersion version)
 {
@@ -338,7 +338,7 @@ static DerivedPathMap<StringSet>::ChildNode parseDerivedPathMapNode(
 
 
 Derivation parseDerivation(
-    const Store & store, std::string && s, std::string_view name,
+    const StoreDirConfig & store, std::string && s, std::string_view name,
     const ExperimentalFeatureSettings & xpSettings)
 {
     Derivation drv;
@@ -471,7 +471,7 @@ static void printUnquotedStrings(std::string & res, ForwardIterator i, ForwardIt
 }
 
 
-static void unparseDerivedPathMapNode(const Store & store, std::string & s, const DerivedPathMap<StringSet>::ChildNode & node)
+static void unparseDerivedPathMapNode(const StoreDirConfig & store, std::string & s, const DerivedPathMap<StringSet>::ChildNode & node)
 {
     s += ',';
     if (node.childMap.empty()) {
@@ -512,7 +512,7 @@ static bool hasDynamicDrvDep(const Derivation & drv)
 }
 
 
-std::string Derivation::unparse(const Store & store, bool maskOutputs,
+std::string Derivation::unparse(const StoreDirConfig & store, bool maskOutputs,
     DerivedPathMap<StringSet>::ChildNode::Map * actualInputs) const
 {
     std::string s;
@@ -846,7 +846,7 @@ std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation &
 }
 
 
-static DerivationOutput readDerivationOutput(Source & in, const Store & store)
+static DerivationOutput readDerivationOutput(Source & in, const StoreDirConfig & store)
 {
     const auto pathS = readString(in);
     const auto hashAlgo = readString(in);
@@ -863,7 +863,7 @@ StringSet BasicDerivation::outputNames() const
     return names;
 }
 
-DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const
+DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const StoreDirConfig & store) const
 {
     DerivationOutputsAndOptPaths outsAndOptPaths;
     for (auto & [outputName, output] : outputs)
@@ -885,7 +885,7 @@ std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath)
 }
 
 
-Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name)
+Source & readDerivation(Source & in, const StoreDirConfig & store, BasicDerivation & drv, std::string_view name)
 {
     drv.name = name;
 
@@ -913,7 +913,7 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv,
 }
 
 
-void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv)
+void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDerivation & drv)
 {
     out << drv.outputs.size();
     for (auto & i : drv.outputs) {
@@ -1154,7 +1154,7 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const
 const Hash impureOutputHash = hashString(htSHA256, "impure");
 
 nlohmann::json DerivationOutput::toJSON(
-    const Store & store, std::string_view drvName, OutputNameView outputName) const
+    const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const
 {
     nlohmann::json res = nlohmann::json::object();
     std::visit(overloaded {
@@ -1181,7 +1181,7 @@ nlohmann::json DerivationOutput::toJSON(
 
 
 DerivationOutput DerivationOutput::fromJSON(
-    const Store & store, std::string_view drvName, OutputNameView outputName,
+    const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName,
     const nlohmann::json & _json,
     const ExperimentalFeatureSettings & xpSettings)
 {
@@ -1250,7 +1250,7 @@ DerivationOutput DerivationOutput::fromJSON(
 }
 
 
-nlohmann::json Derivation::toJSON(const Store & store) const
+nlohmann::json Derivation::toJSON(const StoreDirConfig & store) const
 {
     nlohmann::json res = nlohmann::json::object();
 
@@ -1303,7 +1303,7 @@ nlohmann::json Derivation::toJSON(const Store & store) const
 
 
 Derivation Derivation::fromJSON(
-    const Store & store,
+    const StoreDirConfig & store,
     const nlohmann::json & json,
     const ExperimentalFeatureSettings & xpSettings)
 {
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index fa14e7536..219e8e7d7 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -17,7 +17,7 @@
 
 namespace nix {
 
-class Store;
+struct StoreDirConfig;
 
 /* Abstract syntax of derivations. */
 
@@ -55,7 +55,7 @@ struct DerivationOutput
          * @param drvName The name of the derivation this is an output of, without the `.drv`.
          * @param outputName The name of this output.
          */
-        StorePath path(const Store & store, std::string_view drvName, OutputNameView outputName) const;
+        StorePath path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const;
 
         GENERATE_CMP(CAFixed, me->ca);
     };
@@ -132,17 +132,17 @@ struct DerivationOutput
      * the safer interface provided by
      * BasicDerivation::outputsAndOptPaths
      */
-    std::optional<StorePath> path(const Store & store, std::string_view drvName, OutputNameView outputName) const;
+    std::optional<StorePath> path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const;
 
     nlohmann::json toJSON(
-        const Store & store,
+        const StoreDirConfig & store,
         std::string_view drvName,
         OutputNameView outputName) const;
     /**
      * @param xpSettings Stop-gap to avoid globals during unit tests.
      */
     static DerivationOutput fromJSON(
-        const Store & store,
+        const StoreDirConfig & store,
         std::string_view drvName,
         OutputNameView outputName,
         const nlohmann::json & json,
@@ -304,7 +304,7 @@ struct BasicDerivation
      * augmented with knowledge of the Store paths they would be written
      * into.
      */
-    DerivationOutputsAndOptPaths outputsAndOptPaths(const Store & store) const;
+    DerivationOutputsAndOptPaths outputsAndOptPaths(const StoreDirConfig & store) const;
 
     static std::string_view nameFromPath(const StorePath & storePath);
 
@@ -318,6 +318,8 @@ struct BasicDerivation
         me->name);
 };
 
+class Store;
+
 struct Derivation : BasicDerivation
 {
     /**
@@ -328,7 +330,7 @@ struct Derivation : BasicDerivation
     /**
      * Print a derivation.
      */
-    std::string unparse(const Store & store, bool maskOutputs,
+    std::string unparse(const StoreDirConfig & store, bool maskOutputs,
         DerivedPathMap<StringSet>::ChildNode::Map * actualInputs = nullptr) const;
 
     /**
@@ -365,9 +367,9 @@ struct Derivation : BasicDerivation
     Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { }
     Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { }
 
-    nlohmann::json toJSON(const Store & store) const;
+    nlohmann::json toJSON(const StoreDirConfig & store) const;
     static Derivation fromJSON(
-        const Store & store,
+        const StoreDirConfig & store,
         const nlohmann::json & json,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
@@ -391,7 +393,7 @@ StorePath writeDerivation(Store & store,
  * Read a derivation from a file.
  */
 Derivation parseDerivation(
-    const Store & store,
+    const StoreDirConfig & store,
     std::string && s,
     std::string_view name,
     const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
@@ -493,8 +495,8 @@ extern Sync<DrvHashes> drvHashes;
 struct Source;
 struct Sink;
 
-Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name);
-void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv);
+Source & readDerivation(Source & in, const StoreDirConfig & store, BasicDerivation & drv, std::string_view name);
+void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDerivation & drv);
 
 /**
  * This creates an opaque and almost certainly unique string
diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc
index 47d784deb..3105dbc93 100644
--- a/src/libstore/derived-path.cc
+++ b/src/libstore/derived-path.cc
@@ -1,4 +1,5 @@
 #include "derived-path.hh"
+#include "derivations.hh"
 #include "store-api.hh"
 
 #include <nlohmann/json.hpp>
@@ -32,7 +33,7 @@ CMP(SingleDerivedPath, DerivedPathBuilt, outputs)
 #undef CMP
 #undef CMP_ONE
 
-nlohmann::json DerivedPath::Opaque::toJSON(const Store & store) const
+nlohmann::json DerivedPath::Opaque::toJSON(const StoreDirConfig & store) const
 {
     return store.printStorePath(path);
 }
@@ -86,50 +87,50 @@ nlohmann::json DerivedPath::toJSON(Store & store) const
     }, raw());
 }
 
-std::string DerivedPath::Opaque::to_string(const Store & store) const
+std::string DerivedPath::Opaque::to_string(const StoreDirConfig & store) const
 {
     return store.printStorePath(path);
 }
 
-std::string SingleDerivedPath::Built::to_string(const Store & store) const
+std::string SingleDerivedPath::Built::to_string(const StoreDirConfig & store) const
 {
     return drvPath->to_string(store) + "^" + output;
 }
 
-std::string SingleDerivedPath::Built::to_string_legacy(const Store & store) const
+std::string SingleDerivedPath::Built::to_string_legacy(const StoreDirConfig & store) const
 {
     return drvPath->to_string(store) + "!" + output;
 }
 
-std::string DerivedPath::Built::to_string(const Store & store) const
+std::string DerivedPath::Built::to_string(const StoreDirConfig & store) const
 {
     return drvPath->to_string(store)
         + '^'
         + outputs.to_string();
 }
 
-std::string DerivedPath::Built::to_string_legacy(const Store & store) const
+std::string DerivedPath::Built::to_string_legacy(const StoreDirConfig & store) const
 {
     return drvPath->to_string_legacy(store)
         + "!"
         + outputs.to_string();
 }
 
-std::string SingleDerivedPath::to_string(const Store & store) const
+std::string SingleDerivedPath::to_string(const StoreDirConfig & store) const
 {
     return std::visit(
         [&](const auto & req) { return req.to_string(store); },
         raw());
 }
 
-std::string DerivedPath::to_string(const Store & store) const
+std::string DerivedPath::to_string(const StoreDirConfig & store) const
 {
     return std::visit(
         [&](const auto & req) { return req.to_string(store); },
         raw());
 }
 
-std::string SingleDerivedPath::to_string_legacy(const Store & store) const
+std::string SingleDerivedPath::to_string_legacy(const StoreDirConfig & store) const
 {
     return std::visit(overloaded {
         [&](const SingleDerivedPath::Built & req) { return req.to_string_legacy(store); },
@@ -137,7 +138,7 @@ std::string SingleDerivedPath::to_string_legacy(const Store & store) const
     }, this->raw());
 }
 
-std::string DerivedPath::to_string_legacy(const Store & store) const
+std::string DerivedPath::to_string_legacy(const StoreDirConfig & store) const
 {
     return std::visit(overloaded {
         [&](const DerivedPath::Built & req) { return req.to_string_legacy(store); },
@@ -146,7 +147,7 @@ std::string DerivedPath::to_string_legacy(const Store & store) const
 }
 
 
-DerivedPath::Opaque DerivedPath::Opaque::parse(const Store & store, std::string_view s)
+DerivedPath::Opaque DerivedPath::Opaque::parse(const StoreDirConfig & store, std::string_view s)
 {
     return {store.parseStorePath(s)};
 }
@@ -166,7 +167,7 @@ void drvRequireExperiment(
 }
 
 SingleDerivedPath::Built SingleDerivedPath::Built::parse(
-    const Store & store, ref<SingleDerivedPath> drv,
+    const StoreDirConfig & store, ref<SingleDerivedPath> drv,
     OutputNameView output,
     const ExperimentalFeatureSettings & xpSettings)
 {
@@ -178,7 +179,7 @@ SingleDerivedPath::Built SingleDerivedPath::Built::parse(
 }
 
 DerivedPath::Built DerivedPath::Built::parse(
-    const Store & store, ref<SingleDerivedPath> drv,
+    const StoreDirConfig & store, ref<SingleDerivedPath> drv,
     OutputNameView outputsS,
     const ExperimentalFeatureSettings & xpSettings)
 {
@@ -190,7 +191,7 @@ DerivedPath::Built DerivedPath::Built::parse(
 }
 
 static SingleDerivedPath parseWithSingle(
-    const Store & store, std::string_view s, std::string_view separator,
+    const StoreDirConfig & store, std::string_view s, std::string_view separator,
     const ExperimentalFeatureSettings & xpSettings)
 {
     size_t n = s.rfind(separator);
@@ -207,7 +208,7 @@ static SingleDerivedPath parseWithSingle(
 }
 
 SingleDerivedPath SingleDerivedPath::parse(
-    const Store & store,
+    const StoreDirConfig & store,
     std::string_view s,
     const ExperimentalFeatureSettings & xpSettings)
 {
@@ -215,7 +216,7 @@ SingleDerivedPath SingleDerivedPath::parse(
 }
 
 SingleDerivedPath SingleDerivedPath::parseLegacy(
-    const Store & store,
+    const StoreDirConfig & store,
     std::string_view s,
     const ExperimentalFeatureSettings & xpSettings)
 {
@@ -223,7 +224,7 @@ SingleDerivedPath SingleDerivedPath::parseLegacy(
 }
 
 static DerivedPath parseWith(
-    const Store & store, std::string_view s, std::string_view separator,
+    const StoreDirConfig & store, std::string_view s, std::string_view separator,
     const ExperimentalFeatureSettings & xpSettings)
 {
     size_t n = s.rfind(separator);
@@ -240,7 +241,7 @@ static DerivedPath parseWith(
 }
 
 DerivedPath DerivedPath::parse(
-    const Store & store,
+    const StoreDirConfig & store,
     std::string_view s,
     const ExperimentalFeatureSettings & xpSettings)
 {
@@ -248,7 +249,7 @@ DerivedPath DerivedPath::parse(
 }
 
 DerivedPath DerivedPath::parseLegacy(
-    const Store & store,
+    const StoreDirConfig & store,
     std::string_view s,
     const ExperimentalFeatureSettings & xpSettings)
 {
diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh
index 4d7033df2..b12f9734a 100644
--- a/src/libstore/derived-path.hh
+++ b/src/libstore/derived-path.hh
@@ -12,6 +12,9 @@
 
 namespace nix {
 
+struct StoreDirConfig;
+
+// TODO stop needing this, `toJSON` below should be pure
 class Store;
 
 /**
@@ -24,9 +27,9 @@ class Store;
 struct DerivedPathOpaque {
     StorePath path;
 
-    std::string to_string(const Store & store) const;
-    static DerivedPathOpaque parse(const Store & store, std::string_view);
-    nlohmann::json toJSON(const Store & store) const;
+    std::string to_string(const StoreDirConfig & store) const;
+    static DerivedPathOpaque parse(const StoreDirConfig & store, std::string_view);
+    nlohmann::json toJSON(const StoreDirConfig & store) const;
 
     GENERATE_CMP(DerivedPathOpaque, me->path);
 };
@@ -59,18 +62,18 @@ struct SingleDerivedPathBuilt {
     /**
      * Uses `^` as the separator
      */
-    std::string to_string(const Store & store) const;
+    std::string to_string(const StoreDirConfig & store) const;
     /**
      * Uses `!` as the separator
      */
-    std::string to_string_legacy(const Store & store) const;
+    std::string to_string_legacy(const StoreDirConfig & store) const;
     /**
      * The caller splits on the separator, so it works for both variants.
      *
      * @param xpSettings Stop-gap to avoid globals during unit tests.
      */
     static SingleDerivedPathBuilt parse(
-        const Store & store, ref<SingleDerivedPath> drvPath,
+        const StoreDirConfig & store, ref<SingleDerivedPath> drvPath,
         OutputNameView outputs,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
     nlohmann::json toJSON(Store & store) const;
@@ -120,18 +123,18 @@ struct SingleDerivedPath : _SingleDerivedPathRaw {
     /**
      * Uses `^` as the separator
      */
-    std::string to_string(const Store & store) const;
+    std::string to_string(const StoreDirConfig & store) const;
     /**
      * Uses `!` as the separator
      */
-    std::string to_string_legacy(const Store & store) const;
+    std::string to_string_legacy(const StoreDirConfig & store) const;
     /**
      * Uses `^` as the separator
      *
      * @param xpSettings Stop-gap to avoid globals during unit tests.
      */
     static SingleDerivedPath parse(
-        const Store & store,
+        const StoreDirConfig & store,
         std::string_view,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
     /**
@@ -140,7 +143,7 @@ struct SingleDerivedPath : _SingleDerivedPathRaw {
      * @param xpSettings Stop-gap to avoid globals during unit tests.
      */
     static SingleDerivedPath parseLegacy(
-        const Store & store,
+        const StoreDirConfig & store,
         std::string_view,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
     nlohmann::json toJSON(Store & store) const;
@@ -182,18 +185,18 @@ struct DerivedPathBuilt {
     /**
      * Uses `^` as the separator
      */
-    std::string to_string(const Store & store) const;
+    std::string to_string(const StoreDirConfig & store) const;
     /**
      * Uses `!` as the separator
      */
-    std::string to_string_legacy(const Store & store) const;
+    std::string to_string_legacy(const StoreDirConfig & store) const;
     /**
      * The caller splits on the separator, so it works for both variants.
      *
      * @param xpSettings Stop-gap to avoid globals during unit tests.
      */
     static DerivedPathBuilt parse(
-        const Store & store, ref<SingleDerivedPath>,
+        const StoreDirConfig & store, ref<SingleDerivedPath>,
         std::string_view,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
     nlohmann::json toJSON(Store & store) const;
@@ -242,18 +245,18 @@ struct DerivedPath : _DerivedPathRaw {
     /**
      * Uses `^` as the separator
      */
-    std::string to_string(const Store & store) const;
+    std::string to_string(const StoreDirConfig & store) const;
     /**
      * Uses `!` as the separator
      */
-    std::string to_string_legacy(const Store & store) const;
+    std::string to_string_legacy(const StoreDirConfig & store) const;
     /**
      * Uses `^` as the separator
      *
      * @param xpSettings Stop-gap to avoid globals during unit tests.
      */
     static DerivedPath parse(
-        const Store & store,
+        const StoreDirConfig & store,
         std::string_view,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
     /**
@@ -262,7 +265,7 @@ struct DerivedPath : _DerivedPathRaw {
      * @param xpSettings Stop-gap to avoid globals during unit tests.
      */
     static DerivedPath parseLegacy(
-        const Store & store,
+        const StoreDirConfig & store,
         std::string_view,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
diff --git a/src/libstore/length-prefixed-protocol-helper.hh b/src/libstore/length-prefixed-protocol-helper.hh
index 4061b0cd6..0cf950a47 100644
--- a/src/libstore/length-prefixed-protocol-helper.hh
+++ b/src/libstore/length-prefixed-protocol-helper.hh
@@ -10,7 +10,7 @@
 
 namespace nix {
 
-class Store;
+struct StoreDirConfig;
 
 /**
  * Reusable serialisers for serialization container types in a
@@ -44,8 +44,8 @@ struct LengthPrefixedProtoHelper;
 #define LENGTH_PREFIXED_PROTO_HELPER(Inner, T) \
     struct LengthPrefixedProtoHelper< Inner, T > \
     { \
-        static T read(const Store & store, typename Inner::ReadConn conn); \
-        static void write(const Store & store, typename Inner::WriteConn conn, const T & str); \
+        static T read(const StoreDirConfig & store, typename Inner::ReadConn conn); \
+        static void write(const StoreDirConfig & store, typename Inner::WriteConn conn, const T & str); \
     private: \
         template<typename U> using S = typename Inner::template Serialise<U>; \
     }
@@ -67,7 +67,7 @@ LENGTH_PREFIXED_PROTO_HELPER(Inner, _X);
 template<class Inner, typename T>
 std::vector<T>
 LengthPrefixedProtoHelper<Inner, std::vector<T>>::read(
-    const Store & store, typename Inner::ReadConn conn)
+    const StoreDirConfig & store, typename Inner::ReadConn conn)
 {
     std::vector<T> resSet;
     auto size = readNum<size_t>(conn.from);
@@ -80,7 +80,7 @@ LengthPrefixedProtoHelper<Inner, std::vector<T>>::read(
 template<class Inner, typename T>
 void
 LengthPrefixedProtoHelper<Inner, std::vector<T>>::write(
-    const Store & store, typename Inner::WriteConn conn, const std::vector<T> & resSet)
+    const StoreDirConfig & store, typename Inner::WriteConn conn, const std::vector<T> & resSet)
 {
     conn.to << resSet.size();
     for (auto & key : resSet) {
@@ -91,7 +91,7 @@ LengthPrefixedProtoHelper<Inner, std::vector<T>>::write(
 template<class Inner, typename T>
 std::set<T>
 LengthPrefixedProtoHelper<Inner, std::set<T>>::read(
-    const Store & store, typename Inner::ReadConn conn)
+    const StoreDirConfig & store, typename Inner::ReadConn conn)
 {
     std::set<T> resSet;
     auto size = readNum<size_t>(conn.from);
@@ -104,7 +104,7 @@ LengthPrefixedProtoHelper<Inner, std::set<T>>::read(
 template<class Inner, typename T>
 void
 LengthPrefixedProtoHelper<Inner, std::set<T>>::write(
-    const Store & store, typename Inner::WriteConn conn, const std::set<T> & resSet)
+    const StoreDirConfig & store, typename Inner::WriteConn conn, const std::set<T> & resSet)
 {
     conn.to << resSet.size();
     for (auto & key : resSet) {
@@ -115,7 +115,7 @@ LengthPrefixedProtoHelper<Inner, std::set<T>>::write(
 template<class Inner, typename K, typename V>
 std::map<K, V>
 LengthPrefixedProtoHelper<Inner, std::map<K, V>>::read(
-    const Store & store, typename Inner::ReadConn conn)
+    const StoreDirConfig & store, typename Inner::ReadConn conn)
 {
     std::map<K, V> resMap;
     auto size = readNum<size_t>(conn.from);
@@ -130,7 +130,7 @@ LengthPrefixedProtoHelper<Inner, std::map<K, V>>::read(
 template<class Inner, typename K, typename V>
 void
 LengthPrefixedProtoHelper<Inner, std::map<K, V>>::write(
-    const Store & store, typename Inner::WriteConn conn, const std::map<K, V> & resMap)
+    const StoreDirConfig & store, typename Inner::WriteConn conn, const std::map<K, V> & resMap)
 {
     conn.to << resMap.size();
     for (auto & i : resMap) {
@@ -142,7 +142,7 @@ LengthPrefixedProtoHelper<Inner, std::map<K, V>>::write(
 template<class Inner, typename... Ts>
 std::tuple<Ts...>
 LengthPrefixedProtoHelper<Inner, std::tuple<Ts...>>::read(
-    const Store & store, typename Inner::ReadConn conn)
+    const StoreDirConfig & store, typename Inner::ReadConn conn)
 {
     return std::tuple<Ts...> {
         S<Ts>::read(store, conn)...,
@@ -152,7 +152,7 @@ LengthPrefixedProtoHelper<Inner, std::tuple<Ts...>>::read(
 template<class Inner, typename... Ts>
 void
 LengthPrefixedProtoHelper<Inner, std::tuple<Ts...>>::write(
-    const Store & store, typename Inner::WriteConn conn, const std::tuple<Ts...> & res)
+    const StoreDirConfig & store, typename Inner::WriteConn conn, const std::tuple<Ts...> & res)
 {
     std::apply([&]<typename... Us>(const Us &... args) {
         (S<Us>::write(store, conn, args), ...);
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index a5e9426f8..e091683dc 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -4,6 +4,7 @@
 #include "pathlocks.hh"
 #include "worker-protocol.hh"
 #include "derivations.hh"
+#include "realisation.hh"
 #include "nar-info.hh"
 #include "references.hh"
 #include "callback.hh"
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index 1035691c7..9f63fbbb5 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -4,6 +4,7 @@
 #include "local-store.hh"
 #include "store-api.hh"
 #include "thread-pool.hh"
+#include "realisation.hh"
 #include "topo-sort.hh"
 #include "callback.hh"
 #include "closure.hh"
diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc
index af6837370..026e37647 100644
--- a/src/libstore/path-with-outputs.cc
+++ b/src/libstore/path-with-outputs.cc
@@ -5,7 +5,7 @@
 
 namespace nix {
 
-std::string StorePathWithOutputs::to_string(const Store & store) const
+std::string StorePathWithOutputs::to_string(const StoreDirConfig & store) const
 {
     return outputs.empty()
         ? store.printStorePath(path)
@@ -85,7 +85,7 @@ std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s)
 }
 
 
-StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs)
+StorePathWithOutputs parsePathWithOutputs(const StoreDirConfig & store, std::string_view pathWithOutputs)
 {
     auto [path, outputs] = parsePathWithOutputs(pathWithOutputs);
     return StorePathWithOutputs { store.parseStorePath(path), std::move(outputs) };
diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh
index 57e03252d..5f76a583a 100644
--- a/src/libstore/path-with-outputs.hh
+++ b/src/libstore/path-with-outputs.hh
@@ -6,6 +6,8 @@
 
 namespace nix {
 
+struct StoreDirConfig;
+
 /**
  * This is a deprecated old type just for use by the old CLI, and older
  * versions of the RPC protocols. In new code don't use it; you want
@@ -19,7 +21,7 @@ struct StorePathWithOutputs
     StorePath path;
     std::set<std::string> outputs;
 
-    std::string to_string(const Store & store) const;
+    std::string to_string(const StoreDirConfig & store) const;
 
     DerivedPath toDerivedPath() const;
 
@@ -32,14 +34,14 @@ std::vector<DerivedPath> toDerivedPaths(const std::vector<StorePathWithOutputs>)
 
 std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s);
 
-class Store;
-
 /**
  * Split a string specifying a derivation and a set of outputs
  * (/nix/store/hash-foo!out1,out2,...) into the derivation path
  * and the outputs.
  */
-StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs);
+StorePathWithOutputs parsePathWithOutputs(const StoreDirConfig & store, std::string_view pathWithOutputs);
+
+class Store;
 
 StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs);
 
diff --git a/src/libstore/serve-protocol-impl.hh b/src/libstore/serve-protocol-impl.hh
index a3ce81026..6f3b177ac 100644
--- a/src/libstore/serve-protocol-impl.hh
+++ b/src/libstore/serve-protocol-impl.hh
@@ -16,11 +16,11 @@ namespace nix {
 /* protocol-agnostic templates */
 
 #define SERVE_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \
-    TEMPLATE T ServeProto::Serialise< T >::read(const Store & store, ServeProto::ReadConn conn) \
+    TEMPLATE T ServeProto::Serialise< T >::read(const StoreDirConfig & store, ServeProto::ReadConn conn) \
     { \
         return LengthPrefixedProtoHelper<ServeProto, T >::read(store, conn); \
     } \
-    TEMPLATE void ServeProto::Serialise< T >::write(const Store & store, ServeProto::WriteConn conn, const T & t) \
+    TEMPLATE void ServeProto::Serialise< T >::write(const StoreDirConfig & store, ServeProto::WriteConn conn, const T & t) \
     { \
         LengthPrefixedProtoHelper<ServeProto, T >::write(store, conn, t); \
     }
@@ -41,12 +41,12 @@ SERVE_USE_LENGTH_PREFIX_SERIALISER(
 template<typename T>
 struct ServeProto::Serialise
 {
-    static T read(const Store & store, ServeProto::ReadConn conn)
+    static T read(const StoreDirConfig & store, ServeProto::ReadConn conn)
     {
         return CommonProto::Serialise<T>::read(store,
             CommonProto::ReadConn { .from = conn.from });
     }
-    static void write(const Store & store, ServeProto::WriteConn conn, const T & t)
+    static void write(const StoreDirConfig & store, ServeProto::WriteConn conn, const T & t)
     {
         CommonProto::Serialise<T>::write(store,
             CommonProto::WriteConn { .to = conn.to },
diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc
index 97a0ddf0e..e0ac80c4e 100644
--- a/src/libstore/serve-protocol.cc
+++ b/src/libstore/serve-protocol.cc
@@ -13,7 +13,7 @@ namespace nix {
 
 /* protocol-specific definitions */
 
-BuildResult ServeProto::Serialise<BuildResult>::read(const Store & store, ServeProto::ReadConn conn)
+BuildResult ServeProto::Serialise<BuildResult>::read(const StoreDirConfig & store, ServeProto::ReadConn conn)
 {
     BuildResult status;
     status.status = (BuildResult::Status) readInt(conn.from);
@@ -35,7 +35,7 @@ BuildResult ServeProto::Serialise<BuildResult>::read(const Store & store, ServeP
     return status;
 }
 
-void ServeProto::Serialise<BuildResult>::write(const Store & store, ServeProto::WriteConn conn, const BuildResult & status)
+void ServeProto::Serialise<BuildResult>::write(const StoreDirConfig & store, ServeProto::WriteConn conn, const BuildResult & status)
 {
     conn.to
         << status.status
diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh
index ba159f6e9..6e9d66e2d 100644
--- a/src/libstore/serve-protocol.hh
+++ b/src/libstore/serve-protocol.hh
@@ -13,7 +13,7 @@ namespace nix {
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
 
-class Store;
+struct StoreDirConfig;
 struct Source;
 
 // items being serialised
@@ -72,8 +72,8 @@ struct ServeProto
     // See `worker-protocol.hh` for a longer explanation.
 #if 0
     {
-        static T read(const Store & store, ReadConn conn);
-        static void write(const Store & store, WriteConn conn, const T & t);
+        static T read(const StoreDirConfig & store, ReadConn conn);
+        static void write(const StoreDirConfig & store, WriteConn conn, const T & t);
     };
 #endif
 
@@ -82,7 +82,7 @@ struct ServeProto
      * infer the type instead of having to write it down explicitly.
      */
     template<typename T>
-    static void write(const Store & store, WriteConn conn, const T & t)
+    static void write(const StoreDirConfig & store, WriteConn conn, const T & t)
     {
         ServeProto::Serialise<T>::write(store, conn, t);
     }
@@ -135,8 +135,8 @@ inline std::ostream & operator << (std::ostream & s, ServeProto::Command op)
 #define DECLARE_SERVE_SERIALISER(T) \
     struct ServeProto::Serialise< T > \
     { \
-        static T read(const Store & store, ServeProto::ReadConn conn); \
-        static void write(const Store & store, ServeProto::WriteConn conn, const T & t); \
+        static T read(const StoreDirConfig & store, ServeProto::ReadConn conn); \
+        static void write(const StoreDirConfig & store, ServeProto::WriteConn conn, const T & t); \
     };
 
 template<>
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index a681bb6cf..e44376fa2 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -1,6 +1,8 @@
 #include "crypto.hh"
 #include "source-accessor.hh"
 #include "globals.hh"
+#include "derived-path.hh"
+#include "realisation.hh"
 #include "derivations.hh"
 #include "store-api.hh"
 #include "util.hh"
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index bee5ec16c..4342445ba 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -1,8 +1,6 @@
 #pragma once
 ///@file
 
-#include "nar-info.hh"
-#include "realisation.hh"
 #include "path.hh"
 #include "derived-path.hh"
 #include "hash.hh"
@@ -68,8 +66,13 @@ MakeError(SubstituterDisabled, Error);
 
 MakeError(InvalidStoreURI, Error);
 
+struct Realisation;
+struct RealisedPath;
+struct DrvOutput;
+
 struct BasicDerivation;
 struct Derivation;
+
 struct SourceAccessor;
 class NarInfoDiskCache;
 class Store;
@@ -811,7 +814,7 @@ void copyStorePath(
  */
 std::map<StorePath, StorePath> copyPaths(
     Store & srcStore, Store & dstStore,
-    const RealisedPath::Set &,
+    const std::set<RealisedPath> &,
     RepairFlag repair = NoRepair,
     CheckSigsFlag checkSigs = CheckSigs,
     SubstituteFlag substitute = NoSubstitute);
@@ -828,7 +831,7 @@ std::map<StorePath, StorePath> copyPaths(
  */
 void copyClosure(
     Store & srcStore, Store & dstStore,
-    const RealisedPath::Set & paths,
+    const std::set<RealisedPath> & paths,
     RepairFlag repair = NoRepair,
     CheckSigsFlag checkSigs = CheckSigs,
     SubstituteFlag substitute = NoSubstitute);
diff --git a/src/libstore/worker-protocol-impl.hh b/src/libstore/worker-protocol-impl.hh
index c043588d6..026cc37bc 100644
--- a/src/libstore/worker-protocol-impl.hh
+++ b/src/libstore/worker-protocol-impl.hh
@@ -16,11 +16,11 @@ namespace nix {
 /* protocol-agnostic templates */
 
 #define WORKER_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \
-    TEMPLATE T WorkerProto::Serialise< T >::read(const Store & store, WorkerProto::ReadConn conn) \
+    TEMPLATE T WorkerProto::Serialise< T >::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) \
     { \
         return LengthPrefixedProtoHelper<WorkerProto, T >::read(store, conn); \
     } \
-    TEMPLATE void WorkerProto::Serialise< T >::write(const Store & store, WorkerProto::WriteConn conn, const T & t) \
+    TEMPLATE void WorkerProto::Serialise< T >::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const T & t) \
     { \
         LengthPrefixedProtoHelper<WorkerProto, T >::write(store, conn, t); \
     }
@@ -41,12 +41,12 @@ WORKER_USE_LENGTH_PREFIX_SERIALISER(
 template<typename T>
 struct WorkerProto::Serialise
 {
-    static T read(const Store & store, WorkerProto::ReadConn conn)
+    static T read(const StoreDirConfig & store, WorkerProto::ReadConn conn)
     {
         return CommonProto::Serialise<T>::read(store,
             CommonProto::ReadConn { .from = conn.from });
     }
-    static void write(const Store & store, WorkerProto::WriteConn conn, const T & t)
+    static void write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const T & t)
     {
         CommonProto::Serialise<T>::write(store,
             CommonProto::WriteConn { .to = conn.to },
diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc
index d618b9bd8..4edab7894 100644
--- a/src/libstore/worker-protocol.cc
+++ b/src/libstore/worker-protocol.cc
@@ -14,7 +14,7 @@ namespace nix {
 
 /* protocol-specific definitions */
 
-std::optional<TrustedFlag> WorkerProto::Serialise<std::optional<TrustedFlag>>::read(const Store & store, WorkerProto::ReadConn conn)
+std::optional<TrustedFlag> WorkerProto::Serialise<std::optional<TrustedFlag>>::read(const StoreDirConfig & store, WorkerProto::ReadConn conn)
 {
     auto temp = readNum<uint8_t>(conn.from);
     switch (temp) {
@@ -29,7 +29,7 @@ std::optional<TrustedFlag> WorkerProto::Serialise<std::optional<TrustedFlag>>::r
     }
 }
 
-void WorkerProto::Serialise<std::optional<TrustedFlag>>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional<TrustedFlag> & optTrusted)
+void WorkerProto::Serialise<std::optional<TrustedFlag>>::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const std::optional<TrustedFlag> & optTrusted)
 {
     if (!optTrusted)
         conn.to << (uint8_t)0;
@@ -48,7 +48,7 @@ void WorkerProto::Serialise<std::optional<TrustedFlag>>::write(const Store & sto
 }
 
 
-DerivedPath WorkerProto::Serialise<DerivedPath>::read(const Store & store, WorkerProto::ReadConn conn)
+DerivedPath WorkerProto::Serialise<DerivedPath>::read(const StoreDirConfig & store, WorkerProto::ReadConn conn)
 {
     auto s = readString(conn.from);
     if (GET_PROTOCOL_MINOR(conn.version) >= 30) {
@@ -58,7 +58,7 @@ DerivedPath WorkerProto::Serialise<DerivedPath>::read(const Store & store, Worke
     }
 }
 
-void WorkerProto::Serialise<DerivedPath>::write(const Store & store, WorkerProto::WriteConn conn, const DerivedPath & req)
+void WorkerProto::Serialise<DerivedPath>::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const DerivedPath & req)
 {
     if (GET_PROTOCOL_MINOR(conn.version) >= 30) {
         conn.to << req.to_string_legacy(store);
@@ -82,7 +82,7 @@ void WorkerProto::Serialise<DerivedPath>::write(const Store & store, WorkerProto
 }
 
 
-KeyedBuildResult WorkerProto::Serialise<KeyedBuildResult>::read(const Store & store, WorkerProto::ReadConn conn)
+KeyedBuildResult WorkerProto::Serialise<KeyedBuildResult>::read(const StoreDirConfig & store, WorkerProto::ReadConn conn)
 {
     auto path = WorkerProto::Serialise<DerivedPath>::read(store, conn);
     auto br = WorkerProto::Serialise<BuildResult>::read(store, conn);
@@ -92,14 +92,14 @@ KeyedBuildResult WorkerProto::Serialise<KeyedBuildResult>::read(const Store & st
     };
 }
 
-void WorkerProto::Serialise<KeyedBuildResult>::write(const Store & store, WorkerProto::WriteConn conn, const KeyedBuildResult & res)
+void WorkerProto::Serialise<KeyedBuildResult>::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const KeyedBuildResult & res)
 {
     WorkerProto::write(store, conn, res.path);
     WorkerProto::write(store, conn, static_cast<const BuildResult &>(res));
 }
 
 
-BuildResult WorkerProto::Serialise<BuildResult>::read(const Store & store, WorkerProto::ReadConn conn)
+BuildResult WorkerProto::Serialise<BuildResult>::read(const StoreDirConfig & store, WorkerProto::ReadConn conn)
 {
     BuildResult res;
     res.status = (BuildResult::Status) readInt(conn.from);
@@ -121,7 +121,7 @@ BuildResult WorkerProto::Serialise<BuildResult>::read(const Store & store, Worke
     return res;
 }
 
-void WorkerProto::Serialise<BuildResult>::write(const Store & store, WorkerProto::WriteConn conn, const BuildResult & res)
+void WorkerProto::Serialise<BuildResult>::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const BuildResult & res)
 {
     conn.to
         << res.status
@@ -142,7 +142,7 @@ void WorkerProto::Serialise<BuildResult>::write(const Store & store, WorkerProto
 }
 
 
-ValidPathInfo WorkerProto::Serialise<ValidPathInfo>::read(const Store & store, ReadConn conn)
+ValidPathInfo WorkerProto::Serialise<ValidPathInfo>::read(const StoreDirConfig & store, ReadConn conn)
 {
     auto path = WorkerProto::Serialise<StorePath>::read(store, conn);
     return ValidPathInfo {
@@ -151,14 +151,14 @@ ValidPathInfo WorkerProto::Serialise<ValidPathInfo>::read(const Store & store, R
     };
 }
 
-void WorkerProto::Serialise<ValidPathInfo>::write(const Store & store, WriteConn conn, const ValidPathInfo & pathInfo)
+void WorkerProto::Serialise<ValidPathInfo>::write(const StoreDirConfig & store, WriteConn conn, const ValidPathInfo & pathInfo)
 {
     WorkerProto::write(store, conn, pathInfo.path);
     WorkerProto::write(store, conn, static_cast<const UnkeyedValidPathInfo &>(pathInfo));
 }
 
 
-UnkeyedValidPathInfo WorkerProto::Serialise<UnkeyedValidPathInfo>::read(const Store & store, ReadConn conn)
+UnkeyedValidPathInfo WorkerProto::Serialise<UnkeyedValidPathInfo>::read(const StoreDirConfig & store, ReadConn conn)
 {
     auto deriver = readString(conn.from);
     auto narHash = Hash::parseAny(readString(conn.from), htSHA256);
@@ -174,7 +174,7 @@ UnkeyedValidPathInfo WorkerProto::Serialise<UnkeyedValidPathInfo>::read(const St
     return info;
 }
 
-void WorkerProto::Serialise<UnkeyedValidPathInfo>::write(const Store & store, WriteConn conn, const UnkeyedValidPathInfo & pathInfo)
+void WorkerProto::Serialise<UnkeyedValidPathInfo>::write(const StoreDirConfig & store, WriteConn conn, const UnkeyedValidPathInfo & pathInfo)
 {
     conn.to
         << (pathInfo.deriver ? store.printStorePath(*pathInfo.deriver) : "")
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index dcd54ad16..9b02aa2b5 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -24,7 +24,7 @@ namespace nix {
 #define STDERR_RESULT         0x52534c54
 
 
-class Store;
+struct StoreDirConfig;
 struct Source;
 
 // items being serialised
@@ -100,8 +100,8 @@ struct WorkerProto
     // This makes for a quicker debug cycle, as desired.
 #if 0
     {
-        static T read(const Store & store, ReadConn conn);
-        static void write(const Store & store, WriteConn conn, const T & t);
+        static T read(const StoreDirConfig & store, ReadConn conn);
+        static void write(const StoreDirConfig & store, WriteConn conn, const T & t);
     };
 #endif
 
@@ -110,7 +110,7 @@ struct WorkerProto
      * infer the type instead of having to write it down explicitly.
      */
     template<typename T>
-    static void write(const Store & store, WriteConn conn, const T & t)
+    static void write(const StoreDirConfig & store, WriteConn conn, const T & t)
     {
         WorkerProto::Serialise<T>::write(store, conn, t);
     }
@@ -197,8 +197,8 @@ inline std::ostream & operator << (std::ostream & s, WorkerProto::Op op)
 #define DECLARE_WORKER_SERIALISER(T) \
     struct WorkerProto::Serialise< T > \
     { \
-        static T read(const Store & store, WorkerProto::ReadConn conn); \
-        static void write(const Store & store, WorkerProto::WriteConn conn, const T & t); \
+        static T read(const StoreDirConfig & store, WorkerProto::ReadConn conn); \
+        static void write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const T & t); \
     };
 
 template<>
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 60bc08146..c46095a14 100644
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -13,6 +13,7 @@
 #include "store-api.hh"
 #include "local-fs-store.hh"
 #include "globals.hh"
+#include "realisation.hh"
 #include "derivations.hh"
 #include "util.hh"
 #include "shared.hh"
diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc
index 7f2bb93b6..b64af758f 100644
--- a/src/nix-copy-closure/nix-copy-closure.cc
+++ b/src/nix-copy-closure/nix-copy-closure.cc
@@ -1,4 +1,5 @@
 #include "shared.hh"
+#include "realisation.hh"
 #include "store-api.hh"
 #include "legacy.hh"
 

From 2678b51b31febdc6464935e1680d2272a954c3b5 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 5 Nov 2023 12:17:33 -0500
Subject: [PATCH 061/294] Narrower scope for `nativeSystem`

I don't think we need a CPP defininition and a header entry, and this
way allows constant expression elimination.
---
 src/libstore/build/local-derivation-goal.cc | 2 ++
 src/libutil/error.cc                        | 2 --
 src/libutil/util.hh                         | 6 ------
 3 files changed, 2 insertions(+), 8 deletions(-)

diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index dcb7dc6bc..e1794139f 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -1620,6 +1620,8 @@ void setupSeccomp()
         seccomp_release(ctx);
     });
 
+    constexpr std::string_view nativeSystem = SYSTEM;
+
     if (nativeSystem == "x86_64-linux" &&
         seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0)
         throw SysError("unable to add 32-bit seccomp architecture");
diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index dd9612471..1badc1069 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -7,8 +7,6 @@
 
 namespace nix {
 
-const std::string nativeSystem = SYSTEM;
-
 void BaseError::addTrace(std::shared_ptr<AbstractPos> && e, hintformat hint, bool frame)
 {
     err.traces.push_front(Trace { .pos = std::move(e), .hint = hint, .frame = frame });
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index b302d6f45..75683f8fe 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -34,12 +34,6 @@ struct Source;
 
 void initLibUtil();
 
-/**
- * The system for which Nix is compiled.
- */
-extern const std::string nativeSystem;
-
-
 /**
  * @return an environment variable.
  */

From ac89bb064aeea85a62b82a6daf0ecca7190a28b7 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 25 Oct 2023 00:43:36 -0400
Subject: [PATCH 062/294] Split up `util.{hh,cc}`

All OS and IO operations should be moved out, leaving only some misc
portable pure functions.

This is useful to avoid copious CPP when doing things like Windows and
Emscripten ports.

Newly exposed functions to break cycles:

 - `restoreSignals`
 - `updateWindowSize`
---
 perl/lib/Nix/Store.xs                         |    1 -
 src/libcmd/built-path.hh                      |    3 +
 src/libcmd/common-eval-args.cc                |    1 -
 src/libcmd/editor-for.cc                      |    2 +-
 src/libcmd/installable-attr-path.hh           |    1 -
 src/libcmd/installables.cc                    |    1 +
 src/libcmd/installables.hh                    |    1 -
 src/libcmd/markdown.cc                        |    1 +
 src/libcmd/repl.cc                            |    3 +
 src/libexpr/attr-path.cc                      |    1 -
 src/libexpr/eval-cache.cc                     |    1 +
 src/libexpr/eval-settings.cc                  |    1 +
 src/libexpr/eval-settings.hh                  |    2 +
 src/libexpr/eval.cc                           |    1 +
 src/libexpr/flake/config.cc                   |    3 +-
 src/libexpr/flake/flake.cc                    |    1 +
 src/libexpr/get-drvs.cc                       |    1 -
 src/libexpr/parser.y                          |    1 +
 src/libexpr/primops.cc                        |    1 +
 src/libexpr/search-path.cc                    |    1 -
 src/libexpr/value-to-json.cc                  |    2 +-
 src/libexpr/value-to-xml.cc                   |    2 +-
 src/libexpr/value/context.cc                  |    1 +
 src/libexpr/value/context.hh                  |    1 -
 src/libfetchers/cache.cc                      |    1 +
 src/libfetchers/fetch-settings.hh             |    1 -
 src/libfetchers/git.cc                        |    3 +-
 src/libfetchers/input-accessor.hh             |    2 +
 src/libfetchers/mercurial.cc                  |    2 +
 src/libfetchers/registry.cc                   |    2 +-
 src/libmain/common-args.cc                    |    2 +
 src/libmain/loggers.cc                        |    2 +-
 src/libmain/progress-bar.cc                   |    2 +-
 src/libmain/shared.cc                         |    3 +-
 src/libmain/shared.hh                         |    2 +-
 src/libstore/binary-cache-store.cc            |    1 +
 src/libstore/build/child.cc                   |   37 +
 src/libstore/build/child.hh                   |   11 +
 src/libstore/build/hook-instance.cc           |    2 +
 src/libstore/build/hook-instance.hh           |    1 +
 src/libstore/build/local-derivation-goal.cc   |    3 +
 src/libstore/build/local-derivation-goal.hh   |    1 +
 src/libstore/build/worker.cc                  |    1 +
 src/libstore/common-protocol.cc               |    1 -
 src/libstore/crypto.cc                        |    1 +
 src/libstore/derived-path-map.cc              |    1 +
 src/libstore/derived-path-map.hh              |    1 +
 src/libstore/derived-path.hh                  |    2 +-
 src/libstore/filetransfer.cc                  |    3 +-
 src/libstore/gc.cc                            |    7 +
 src/libstore/globals.cc                       |   13 +-
 src/libstore/globals.hh                       |    2 +-
 src/libstore/local-store.cc                   |    1 +
 src/libstore/local-store.hh                   |    1 -
 src/libstore/lock.cc                          |    1 +
 src/libstore/machines.cc                      |    1 -
 src/libstore/nar-info-disk-cache.cc           |    1 +
 src/libstore/optimise-store.cc                |    2 +-
 src/libstore/path-references.cc               |    1 -
 src/libstore/path-references.hh               |    1 +
 src/libstore/pathlocks.cc                     |    1 +
 src/libstore/pathlocks.hh                     |    2 +-
 src/libstore/profiles.cc                      |    2 +-
 src/libstore/remote-store-connection.hh       |    3 +
 src/libstore/serve-protocol.cc                |    1 -
 src/libstore/sqlite.cc                        |    1 +
 src/libstore/ssh.cc                           |    3 +
 src/libstore/ssh.hh                           |    3 +-
 src/libstore/store-api.cc                     |    2 +
 src/libstore/tests/machines.cc                |    2 +
 src/libstore/tests/protocol.hh                |    3 +
 src/libstore/uds-remote-store.cc              |    1 +
 src/libstore/worker-protocol.cc               |    1 -
 src/libutil/archive.cc                        |    3 +-
 src/libutil/args.cc                           |    3 +
 src/libutil/args.hh                           |    5 +-
 src/libutil/canon-path.cc                     |    2 +-
 src/libutil/cgroup.cc                         |    1 +
 src/libutil/compression.cc                    |    2 +-
 src/libutil/config.cc                         |    2 +
 src/libutil/current-process.cc                |  110 +
 src/libutil/current-process.hh                |   34 +
 src/libutil/environment-variables.cc          |   49 +
 src/libutil/environment-variables.hh          |   41 +
 src/libutil/error.cc                          |    3 +
 src/libutil/file-descriptor.cc                |  254 +++
 src/libutil/file-descriptor.hh                |   84 +
 src/libutil/file-system.cc                    |  647 ++++++
 src/libutil/file-system.hh                    |  238 +++
 src/libutil/filesystem.cc                     |  162 --
 src/libutil/fs-sink.hh                        |    1 +
 src/libutil/hash.cc                           |    1 -
 src/libutil/hash.hh                           |    1 +
 src/libutil/logging.cc                        |    3 +
 src/libutil/monitor-fd.hh                     |    2 +
 src/libutil/namespaces.cc                     |   69 +-
 src/libutil/namespaces.hh                     |   23 +
 src/libutil/posix-source-accessor.cc          |    1 +
 src/libutil/processes.cc                      |  421 ++++
 src/libutil/processes.hh                      |  123 ++
 src/libutil/references.cc                     |    1 -
 src/libutil/serialise.cc                      |    2 +-
 src/libutil/serialise.hh                      |    1 +
 src/libutil/signals.cc                        |  188 ++
 src/libutil/signals.hh                        |  104 +
 src/libutil/suggestions.cc                    |    4 +-
 src/libutil/tarfile.cc                        |    1 +
 src/libutil/terminal.cc                       |  108 +
 src/libutil/terminal.hh                       |   38 +
 src/libutil/tests/logging.cc                  |    1 -
 src/libutil/tests/tests.cc                    |    3 +
 src/libutil/thread-pool.cc                    |    2 +
 src/libutil/thread-pool.hh                    |    2 +-
 src/libutil/unix-domain-socket.cc             |  100 +
 src/libutil/unix-domain-socket.hh             |   31 +
 src/libutil/users.cc                          |  116 ++
 src/libutil/users.hh                          |   58 +
 src/libutil/util.cc                           | 1816 +----------------
 src/libutil/util.hh                           |  617 ------
 src/nix-build/nix-build.cc                    |    2 +-
 src/nix-channel/nix-channel.cc                |    2 +-
 .../nix-collect-garbage.cc                    |    2 +
 src/nix-env/nix-env.cc                        |    2 +-
 src/nix-env/user-env.cc                       |    1 -
 src/nix-instantiate/nix-instantiate.cc        |    1 -
 src/nix-store/dotgraph.cc                     |    1 -
 src/nix-store/graphml.cc                      |    1 -
 src/nix-store/nix-store.cc                    |    1 -
 src/nix/daemon.cc                             |    3 +-
 src/nix/develop.cc                            |    1 -
 src/nix/doctor.cc                             |    1 -
 src/nix/edit.cc                               |    1 +
 src/nix/flake.cc                              |    1 +
 src/nix/main.cc                               |    2 +
 src/nix/run.cc                                |    1 +
 src/nix/sigs.cc                               |    1 +
 src/nix/upgrade-nix.cc                        |    1 +
 src/nix/verify.cc                             |    1 +
 138 files changed, 3028 insertions(+), 2654 deletions(-)
 create mode 100644 src/libstore/build/child.cc
 create mode 100644 src/libstore/build/child.hh
 create mode 100644 src/libutil/current-process.cc
 create mode 100644 src/libutil/current-process.hh
 create mode 100644 src/libutil/environment-variables.cc
 create mode 100644 src/libutil/environment-variables.hh
 create mode 100644 src/libutil/file-descriptor.cc
 create mode 100644 src/libutil/file-descriptor.hh
 create mode 100644 src/libutil/file-system.cc
 create mode 100644 src/libutil/file-system.hh
 delete mode 100644 src/libutil/filesystem.cc
 create mode 100644 src/libutil/processes.cc
 create mode 100644 src/libutil/processes.hh
 create mode 100644 src/libutil/signals.cc
 create mode 100644 src/libutil/signals.hh
 create mode 100644 src/libutil/terminal.cc
 create mode 100644 src/libutil/terminal.hh
 create mode 100644 src/libutil/unix-domain-socket.cc
 create mode 100644 src/libutil/unix-domain-socket.hh
 create mode 100644 src/libutil/users.cc
 create mode 100644 src/libutil/users.hh

diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index 08f812b31..f89ac4077 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -11,7 +11,6 @@
 #include "derivations.hh"
 #include "globals.hh"
 #include "store-api.hh"
-#include "util.hh"
 #include "crypto.hh"
 
 #include <sodium.h>
diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh
index e677bc810..7154cc504 100644
--- a/src/libcmd/built-path.hh
+++ b/src/libcmd/built-path.hh
@@ -1,3 +1,6 @@
+#pragma once
+///@file
+
 #include "derived-path.hh"
 #include "realisation.hh"
 
diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc
index e53bc4c01..91fa881b1 100644
--- a/src/libcmd/common-eval-args.cc
+++ b/src/libcmd/common-eval-args.cc
@@ -2,7 +2,6 @@
 #include "common-eval-args.hh"
 #include "shared.hh"
 #include "filetransfer.hh"
-#include "util.hh"
 #include "eval.hh"
 #include "fetchers.hh"
 #include "registry.hh"
diff --git a/src/libcmd/editor-for.cc b/src/libcmd/editor-for.cc
index a17c6f12a..619d3673f 100644
--- a/src/libcmd/editor-for.cc
+++ b/src/libcmd/editor-for.cc
@@ -1,5 +1,5 @@
-#include "util.hh"
 #include "editor-for.hh"
+#include "environment-variables.hh"
 
 namespace nix {
 
diff --git a/src/libcmd/installable-attr-path.hh b/src/libcmd/installable-attr-path.hh
index e9f0c33da..86c2f8219 100644
--- a/src/libcmd/installable-attr-path.hh
+++ b/src/libcmd/installable-attr-path.hh
@@ -4,7 +4,6 @@
 #include "globals.hh"
 #include "installable-value.hh"
 #include "outputs-spec.hh"
-#include "util.hh"
 #include "command.hh"
 #include "attr-path.hh"
 #include "common-eval-args.hh"
diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index bc0b8a988..e7f58556f 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -4,6 +4,7 @@
 #include "installable-attr-path.hh"
 #include "installable-flake.hh"
 #include "outputs-spec.hh"
+#include "users.hh"
 #include "util.hh"
 #include "command.hh"
 #include "attr-path.hh"
diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh
index b0dc0dc02..e087f935c 100644
--- a/src/libcmd/installables.hh
+++ b/src/libcmd/installables.hh
@@ -1,7 +1,6 @@
 #pragma once
 ///@file
 
-#include "util.hh"
 #include "path.hh"
 #include "outputs-spec.hh"
 #include "derived-path.hh"
diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc
index 668a07763..8b3bbc1b5 100644
--- a/src/libcmd/markdown.cc
+++ b/src/libcmd/markdown.cc
@@ -1,6 +1,7 @@
 #include "markdown.hh"
 #include "util.hh"
 #include "finally.hh"
+#include "terminal.hh"
 
 #include <sys/queue.h>
 #include <lowdown.h>
diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc
index 2e17a29a7..bf5643a5c 100644
--- a/src/libcmd/repl.cc
+++ b/src/libcmd/repl.cc
@@ -22,6 +22,7 @@ extern "C" {
 #include "repl.hh"
 
 #include "ansicolor.hh"
+#include "signals.hh"
 #include "shared.hh"
 #include "eval.hh"
 #include "eval-cache.hh"
@@ -36,6 +37,8 @@ extern "C" {
 #include "globals.hh"
 #include "flake/flake.hh"
 #include "flake/lockfile.hh"
+#include "users.hh"
+#include "terminal.hh"
 #include "editor-for.hh"
 #include "finally.hh"
 #include "markdown.hh"
diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc
index d12345710..7481a2232 100644
--- a/src/libexpr/attr-path.cc
+++ b/src/libexpr/attr-path.cc
@@ -1,6 +1,5 @@
 #include "attr-path.hh"
 #include "eval-inline.hh"
-#include "util.hh"
 
 
 namespace nix {
diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc
index 10fc799a9..6c0e33709 100644
--- a/src/libexpr/eval-cache.cc
+++ b/src/libexpr/eval-cache.cc
@@ -1,3 +1,4 @@
+#include "users.hh"
 #include "eval-cache.hh"
 #include "sqlite.hh"
 #include "eval.hh"
diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc
index 93b4a5289..444a7d7d6 100644
--- a/src/libexpr/eval-settings.cc
+++ b/src/libexpr/eval-settings.cc
@@ -1,3 +1,4 @@
+#include "users.hh"
 #include "globals.hh"
 #include "profiles.hh"
 #include "eval.hh"
diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh
index 5473d688e..db2971acb 100644
--- a/src/libexpr/eval-settings.hh
+++ b/src/libexpr/eval-settings.hh
@@ -1,4 +1,6 @@
 #pragma once
+///@file
+
 #include "config.hh"
 
 namespace nix {
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index d26cde423..dfe81cbf7 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -14,6 +14,7 @@
 #include "print.hh"
 #include "fs-input-accessor.hh"
 #include "memory-input-accessor.hh"
+#include "signals.hh"
 
 #include <algorithm>
 #include <chrono>
diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc
index e89014862..3c7ed5d8a 100644
--- a/src/libexpr/flake/config.cc
+++ b/src/libexpr/flake/config.cc
@@ -1,6 +1,7 @@
-#include "flake.hh"
+#include "users.hh"
 #include "globals.hh"
 #include "fetch-settings.hh"
+#include "flake.hh"
 
 #include <nlohmann/json.hpp>
 
diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index ded132695..54de53e0b 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -1,3 +1,4 @@
+#include "terminal.hh"
 #include "flake.hh"
 #include "eval.hh"
 #include "eval-settings.hh"
diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc
index fe3e6f7ee..d4e946d81 100644
--- a/src/libexpr/get-drvs.cc
+++ b/src/libexpr/get-drvs.cc
@@ -1,5 +1,4 @@
 #include "get-drvs.hh"
-#include "util.hh"
 #include "eval-inline.hh"
 #include "derivations.hh"
 #include "store-api.hh"
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 607795937..b86cef217 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -19,6 +19,7 @@
 #include <variant>
 
 #include "util.hh"
+#include "users.hh"
 
 #include "nixexpr.hh"
 #include "eval.hh"
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index e3c775d90..36340d0f9 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -10,6 +10,7 @@
 #include "path-references.hh"
 #include "store-api.hh"
 #include "util.hh"
+#include "processes.hh"
 #include "value-to-json.hh"
 #include "value-to-xml.hh"
 #include "primops.hh"
diff --git a/src/libexpr/search-path.cc b/src/libexpr/search-path.cc
index 180d5f8b1..a25767496 100644
--- a/src/libexpr/search-path.cc
+++ b/src/libexpr/search-path.cc
@@ -1,5 +1,4 @@
 #include "search-path.hh"
-#include "util.hh"
 
 namespace nix {
 
diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc
index cbc91f509..74b3ebf13 100644
--- a/src/libexpr/value-to-json.cc
+++ b/src/libexpr/value-to-json.cc
@@ -1,7 +1,7 @@
 #include "value-to-json.hh"
 #include "eval-inline.hh"
-#include "util.hh"
 #include "store-api.hh"
+#include "signals.hh"
 
 #include <cstdlib>
 #include <iomanip>
diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc
index bd7a4ae30..5032115bb 100644
--- a/src/libexpr/value-to-xml.cc
+++ b/src/libexpr/value-to-xml.cc
@@ -1,7 +1,7 @@
 #include "value-to-xml.hh"
 #include "xml-writer.hh"
 #include "eval-inline.hh"
-#include "util.hh"
+#include "signals.hh"
 
 #include <cstdlib>
 
diff --git a/src/libexpr/value/context.cc b/src/libexpr/value/context.cc
index 22361d8fa..6d9633268 100644
--- a/src/libexpr/value/context.cc
+++ b/src/libexpr/value/context.cc
@@ -1,3 +1,4 @@
+#include "util.hh"
 #include "value/context.hh"
 
 #include <optional>
diff --git a/src/libexpr/value/context.hh b/src/libexpr/value/context.hh
index 9f1d59317..51fd30a44 100644
--- a/src/libexpr/value/context.hh
+++ b/src/libexpr/value/context.hh
@@ -1,7 +1,6 @@
 #pragma once
 ///@file
 
-#include "util.hh"
 #include "comparator.hh"
 #include "derived-path.hh"
 #include "variant-wrapper.hh"
diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc
index 0c8ecac9d..b72a464e8 100644
--- a/src/libfetchers/cache.cc
+++ b/src/libfetchers/cache.cc
@@ -1,4 +1,5 @@
 #include "cache.hh"
+#include "users.hh"
 #include "sqlite.hh"
 #include "sync.hh"
 #include "store-api.hh"
diff --git a/src/libfetchers/fetch-settings.hh b/src/libfetchers/fetch-settings.hh
index 6108a179c..f095963a8 100644
--- a/src/libfetchers/fetch-settings.hh
+++ b/src/libfetchers/fetch-settings.hh
@@ -3,7 +3,6 @@
 
 #include "types.hh"
 #include "config.hh"
-#include "util.hh"
 
 #include <map>
 #include <limits>
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 72fba0582..cc735996b 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -1,11 +1,12 @@
 #include "fetchers.hh"
+#include "users.hh"
 #include "cache.hh"
 #include "globals.hh"
 #include "tarfile.hh"
 #include "store-api.hh"
 #include "url-parts.hh"
 #include "pathlocks.hh"
-#include "util.hh"
+#include "processes.hh"
 #include "git.hh"
 
 #include "fetch-settings.hh"
diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh
index 5dc05a363..6857ce156 100644
--- a/src/libfetchers/input-accessor.hh
+++ b/src/libfetchers/input-accessor.hh
@@ -1,8 +1,10 @@
 #pragma once
+///@file
 
 #include "source-accessor.hh"
 #include "ref.hh"
 #include "types.hh"
+#include "file-system.hh"
 #include "repair-flag.hh"
 #include "content-address.hh"
 
diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc
index eda33dfe7..9244acf39 100644
--- a/src/libfetchers/mercurial.cc
+++ b/src/libfetchers/mercurial.cc
@@ -1,4 +1,6 @@
 #include "fetchers.hh"
+#include "processes.hh"
+#include "users.hh"
 #include "cache.hh"
 #include "globals.hh"
 #include "tarfile.hh"
diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc
index a0fff9ceb..9c7bc0cfe 100644
--- a/src/libfetchers/registry.cc
+++ b/src/libfetchers/registry.cc
@@ -1,6 +1,6 @@
 #include "registry.hh"
 #include "tarball.hh"
-#include "util.hh"
+#include "users.hh"
 #include "globals.hh"
 #include "store-api.hh"
 #include "local-fs-store.hh"
diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc
index 205b77808..5b49aaabc 100644
--- a/src/libmain/common-args.cc
+++ b/src/libmain/common-args.cc
@@ -1,7 +1,9 @@
 #include "common-args.hh"
 #include "args/root.hh"
 #include "globals.hh"
+#include "logging.hh"
 #include "loggers.hh"
+#include "util.hh"
 
 namespace nix {
 
diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc
index cda5cb939..9829859de 100644
--- a/src/libmain/loggers.cc
+++ b/src/libmain/loggers.cc
@@ -1,6 +1,6 @@
 #include "loggers.hh"
+#include "environment-variables.hh"
 #include "progress-bar.hh"
-#include "util.hh"
 
 namespace nix {
 
diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc
index 45b1fdfd1..a7aee47c3 100644
--- a/src/libmain/progress-bar.cc
+++ b/src/libmain/progress-bar.cc
@@ -1,5 +1,5 @@
 #include "progress-bar.hh"
-#include "util.hh"
+#include "terminal.hh"
 #include "sync.hh"
 #include "store-api.hh"
 #include "names.hh"
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 9c2ad039a..862ef355b 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -1,10 +1,11 @@
 #include "globals.hh"
+#include "current-process.hh"
 #include "shared.hh"
 #include "store-api.hh"
 #include "gc-store.hh"
-#include "util.hh"
 #include "loggers.hh"
 #include "progress-bar.hh"
+#include "signals.hh"
 
 #include <algorithm>
 #include <cctype>
diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh
index 3159fe479..c68f6cd83 100644
--- a/src/libmain/shared.hh
+++ b/src/libmain/shared.hh
@@ -1,7 +1,7 @@
 #pragma once
 ///@file
 
-#include "util.hh"
+#include "processes.hh"
 #include "args.hh"
 #include "args/root.hh"
 #include "common-args.hh"
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 6a52c4c51..ae483c95e 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -11,6 +11,7 @@
 #include "nar-accessor.hh"
 #include "thread-pool.hh"
 #include "callback.hh"
+#include "signals.hh"
 
 #include <chrono>
 #include <future>
diff --git a/src/libstore/build/child.cc b/src/libstore/build/child.cc
new file mode 100644
index 000000000..aa31c3caf
--- /dev/null
+++ b/src/libstore/build/child.cc
@@ -0,0 +1,37 @@
+#include "child.hh"
+#include "current-process.hh"
+#include "logging.hh"
+
+#include <fcntl.h>
+#include <unistd.h>
+
+namespace nix {
+
+void commonChildInit()
+{
+    logger = makeSimpleLogger();
+
+    const static std::string pathNullDevice = "/dev/null";
+    restoreProcessContext(false);
+
+    /* Put the child in a separate session (and thus a separate
+       process group) so that it has no controlling terminal (meaning
+       that e.g. ssh cannot open /dev/tty) and it doesn't receive
+       terminal signals. */
+    if (setsid() == -1)
+        throw SysError("creating a new session");
+
+    /* Dup stderr to stdout. */
+    if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1)
+        throw SysError("cannot dup stderr into stdout");
+
+    /* Reroute stdin to /dev/null. */
+    int fdDevNull = open(pathNullDevice.c_str(), O_RDWR);
+    if (fdDevNull == -1)
+        throw SysError("cannot open '%1%'", pathNullDevice);
+    if (dup2(fdDevNull, STDIN_FILENO) == -1)
+        throw SysError("cannot dup null device into stdin");
+    close(fdDevNull);
+}
+
+}
diff --git a/src/libstore/build/child.hh b/src/libstore/build/child.hh
new file mode 100644
index 000000000..3dfc552b9
--- /dev/null
+++ b/src/libstore/build/child.hh
@@ -0,0 +1,11 @@
+#pragma once
+///@file
+
+namespace nix {
+
+/**
+ * Common initialisation performed in child processes.
+ */
+void commonChildInit();
+
+}
diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/build/hook-instance.cc
index 337c60bd4..5d045ec3d 100644
--- a/src/libstore/build/hook-instance.cc
+++ b/src/libstore/build/hook-instance.cc
@@ -1,5 +1,7 @@
 #include "globals.hh"
 #include "hook-instance.hh"
+#include "file-system.hh"
+#include "child.hh"
 
 namespace nix {
 
diff --git a/src/libstore/build/hook-instance.hh b/src/libstore/build/hook-instance.hh
index d84f62877..61cf534f4 100644
--- a/src/libstore/build/hook-instance.hh
+++ b/src/libstore/build/hook-instance.hh
@@ -3,6 +3,7 @@
 
 #include "logging.hh"
 #include "serialise.hh"
+#include "processes.hh"
 
 namespace nix {
 
diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index e1794139f..adb011e30 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -15,7 +15,10 @@
 #include "json-utils.hh"
 #include "cgroup.hh"
 #include "personality.hh"
+#include "current-process.hh"
 #include "namespaces.hh"
+#include "child.hh"
+#include "unix-domain-socket.hh"
 
 #include <regex>
 #include <queue>
diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh
index 1cb68a869..88152a645 100644
--- a/src/libstore/build/local-derivation-goal.hh
+++ b/src/libstore/build/local-derivation-goal.hh
@@ -3,6 +3,7 @@
 
 #include "derivation-goal.hh"
 #include "local-store.hh"
+#include "processes.hh"
 
 namespace nix {
 
diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc
index 37cb86b91..01914e2d6 100644
--- a/src/libstore/build/worker.cc
+++ b/src/libstore/build/worker.cc
@@ -4,6 +4,7 @@
 #include "drv-output-substitution-goal.hh"
 #include "local-derivation-goal.hh"
 #include "hook-instance.hh"
+#include "signals.hh"
 
 #include <poll.h>
 
diff --git a/src/libstore/common-protocol.cc b/src/libstore/common-protocol.cc
index f906814bc..68445258f 100644
--- a/src/libstore/common-protocol.cc
+++ b/src/libstore/common-protocol.cc
@@ -1,5 +1,4 @@
 #include "serialise.hh"
-#include "util.hh"
 #include "path-with-outputs.hh"
 #include "store-api.hh"
 #include "build-result.hh"
diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc
index 1027469c9..1b705733c 100644
--- a/src/libstore/crypto.cc
+++ b/src/libstore/crypto.cc
@@ -1,4 +1,5 @@
 #include "crypto.hh"
+#include "file-system.hh"
 #include "util.hh"
 #include "globals.hh"
 
diff --git a/src/libstore/derived-path-map.cc b/src/libstore/derived-path-map.cc
index 5982c04b3..4c1ea417a 100644
--- a/src/libstore/derived-path-map.cc
+++ b/src/libstore/derived-path-map.cc
@@ -1,4 +1,5 @@
 #include "derived-path-map.hh"
+#include "util.hh"
 
 namespace nix {
 
diff --git a/src/libstore/derived-path-map.hh b/src/libstore/derived-path-map.hh
index 4d72b301e..393cdedf7 100644
--- a/src/libstore/derived-path-map.hh
+++ b/src/libstore/derived-path-map.hh
@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "types.hh"
 #include "derived-path.hh"
diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh
index 4d7033df2..6c5dfeed9 100644
--- a/src/libstore/derived-path.hh
+++ b/src/libstore/derived-path.hh
@@ -1,10 +1,10 @@
 #pragma once
 ///@file
 
-#include "util.hh"
 #include "path.hh"
 #include "outputs-spec.hh"
 #include "comparator.hh"
+#include "config.hh"
 
 #include <variant>
 
diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc
index a283af5a2..dcbec4acd 100644
--- a/src/libstore/filetransfer.cc
+++ b/src/libstore/filetransfer.cc
@@ -1,11 +1,12 @@
 #include "filetransfer.hh"
-#include "util.hh"
+#include "namespaces.hh"
 #include "globals.hh"
 #include "store-api.hh"
 #include "s3.hh"
 #include "compression.hh"
 #include "finally.hh"
 #include "callback.hh"
+#include "signals.hh"
 
 #if ENABLE_S3
 #include <aws/core/client/ClientConfiguration.h>
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index fb7895817..8d05ae4bd 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -2,6 +2,13 @@
 #include "globals.hh"
 #include "local-store.hh"
 #include "finally.hh"
+#include "unix-domain-socket.hh"
+#include "signals.hh"
+
+#if !defined(__linux__)
+// For shelling out to lsof
+# include "processes.hh"
+#endif
 
 #include <functional>
 #include <queue>
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 9c25d9868..cc416a4d6 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -1,7 +1,8 @@
 #include "globals.hh"
-#include "util.hh"
+#include "current-process.hh"
 #include "archive.hh"
 #include "args.hh"
+#include "users.hh"
 #include "abstract-setting-to-json.hh"
 #include "compute-levels.hh"
 
@@ -17,9 +18,13 @@
 #include <sodium/core.h>
 
 #ifdef __GLIBC__
-#include <gnu/lib-names.h>
-#include <nss.h>
-#include <dlfcn.h>
+# include <gnu/lib-names.h>
+# include <nss.h>
+# include <dlfcn.h>
+#endif
+
+#if __APPLE__
+# include "processes.hh"
 #endif
 
 #include "config-impl.hh"
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 12fb48d93..8e034f5a9 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -3,7 +3,7 @@
 
 #include "types.hh"
 #include "config.hh"
-#include "util.hh"
+#include "environment-variables.hh"
 #include "experimental-features.hh"
 
 #include <map>
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index a5e9426f8..2a3582ad8 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -10,6 +10,7 @@
 #include "topo-sort.hh"
 #include "finally.hh"
 #include "compression.hh"
+#include "signals.hh"
 
 #include <iostream>
 #include <algorithm>
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index fe26a0f27..6d589bee5 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -7,7 +7,6 @@
 #include "store-api.hh"
 #include "indirect-root-store.hh"
 #include "sync.hh"
-#include "util.hh"
 
 #include <chrono>
 #include <future>
diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc
index 165e4969f..87f55ce49 100644
--- a/src/libstore/lock.cc
+++ b/src/libstore/lock.cc
@@ -1,4 +1,5 @@
 #include "lock.hh"
+#include "file-system.hh"
 #include "globals.hh"
 #include "pathlocks.hh"
 
diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc
index e87f46980..512115893 100644
--- a/src/libstore/machines.cc
+++ b/src/libstore/machines.cc
@@ -1,5 +1,4 @@
 #include "machines.hh"
-#include "util.hh"
 #include "globals.hh"
 #include "store-api.hh"
 
diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index cdbcf7e74..e50c15939 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -1,4 +1,5 @@
 #include "nar-info-disk-cache.hh"
+#include "users.hh"
 #include "sync.hh"
 #include "sqlite.hh"
 #include "globals.hh"
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index 23c6a41e4..a4ac413b3 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -1,6 +1,6 @@
-#include "util.hh"
 #include "local-store.hh"
 #include "globals.hh"
+#include "signals.hh"
 
 #include <cstdlib>
 #include <cstring>
diff --git a/src/libstore/path-references.cc b/src/libstore/path-references.cc
index 33cf66ce3..274b596c0 100644
--- a/src/libstore/path-references.cc
+++ b/src/libstore/path-references.cc
@@ -1,6 +1,5 @@
 #include "path-references.hh"
 #include "hash.hh"
-#include "util.hh"
 #include "archive.hh"
 
 #include <map>
diff --git a/src/libstore/path-references.hh b/src/libstore/path-references.hh
index 7b44e3261..0553003f8 100644
--- a/src/libstore/path-references.hh
+++ b/src/libstore/path-references.hh
@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "references.hh"
 #include "path.hh"
diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc
index adc763e6a..2b5b8dfe7 100644
--- a/src/libstore/pathlocks.cc
+++ b/src/libstore/pathlocks.cc
@@ -1,6 +1,7 @@
 #include "pathlocks.hh"
 #include "util.hh"
 #include "sync.hh"
+#include "signals.hh"
 
 #include <cerrno>
 #include <cstdlib>
diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh
index 4921df352..7fcfa2e40 100644
--- a/src/libstore/pathlocks.hh
+++ b/src/libstore/pathlocks.hh
@@ -1,7 +1,7 @@
 #pragma once
 ///@file
 
-#include "util.hh"
+#include "file-descriptor.hh"
 
 namespace nix {
 
diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc
index 239047dd6..e8b88693d 100644
--- a/src/libstore/profiles.cc
+++ b/src/libstore/profiles.cc
@@ -1,7 +1,7 @@
 #include "profiles.hh"
 #include "store-api.hh"
 #include "local-fs-store.hh"
-#include "util.hh"
+#include "users.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
diff --git a/src/libstore/remote-store-connection.hh b/src/libstore/remote-store-connection.hh
index e4a9cacb9..44328b06b 100644
--- a/src/libstore/remote-store-connection.hh
+++ b/src/libstore/remote-store-connection.hh
@@ -1,3 +1,6 @@
+#pragma once
+///@file
+
 #include "remote-store.hh"
 #include "worker-protocol.hh"
 #include "pool.hh"
diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc
index 97a0ddf0e..9bfcc279c 100644
--- a/src/libstore/serve-protocol.cc
+++ b/src/libstore/serve-protocol.cc
@@ -1,5 +1,4 @@
 #include "serialise.hh"
-#include "util.hh"
 #include "path-with-outputs.hh"
 #include "store-api.hh"
 #include "build-result.hh"
diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc
index 7c8decb74..d7432a305 100644
--- a/src/libstore/sqlite.cc
+++ b/src/libstore/sqlite.cc
@@ -2,6 +2,7 @@
 #include "globals.hh"
 #include "util.hh"
 #include "url.hh"
+#include "signals.hh"
 
 #include <sqlite3.h>
 
diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc
index da32f1b79..03b2f0be9 100644
--- a/src/libstore/ssh.cc
+++ b/src/libstore/ssh.cc
@@ -1,5 +1,8 @@
 #include "ssh.hh"
 #include "finally.hh"
+#include "current-process.hh"
+#include "environment-variables.hh"
+#include "util.hh"
 
 namespace nix {
 
diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh
index 94b952af9..bfcd6f21c 100644
--- a/src/libstore/ssh.hh
+++ b/src/libstore/ssh.hh
@@ -1,8 +1,9 @@
 #pragma once
 ///@file
 
-#include "util.hh"
 #include "sync.hh"
+#include "processes.hh"
+#include "file-system.hh"
 
 namespace nix {
 
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 646b0ec7d..c9ebb6c14 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -14,6 +14,8 @@
 // FIXME this should not be here, see TODO below on
 // `addMultipleToStore`.
 #include "worker-protocol.hh"
+#include "signals.hh"
+#include "users.hh"
 
 #include <nlohmann/json.hpp>
 #include <regex>
diff --git a/src/libstore/tests/machines.cc b/src/libstore/tests/machines.cc
index f51052b14..fede328ea 100644
--- a/src/libstore/tests/machines.cc
+++ b/src/libstore/tests/machines.cc
@@ -1,5 +1,7 @@
 #include "machines.hh"
 #include "globals.hh"
+#include "file-system.hh"
+#include "util.hh"
 
 #include <gmock/gmock-matchers.h>
 
diff --git a/src/libstore/tests/protocol.hh b/src/libstore/tests/protocol.hh
index 0378b3e1f..466032a79 100644
--- a/src/libstore/tests/protocol.hh
+++ b/src/libstore/tests/protocol.hh
@@ -1,3 +1,6 @@
+#pragma once
+///@file
+
 #include <nlohmann/json.hpp>
 #include <gtest/gtest.h>
 
diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc
index 99589f8b2..226cdf717 100644
--- a/src/libstore/uds-remote-store.cc
+++ b/src/libstore/uds-remote-store.cc
@@ -1,4 +1,5 @@
 #include "uds-remote-store.hh"
+#include "unix-domain-socket.hh"
 #include "worker-protocol.hh"
 
 #include <sys/types.h>
diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc
index d618b9bd8..1d202f8d1 100644
--- a/src/libstore/worker-protocol.cc
+++ b/src/libstore/worker-protocol.cc
@@ -1,5 +1,4 @@
 #include "serialise.hh"
-#include "util.hh"
 #include "path-with-outputs.hh"
 #include "store-api.hh"
 #include "build-result.hh"
diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index 4ca84d357..465df2073 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -6,9 +6,10 @@
 #include <strings.h> // for strcasecmp
 
 #include "archive.hh"
-#include "util.hh"
 #include "config.hh"
 #include "posix-source-accessor.hh"
+#include "file-system.hh"
+#include "signals.hh"
 
 namespace nix {
 
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 811353c18..0b65519a3 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -1,6 +1,9 @@
 #include "args.hh"
 #include "args/root.hh"
 #include "hash.hh"
+#include "environment-variables.hh"
+#include "signals.hh"
+#include "users.hh"
 #include "json-utils.hh"
 
 #include <glob.h>
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index e3b41313f..45fd678e7 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -2,12 +2,15 @@
 ///@file
 
 #include <iostream>
+#include <functional>
 #include <map>
 #include <memory>
+#include <optional>
 
 #include <nlohmann/json_fwd.hpp>
 
-#include "util.hh"
+#include "types.hh"
+#include "experimental-features.hh"
 
 namespace nix {
 
diff --git a/src/libutil/canon-path.cc b/src/libutil/canon-path.cc
index 040464532..f678fae94 100644
--- a/src/libutil/canon-path.cc
+++ b/src/libutil/canon-path.cc
@@ -1,5 +1,5 @@
 #include "canon-path.hh"
-#include "util.hh"
+#include "file-system.hh"
 
 namespace nix {
 
diff --git a/src/libutil/cgroup.cc b/src/libutil/cgroup.cc
index a008481ca..4c2bf31ff 100644
--- a/src/libutil/cgroup.cc
+++ b/src/libutil/cgroup.cc
@@ -2,6 +2,7 @@
 
 #include "cgroup.hh"
 #include "util.hh"
+#include "file-system.hh"
 #include "finally.hh"
 
 #include <chrono>
diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index ba0847cde..d06f1f87b 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -1,6 +1,6 @@
 #include "compression.hh"
+#include "signals.hh"
 #include "tarfile.hh"
-#include "util.hh"
 #include "finally.hh"
 #include "logging.hh"
 
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 8e06273ee..0bf36c987 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -2,6 +2,8 @@
 #include "args.hh"
 #include "abstract-setting-to-json.hh"
 #include "experimental-features.hh"
+#include "util.hh"
+#include "file-system.hh"
 
 #include "config-impl.hh"
 
diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc
new file mode 100644
index 000000000..352a6a0fb
--- /dev/null
+++ b/src/libutil/current-process.cc
@@ -0,0 +1,110 @@
+#include "current-process.hh"
+#include "namespaces.hh"
+#include "util.hh"
+#include "finally.hh"
+#include "file-system.hh"
+#include "processes.hh"
+#include "signals.hh"
+
+#ifdef __APPLE__
+# include <mach-o/dyld.h>
+#endif
+
+#if __linux__
+# include <mutex>
+# include <sys/resource.h>
+# include "cgroup.hh"
+#endif
+
+#include <sys/mount.h>
+
+namespace nix {
+
+unsigned int getMaxCPU()
+{
+    #if __linux__
+    try {
+        auto cgroupFS = getCgroupFS();
+        if (!cgroupFS) return 0;
+
+        auto cgroups = getCgroups("/proc/self/cgroup");
+        auto cgroup = cgroups[""];
+        if (cgroup == "") return 0;
+
+        auto cpuFile = *cgroupFS + "/" + cgroup + "/cpu.max";
+
+        auto cpuMax = readFile(cpuFile);
+        auto cpuMaxParts = tokenizeString<std::vector<std::string>>(cpuMax, " \n");
+        auto quota = cpuMaxParts[0];
+        auto period = cpuMaxParts[1];
+        if (quota != "max")
+                return std::ceil(std::stoi(quota) / std::stof(period));
+    } catch (Error &) { ignoreException(lvlDebug); }
+    #endif
+
+    return 0;
+}
+
+
+//////////////////////////////////////////////////////////////////////
+
+
+#if __linux__
+rlim_t savedStackSize = 0;
+#endif
+
+void setStackSize(size_t stackSize)
+{
+    #if __linux__
+    struct rlimit limit;
+    if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) {
+        savedStackSize = limit.rlim_cur;
+        limit.rlim_cur = stackSize;
+        setrlimit(RLIMIT_STACK, &limit);
+    }
+    #endif
+}
+
+void restoreProcessContext(bool restoreMounts)
+{
+    restoreSignals();
+    if (restoreMounts) {
+        restoreMountNamespace();
+    }
+
+    #if __linux__
+    if (savedStackSize) {
+        struct rlimit limit;
+        if (getrlimit(RLIMIT_STACK, &limit) == 0) {
+            limit.rlim_cur = savedStackSize;
+            setrlimit(RLIMIT_STACK, &limit);
+        }
+    }
+    #endif
+}
+
+
+//////////////////////////////////////////////////////////////////////
+
+
+std::optional<Path> getSelfExe()
+{
+    static auto cached = []() -> std::optional<Path>
+    {
+        #if __linux__
+        return readLink("/proc/self/exe");
+        #elif __APPLE__
+        char buf[1024];
+        uint32_t size = sizeof(buf);
+        if (_NSGetExecutablePath(buf, &size) == 0)
+            return buf;
+        else
+            return std::nullopt;
+        #else
+        return std::nullopt;
+        #endif
+    }();
+    return cached;
+}
+
+}
diff --git a/src/libutil/current-process.hh b/src/libutil/current-process.hh
new file mode 100644
index 000000000..826d6fe20
--- /dev/null
+++ b/src/libutil/current-process.hh
@@ -0,0 +1,34 @@
+#pragma once
+///@file
+
+#include <optional>
+
+#include "types.hh"
+
+namespace nix {
+
+/**
+ * If cgroups are active, attempt to calculate the number of CPUs available.
+ * If cgroups are unavailable or if cpu.max is set to "max", return 0.
+ */
+unsigned int getMaxCPU();
+
+/**
+ * Change the stack size.
+ */
+void setStackSize(size_t stackSize);
+
+/**
+ * Restore the original inherited Unix process context (such as signal
+ * masks, stack size).
+
+ * See startSignalHandlerThread(), saveSignalMask().
+ */
+void restoreProcessContext(bool restoreMounts = true);
+
+/**
+ * @return the path of the current executable.
+ */
+std::optional<Path> getSelfExe();
+
+}
diff --git a/src/libutil/environment-variables.cc b/src/libutil/environment-variables.cc
new file mode 100644
index 000000000..6618d7872
--- /dev/null
+++ b/src/libutil/environment-variables.cc
@@ -0,0 +1,49 @@
+#include "util.hh"
+#include "environment-variables.hh"
+
+extern char * * environ __attribute__((weak));
+
+namespace nix {
+
+std::optional<std::string> getEnv(const std::string & key)
+{
+    char * value = getenv(key.c_str());
+    if (!value) return {};
+    return std::string(value);
+}
+
+std::optional<std::string> getEnvNonEmpty(const std::string & key) {
+    auto value = getEnv(key);
+    if (value == "") return {};
+    return value;
+}
+
+std::map<std::string, std::string> getEnv()
+{
+    std::map<std::string, std::string> env;
+    for (size_t i = 0; environ[i]; ++i) {
+        auto s = environ[i];
+        auto eq = strchr(s, '=');
+        if (!eq)
+            // invalid env, just keep going
+            continue;
+        env.emplace(std::string(s, eq), std::string(eq + 1));
+    }
+    return env;
+}
+
+
+void clearEnv()
+{
+    for (auto & name : getEnv())
+        unsetenv(name.first.c_str());
+}
+
+void replaceEnv(const std::map<std::string, std::string> & newEnv)
+{
+    clearEnv();
+    for (auto & newEnvVar : newEnv)
+        setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1);
+}
+
+}
diff --git a/src/libutil/environment-variables.hh b/src/libutil/environment-variables.hh
new file mode 100644
index 000000000..21eb4619b
--- /dev/null
+++ b/src/libutil/environment-variables.hh
@@ -0,0 +1,41 @@
+#pragma once
+/**
+ * @file
+ *
+ * Utilities for working with the current process's environment
+ * variables.
+ */
+
+#include <optional>
+
+#include "types.hh"
+
+namespace nix {
+
+/**
+ * @return an environment variable.
+ */
+std::optional<std::string> getEnv(const std::string & key);
+
+/**
+ * @return a non empty environment variable. Returns nullopt if the env
+ * variable is set to ""
+ */
+std::optional<std::string> getEnvNonEmpty(const std::string & key);
+
+/**
+ * Get the entire environment.
+ */
+std::map<std::string, std::string> getEnv();
+
+/**
+ * Clear the environment.
+ */
+void clearEnv();
+
+/**
+ * Replace the entire environment with the given one.
+ */
+void replaceEnv(const std::map<std::string, std::string> & newEnv);
+
+}
diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 1badc1069..8488e7e21 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -1,4 +1,7 @@
 #include "error.hh"
+#include "environment-variables.hh"
+#include "signals.hh"
+#include "terminal.hh"
 
 #include <iostream>
 #include <optional>
diff --git a/src/libutil/file-descriptor.cc b/src/libutil/file-descriptor.cc
new file mode 100644
index 000000000..38dd70c8e
--- /dev/null
+++ b/src/libutil/file-descriptor.cc
@@ -0,0 +1,254 @@
+#include "file-system.hh"
+#include "signals.hh"
+#include "finally.hh"
+#include "serialise.hh"
+
+#include <fcntl.h>
+#include <unistd.h>
+
+namespace nix {
+
+std::string readFile(int fd)
+{
+    struct stat st;
+    if (fstat(fd, &st) == -1)
+        throw SysError("statting file");
+
+    return drainFD(fd, true, st.st_size);
+}
+
+
+void readFull(int fd, char * buf, size_t count)
+{
+    while (count) {
+        checkInterrupt();
+        ssize_t res = read(fd, buf, count);
+        if (res == -1) {
+            if (errno == EINTR) continue;
+            throw SysError("reading from file");
+        }
+        if (res == 0) throw EndOfFile("unexpected end-of-file");
+        count -= res;
+        buf += res;
+    }
+}
+
+
+void writeFull(int fd, std::string_view s, bool allowInterrupts)
+{
+    while (!s.empty()) {
+        if (allowInterrupts) checkInterrupt();
+        ssize_t res = write(fd, s.data(), s.size());
+        if (res == -1 && errno != EINTR)
+            throw SysError("writing to file");
+        if (res > 0)
+            s.remove_prefix(res);
+    }
+}
+
+
+std::string readLine(int fd)
+{
+    std::string s;
+    while (1) {
+        checkInterrupt();
+        char ch;
+        // FIXME: inefficient
+        ssize_t rd = read(fd, &ch, 1);
+        if (rd == -1) {
+            if (errno != EINTR)
+                throw SysError("reading a line");
+        } else if (rd == 0)
+            throw EndOfFile("unexpected EOF reading a line");
+        else {
+            if (ch == '\n') return s;
+            s += ch;
+        }
+    }
+}
+
+
+void writeLine(int fd, std::string s)
+{
+    s += '\n';
+    writeFull(fd, s);
+}
+
+
+std::string drainFD(int fd, bool block, const size_t reserveSize)
+{
+    // the parser needs two extra bytes to append terminating characters, other users will
+    // not care very much about the extra memory.
+    StringSink sink(reserveSize + 2);
+    drainFD(fd, sink, block);
+    return std::move(sink.s);
+}
+
+
+void drainFD(int fd, Sink & sink, bool block)
+{
+    // silence GCC maybe-uninitialized warning in finally
+    int saved = 0;
+
+    if (!block) {
+        saved = fcntl(fd, F_GETFL);
+        if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1)
+            throw SysError("making file descriptor non-blocking");
+    }
+
+    Finally finally([&]() {
+        if (!block) {
+            if (fcntl(fd, F_SETFL, saved) == -1)
+                throw SysError("making file descriptor blocking");
+        }
+    });
+
+    std::vector<unsigned char> buf(64 * 1024);
+    while (1) {
+        checkInterrupt();
+        ssize_t rd = read(fd, buf.data(), buf.size());
+        if (rd == -1) {
+            if (!block && (errno == EAGAIN || errno == EWOULDBLOCK))
+                break;
+            if (errno != EINTR)
+                throw SysError("reading from file");
+        }
+        else if (rd == 0) break;
+        else sink({(char *) buf.data(), (size_t) rd});
+    }
+}
+
+//////////////////////////////////////////////////////////////////////
+
+AutoCloseFD::AutoCloseFD() : fd{-1} {}
+
+
+AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {}
+
+
+AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd}
+{
+    that.fd = -1;
+}
+
+
+AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that)
+{
+    close();
+    fd = that.fd;
+    that.fd = -1;
+    return *this;
+}
+
+
+AutoCloseFD::~AutoCloseFD()
+{
+    try {
+        close();
+    } catch (...) {
+        ignoreException();
+    }
+}
+
+
+int AutoCloseFD::get() const
+{
+    return fd;
+}
+
+
+void AutoCloseFD::close()
+{
+    if (fd != -1) {
+        if (::close(fd) == -1)
+            /* This should never happen. */
+            throw SysError("closing file descriptor %1%", fd);
+        fd = -1;
+    }
+}
+
+void AutoCloseFD::fsync()
+{
+  if (fd != -1) {
+      int result;
+#if __APPLE__
+      result = ::fcntl(fd, F_FULLFSYNC);
+#else
+      result = ::fsync(fd);
+#endif
+      if (result == -1)
+          throw SysError("fsync file descriptor %1%", fd);
+  }
+}
+
+
+AutoCloseFD::operator bool() const
+{
+    return fd != -1;
+}
+
+
+int AutoCloseFD::release()
+{
+    int oldFD = fd;
+    fd = -1;
+    return oldFD;
+}
+
+
+void Pipe::create()
+{
+    int fds[2];
+#if HAVE_PIPE2
+    if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe");
+#else
+    if (pipe(fds) != 0) throw SysError("creating pipe");
+    closeOnExec(fds[0]);
+    closeOnExec(fds[1]);
+#endif
+    readSide = fds[0];
+    writeSide = fds[1];
+}
+
+
+void Pipe::close()
+{
+    readSide.close();
+    writeSide.close();
+}
+
+//////////////////////////////////////////////////////////////////////
+
+void closeMostFDs(const std::set<int> & exceptions)
+{
+#if __linux__
+    try {
+        for (auto & s : readDirectory("/proc/self/fd")) {
+            auto fd = std::stoi(s.name);
+            if (!exceptions.count(fd)) {
+                debug("closing leaked FD %d", fd);
+                close(fd);
+            }
+        }
+        return;
+    } catch (SysError &) {
+    }
+#endif
+
+    int maxFD = 0;
+    maxFD = sysconf(_SC_OPEN_MAX);
+    for (int fd = 0; fd < maxFD; ++fd)
+        if (!exceptions.count(fd))
+            close(fd); /* ignore result */
+}
+
+
+void closeOnExec(int fd)
+{
+    int prev;
+    if ((prev = fcntl(fd, F_GETFD, 0)) == -1 ||
+        fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1)
+        throw SysError("setting close-on-exec flag");
+}
+
+}
diff --git a/src/libutil/file-descriptor.hh b/src/libutil/file-descriptor.hh
new file mode 100644
index 000000000..80ec86135
--- /dev/null
+++ b/src/libutil/file-descriptor.hh
@@ -0,0 +1,84 @@
+#pragma once
+///@file
+
+#include "types.hh"
+#include "error.hh"
+
+namespace nix {
+
+struct Sink;
+struct Source;
+
+/**
+ * Read the contents of a resource into a string.
+ */
+std::string readFile(int fd);
+
+/**
+ * Wrappers arount read()/write() that read/write exactly the
+ * requested number of bytes.
+ */
+void readFull(int fd, char * buf, size_t count);
+
+void writeFull(int fd, std::string_view s, bool allowInterrupts = true);
+
+/**
+ * Read a line from a file descriptor.
+ */
+std::string readLine(int fd);
+
+/**
+ * Write a line to a file descriptor.
+ */
+void writeLine(int fd, std::string s);
+
+/**
+ * Read a file descriptor until EOF occurs.
+ */
+std::string drainFD(int fd, bool block = true, const size_t reserveSize=0);
+
+void drainFD(int fd, Sink & sink, bool block = true);
+
+/**
+ * Automatic cleanup of resources.
+ */
+class AutoCloseFD
+{
+    int fd;
+public:
+    AutoCloseFD();
+    AutoCloseFD(int fd);
+    AutoCloseFD(const AutoCloseFD & fd) = delete;
+    AutoCloseFD(AutoCloseFD&& fd);
+    ~AutoCloseFD();
+    AutoCloseFD& operator =(const AutoCloseFD & fd) = delete;
+    AutoCloseFD& operator =(AutoCloseFD&& fd);
+    int get() const;
+    explicit operator bool() const;
+    int release();
+    void close();
+    void fsync();
+};
+
+class Pipe
+{
+public:
+    AutoCloseFD readSide, writeSide;
+    void create();
+    void close();
+};
+
+/**
+ * Close all file descriptors except those listed in the given set.
+ * Good practice in child processes.
+ */
+void closeMostFDs(const std::set<int> & exceptions);
+
+/**
+ * Set the close-on-exec flag for the given file descriptor.
+ */
+void closeOnExec(int fd);
+
+MakeError(EndOfFile, Error);
+
+}
diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc
new file mode 100644
index 000000000..c96effff9
--- /dev/null
+++ b/src/libutil/file-system.cc
@@ -0,0 +1,647 @@
+#include "environment-variables.hh"
+#include "file-system.hh"
+#include "signals.hh"
+#include "finally.hh"
+#include "serialise.hh"
+
+#include <atomic>
+#include <cerrno>
+#include <climits>
+#include <cstdio>
+#include <cstdlib>
+#include <sstream>
+#include <filesystem>
+
+#include <fcntl.h>
+#include <sys/types.h>
+#include <sys/time.h>
+#include <unistd.h>
+
+namespace fs = std::filesystem;
+
+namespace nix {
+
+Path absPath(Path path, std::optional<PathView> dir, bool resolveSymlinks)
+{
+    if (path[0] != '/') {
+        if (!dir) {
+#ifdef __GNU__
+            /* GNU (aka. GNU/Hurd) doesn't have any limitation on path
+               lengths and doesn't define `PATH_MAX'.  */
+            char *buf = getcwd(NULL, 0);
+            if (buf == NULL)
+#else
+            char buf[PATH_MAX];
+            if (!getcwd(buf, sizeof(buf)))
+#endif
+                throw SysError("cannot get cwd");
+            path = concatStrings(buf, "/", path);
+#ifdef __GNU__
+            free(buf);
+#endif
+        } else
+            path = concatStrings(*dir, "/", path);
+    }
+    return canonPath(path, resolveSymlinks);
+}
+
+
+Path canonPath(PathView path, bool resolveSymlinks)
+{
+    assert(path != "");
+
+    std::string s;
+    s.reserve(256);
+
+    if (path[0] != '/')
+        throw Error("not an absolute path: '%1%'", path);
+
+    std::string temp;
+
+    /* Count the number of times we follow a symlink and stop at some
+       arbitrary (but high) limit to prevent infinite loops. */
+    unsigned int followCount = 0, maxFollow = 1024;
+
+    while (1) {
+
+        /* Skip slashes. */
+        while (!path.empty() && path[0] == '/') path.remove_prefix(1);
+        if (path.empty()) break;
+
+        /* Ignore `.'. */
+        if (path == "." || path.substr(0, 2) == "./")
+            path.remove_prefix(1);
+
+        /* If `..', delete the last component. */
+        else if (path == ".." || path.substr(0, 3) == "../")
+        {
+            if (!s.empty()) s.erase(s.rfind('/'));
+            path.remove_prefix(2);
+        }
+
+        /* Normal component; copy it. */
+        else {
+            s += '/';
+            if (const auto slash = path.find('/'); slash == std::string::npos) {
+                s += path;
+                path = {};
+            } else {
+                s += path.substr(0, slash);
+                path = path.substr(slash);
+            }
+
+            /* If s points to a symlink, resolve it and continue from there */
+            if (resolveSymlinks && isLink(s)) {
+                if (++followCount >= maxFollow)
+                    throw Error("infinite symlink recursion in path '%1%'", path);
+                temp = concatStrings(readLink(s), path);
+                path = temp;
+                if (!temp.empty() && temp[0] == '/') {
+                    s.clear();  /* restart for symlinks pointing to absolute path */
+                } else {
+                    s = dirOf(s);
+                    if (s == "/") {  // we don’t want trailing slashes here, which dirOf only produces if s = /
+                        s.clear();
+                    }
+                }
+            }
+        }
+    }
+
+    return s.empty() ? "/" : std::move(s);
+}
+
+
+Path dirOf(const PathView path)
+{
+    Path::size_type pos = path.rfind('/');
+    if (pos == std::string::npos)
+        return ".";
+    return pos == 0 ? "/" : Path(path, 0, pos);
+}
+
+
+std::string_view baseNameOf(std::string_view path)
+{
+    if (path.empty())
+        return "";
+
+    auto last = path.size() - 1;
+    if (path[last] == '/' && last > 0)
+        last -= 1;
+
+    auto pos = path.rfind('/', last);
+    if (pos == std::string::npos)
+        pos = 0;
+    else
+        pos += 1;
+
+    return path.substr(pos, last - pos + 1);
+}
+
+
+bool isInDir(std::string_view path, std::string_view dir)
+{
+    return path.substr(0, 1) == "/"
+        && path.substr(0, dir.size()) == dir
+        && path.size() >= dir.size() + 2
+        && path[dir.size()] == '/';
+}
+
+
+bool isDirOrInDir(std::string_view path, std::string_view dir)
+{
+    return path == dir || isInDir(path, dir);
+}
+
+
+struct stat stat(const Path & path)
+{
+    struct stat st;
+    if (stat(path.c_str(), &st))
+        throw SysError("getting status of '%1%'", path);
+    return st;
+}
+
+
+struct stat lstat(const Path & path)
+{
+    struct stat st;
+    if (lstat(path.c_str(), &st))
+        throw SysError("getting status of '%1%'", path);
+    return st;
+}
+
+
+bool pathExists(const Path & path)
+{
+    int res;
+    struct stat st;
+    res = lstat(path.c_str(), &st);
+    if (!res) return true;
+    if (errno != ENOENT && errno != ENOTDIR)
+        throw SysError("getting status of %1%", path);
+    return false;
+}
+
+bool pathAccessible(const Path & path)
+{
+    try {
+        return pathExists(path);
+    } catch (SysError & e) {
+        // swallow EPERM
+        if (e.errNo == EPERM) return false;
+        throw;
+    }
+}
+
+
+Path readLink(const Path & path)
+{
+    checkInterrupt();
+    std::vector<char> buf;
+    for (ssize_t bufSize = PATH_MAX/4; true; bufSize += bufSize/2) {
+        buf.resize(bufSize);
+        ssize_t rlSize = readlink(path.c_str(), buf.data(), bufSize);
+        if (rlSize == -1)
+            if (errno == EINVAL)
+                throw Error("'%1%' is not a symlink", path);
+            else
+                throw SysError("reading symbolic link '%1%'", path);
+        else if (rlSize < bufSize)
+            return std::string(buf.data(), rlSize);
+    }
+}
+
+
+bool isLink(const Path & path)
+{
+    struct stat st = lstat(path);
+    return S_ISLNK(st.st_mode);
+}
+
+
+DirEntries readDirectory(DIR *dir, const Path & path)
+{
+    DirEntries entries;
+    entries.reserve(64);
+
+    struct dirent * dirent;
+    while (errno = 0, dirent = readdir(dir)) { /* sic */
+        checkInterrupt();
+        std::string name = dirent->d_name;
+        if (name == "." || name == "..") continue;
+        entries.emplace_back(name, dirent->d_ino,
+#ifdef HAVE_STRUCT_DIRENT_D_TYPE
+            dirent->d_type
+#else
+            DT_UNKNOWN
+#endif
+        );
+    }
+    if (errno) throw SysError("reading directory '%1%'", path);
+
+    return entries;
+}
+
+DirEntries readDirectory(const Path & path)
+{
+    AutoCloseDir dir(opendir(path.c_str()));
+    if (!dir) throw SysError("opening directory '%1%'", path);
+
+    return readDirectory(dir.get(), path);
+}
+
+
+unsigned char getFileType(const Path & path)
+{
+    struct stat st = lstat(path);
+    if (S_ISDIR(st.st_mode)) return DT_DIR;
+    if (S_ISLNK(st.st_mode)) return DT_LNK;
+    if (S_ISREG(st.st_mode)) return DT_REG;
+    return DT_UNKNOWN;
+}
+
+
+std::string readFile(const Path & path)
+{
+    AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
+    if (!fd)
+        throw SysError("opening file '%1%'", path);
+    return readFile(fd.get());
+}
+
+
+void readFile(const Path & path, Sink & sink)
+{
+    AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
+    if (!fd)
+        throw SysError("opening file '%s'", path);
+    drainFD(fd.get(), sink);
+}
+
+
+void writeFile(const Path & path, std::string_view s, mode_t mode, bool sync)
+{
+    AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode);
+    if (!fd)
+        throw SysError("opening file '%1%'", path);
+    try {
+        writeFull(fd.get(), s);
+    } catch (Error & e) {
+        e.addTrace({}, "writing file '%1%'", path);
+        throw;
+    }
+    if (sync)
+        fd.fsync();
+    // Explicitly close to make sure exceptions are propagated.
+    fd.close();
+    if (sync)
+        syncParent(path);
+}
+
+
+void writeFile(const Path & path, Source & source, mode_t mode, bool sync)
+{
+    AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode);
+    if (!fd)
+        throw SysError("opening file '%1%'", path);
+
+    std::vector<char> buf(64 * 1024);
+
+    try {
+        while (true) {
+            try {
+                auto n = source.read(buf.data(), buf.size());
+                writeFull(fd.get(), {buf.data(), n});
+            } catch (EndOfFile &) { break; }
+        }
+    } catch (Error & e) {
+        e.addTrace({}, "writing file '%1%'", path);
+        throw;
+    }
+    if (sync)
+        fd.fsync();
+    // Explicitly close to make sure exceptions are propagated.
+    fd.close();
+    if (sync)
+        syncParent(path);
+}
+
+void syncParent(const Path & path)
+{
+    AutoCloseFD fd = open(dirOf(path).c_str(), O_RDONLY, 0);
+    if (!fd)
+        throw SysError("opening file '%1%'", path);
+    fd.fsync();
+}
+
+
+static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed)
+{
+    checkInterrupt();
+
+    std::string name(baseNameOf(path));
+
+    struct stat st;
+    if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) {
+        if (errno == ENOENT) return;
+        throw SysError("getting status of '%1%'", path);
+    }
+
+    if (!S_ISDIR(st.st_mode)) {
+        /* We are about to delete a file. Will it likely free space? */
+
+        switch (st.st_nlink) {
+            /* Yes: last link. */
+            case 1:
+                bytesFreed += st.st_size;
+                break;
+            /* Maybe: yes, if 'auto-optimise-store' or manual optimisation
+               was performed. Instead of checking for real let's assume
+               it's an optimised file and space will be freed.
+
+               In worst case we will double count on freed space for files
+               with exactly two hardlinks for unoptimised packages.
+             */
+            case 2:
+                bytesFreed += st.st_size;
+                break;
+            /* No: 3+ links. */
+            default:
+                break;
+        }
+    }
+
+    if (S_ISDIR(st.st_mode)) {
+        /* Make the directory accessible. */
+        const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR;
+        if ((st.st_mode & PERM_MASK) != PERM_MASK) {
+            if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1)
+                throw SysError("chmod '%1%'", path);
+        }
+
+        int fd = openat(parentfd, path.c_str(), O_RDONLY);
+        if (fd == -1)
+            throw SysError("opening directory '%1%'", path);
+        AutoCloseDir dir(fdopendir(fd));
+        if (!dir)
+            throw SysError("opening directory '%1%'", path);
+        for (auto & i : readDirectory(dir.get(), path))
+            _deletePath(dirfd(dir.get()), path + "/" + i.name, bytesFreed);
+    }
+
+    int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0;
+    if (unlinkat(parentfd, name.c_str(), flags) == -1) {
+        if (errno == ENOENT) return;
+        throw SysError("cannot unlink '%1%'", path);
+    }
+}
+
+static void _deletePath(const Path & path, uint64_t & bytesFreed)
+{
+    Path dir = dirOf(path);
+    if (dir == "")
+        dir = "/";
+
+    AutoCloseFD dirfd{open(dir.c_str(), O_RDONLY)};
+    if (!dirfd) {
+        if (errno == ENOENT) return;
+        throw SysError("opening directory '%1%'", path);
+    }
+
+    _deletePath(dirfd.get(), path, bytesFreed);
+}
+
+
+void deletePath(const Path & path)
+{
+    uint64_t dummy;
+    deletePath(path, dummy);
+}
+
+
+Paths createDirs(const Path & path)
+{
+    Paths created;
+    if (path == "/") return created;
+
+    struct stat st;
+    if (lstat(path.c_str(), &st) == -1) {
+        created = createDirs(dirOf(path));
+        if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST)
+            throw SysError("creating directory '%1%'", path);
+        st = lstat(path);
+        created.push_back(path);
+    }
+
+    if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1)
+        throw SysError("statting symlink '%1%'", path);
+
+    if (!S_ISDIR(st.st_mode)) throw Error("'%1%' is not a directory", path);
+
+    return created;
+}
+
+
+void deletePath(const Path & path, uint64_t & bytesFreed)
+{
+    //Activity act(*logger, lvlDebug, "recursively deleting path '%1%'", path);
+    bytesFreed = 0;
+    _deletePath(path, bytesFreed);
+}
+
+
+//////////////////////////////////////////////////////////////////////
+
+AutoDelete::AutoDelete() : del{false} {}
+
+AutoDelete::AutoDelete(const std::string & p, bool recursive) : path(p)
+{
+    del = true;
+    this->recursive = recursive;
+}
+
+AutoDelete::~AutoDelete()
+{
+    try {
+        if (del) {
+            if (recursive)
+                deletePath(path);
+            else {
+                if (remove(path.c_str()) == -1)
+                    throw SysError("cannot unlink '%1%'", path);
+            }
+        }
+    } catch (...) {
+        ignoreException();
+    }
+}
+
+void AutoDelete::cancel()
+{
+    del = false;
+}
+
+void AutoDelete::reset(const Path & p, bool recursive) {
+    path = p;
+    this->recursive = recursive;
+    del = true;
+}
+
+//////////////////////////////////////////////////////////////////////
+
+//////////////////////////////////////////////////////////////////////
+
+static Path tempName(Path tmpRoot, const Path & prefix, bool includePid,
+    std::atomic<unsigned int> & counter)
+{
+    tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true);
+    if (includePid)
+        return fmt("%1%/%2%-%3%-%4%", tmpRoot, prefix, getpid(), counter++);
+    else
+        return fmt("%1%/%2%-%3%", tmpRoot, prefix, counter++);
+}
+
+Path createTempDir(const Path & tmpRoot, const Path & prefix,
+    bool includePid, bool useGlobalCounter, mode_t mode)
+{
+    static std::atomic<unsigned int> globalCounter = 0;
+    std::atomic<unsigned int> localCounter = 0;
+    auto & counter(useGlobalCounter ? globalCounter : localCounter);
+
+    while (1) {
+        checkInterrupt();
+        Path tmpDir = tempName(tmpRoot, prefix, includePid, counter);
+        if (mkdir(tmpDir.c_str(), mode) == 0) {
+#if __FreeBSD__
+            /* Explicitly set the group of the directory.  This is to
+               work around around problems caused by BSD's group
+               ownership semantics (directories inherit the group of
+               the parent).  For instance, the group of /tmp on
+               FreeBSD is "wheel", so all directories created in /tmp
+               will be owned by "wheel"; but if the user is not in
+               "wheel", then "tar" will fail to unpack archives that
+               have the setgid bit set on directories. */
+            if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0)
+                throw SysError("setting group of directory '%1%'", tmpDir);
+#endif
+            return tmpDir;
+        }
+        if (errno != EEXIST)
+            throw SysError("creating directory '%1%'", tmpDir);
+    }
+}
+
+
+std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix)
+{
+    Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX");
+    // Strictly speaking, this is UB, but who cares...
+    // FIXME: use O_TMPFILE.
+    AutoCloseFD fd(mkstemp((char *) tmpl.c_str()));
+    if (!fd)
+        throw SysError("creating temporary file '%s'", tmpl);
+    closeOnExec(fd.get());
+    return {std::move(fd), tmpl};
+}
+
+void createSymlink(const Path & target, const Path & link)
+{
+    if (symlink(target.c_str(), link.c_str()))
+        throw SysError("creating symlink from '%1%' to '%2%'", link, target);
+}
+
+void replaceSymlink(const Path & target, const Path & link)
+{
+    for (unsigned int n = 0; true; n++) {
+        Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link)));
+
+        try {
+            createSymlink(target, tmp);
+        } catch (SysError & e) {
+            if (e.errNo == EEXIST) continue;
+            throw;
+        }
+
+        renameFile(tmp, link);
+
+        break;
+    }
+}
+
+void setWriteTime(const fs::path & p, const struct stat & st)
+{
+    struct timeval times[2];
+    times[0] = {
+        .tv_sec = st.st_atime,
+        .tv_usec = 0,
+    };
+    times[1] = {
+        .tv_sec = st.st_mtime,
+        .tv_usec = 0,
+    };
+    if (lutimes(p.c_str(), times) != 0)
+        throw SysError("changing modification time of '%s'", p);
+}
+
+void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete)
+{
+    // TODO: Rewrite the `is_*` to use `symlink_status()`
+    auto statOfFrom = lstat(from.path().c_str());
+    auto fromStatus = from.symlink_status();
+
+    // Mark the directory as writable so that we can delete its children
+    if (andDelete && fs::is_directory(fromStatus)) {
+        fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow);
+    }
+
+
+    if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) {
+        fs::copy(from.path(), to, fs::copy_options::copy_symlinks | fs::copy_options::overwrite_existing);
+    } else if (fs::is_directory(fromStatus)) {
+        fs::create_directory(to);
+        for (auto & entry : fs::directory_iterator(from.path())) {
+            copy(entry, to / entry.path().filename(), andDelete);
+        }
+    } else {
+        throw Error("file '%s' has an unsupported type", from.path());
+    }
+
+    setWriteTime(to, statOfFrom);
+    if (andDelete) {
+        if (!fs::is_symlink(fromStatus))
+            fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow);
+        fs::remove(from.path());
+    }
+}
+
+void renameFile(const Path & oldName, const Path & newName)
+{
+    fs::rename(oldName, newName);
+}
+
+void moveFile(const Path & oldName, const Path & newName)
+{
+    try {
+        renameFile(oldName, newName);
+    } catch (fs::filesystem_error & e) {
+        auto oldPath = fs::path(oldName);
+        auto newPath = fs::path(newName);
+        // For the move to be as atomic as possible, copy to a temporary
+        // directory
+        fs::path temp = createTempDir(newPath.parent_path(), "rename-tmp");
+        Finally removeTemp = [&]() { fs::remove(temp); };
+        auto tempCopyTarget = temp / "copy-target";
+        if (e.code().value() == EXDEV) {
+            fs::remove(newPath);
+            warn("Can’t rename %s as %s, copying instead", oldName, newName);
+            copy(fs::directory_entry(oldPath), tempCopyTarget, true);
+            renameFile(tempCopyTarget, newPath);
+        }
+    }
+}
+
+//////////////////////////////////////////////////////////////////////
+
+}
diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh
new file mode 100644
index 000000000..4637507b3
--- /dev/null
+++ b/src/libutil/file-system.hh
@@ -0,0 +1,238 @@
+#pragma once
+/**
+ * @file
+ *
+ * Utiltities for working with the file sytem and file paths.
+ */
+
+#include "types.hh"
+#include "error.hh"
+#include "logging.hh"
+#include "file-descriptor.hh"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <signal.h>
+
+#include <boost/lexical_cast.hpp>
+
+#include <atomic>
+#include <functional>
+#include <map>
+#include <sstream>
+#include <optional>
+
+#ifndef HAVE_STRUCT_DIRENT_D_TYPE
+#define DT_UNKNOWN 0
+#define DT_REG 1
+#define DT_LNK 2
+#define DT_DIR 3
+#endif
+
+namespace nix {
+
+struct Sink;
+struct Source;
+
+/**
+ * @return An absolutized path, resolving paths relative to the
+ * specified directory, or the current directory otherwise.  The path
+ * is also canonicalised.
+ */
+Path absPath(Path path,
+    std::optional<PathView> dir = {},
+    bool resolveSymlinks = false);
+
+/**
+ * Canonicalise a path by removing all `.` or `..` components and
+ * double or trailing slashes.  Optionally resolves all symlink
+ * components such that each component of the resulting path is *not*
+ * a symbolic link.
+ */
+Path canonPath(PathView path, bool resolveSymlinks = false);
+
+/**
+ * @return The directory part of the given canonical path, i.e.,
+ * everything before the final `/`.  If the path is the root or an
+ * immediate child thereof (e.g., `/foo`), this means `/`
+ * is returned.
+ */
+Path dirOf(const PathView path);
+
+/**
+ * @return the base name of the given canonical path, i.e., everything
+ * following the final `/` (trailing slashes are removed).
+ */
+std::string_view baseNameOf(std::string_view path);
+
+/**
+ * Check whether 'path' is a descendant of 'dir'. Both paths must be
+ * canonicalized.
+ */
+bool isInDir(std::string_view path, std::string_view dir);
+
+/**
+ * Check whether 'path' is equal to 'dir' or a descendant of
+ * 'dir'. Both paths must be canonicalized.
+ */
+bool isDirOrInDir(std::string_view path, std::string_view dir);
+
+/**
+ * Get status of `path`.
+ */
+struct stat stat(const Path & path);
+struct stat lstat(const Path & path);
+
+/**
+ * @return true iff the given path exists.
+ */
+bool pathExists(const Path & path);
+
+/**
+ * A version of pathExists that returns false on a permission error.
+ * Useful for inferring default paths across directories that might not
+ * be readable.
+ * @return true iff the given path can be accessed and exists
+ */
+bool pathAccessible(const Path & path);
+
+/**
+ * Read the contents (target) of a symbolic link.  The result is not
+ * in any way canonicalised.
+ */
+Path readLink(const Path & path);
+
+bool isLink(const Path & path);
+
+/**
+ * Read the contents of a directory.  The entries `.` and `..` are
+ * removed.
+ */
+struct DirEntry
+{
+    std::string name;
+    ino_t ino;
+    /**
+     * one of DT_*
+     */
+    unsigned char type;
+    DirEntry(std::string name, ino_t ino, unsigned char type)
+        : name(std::move(name)), ino(ino), type(type) { }
+};
+
+typedef std::vector<DirEntry> DirEntries;
+
+DirEntries readDirectory(const Path & path);
+
+unsigned char getFileType(const Path & path);
+
+/**
+ * Read the contents of a file into a string.
+ */
+std::string readFile(const Path & path);
+void readFile(const Path & path, Sink & sink);
+
+/**
+ * Write a string to a file.
+ */
+void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false);
+
+void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false);
+
+/**
+ * Flush a file's parent directory to disk
+ */
+void syncParent(const Path & path);
+
+/**
+ * Delete a path; i.e., in the case of a directory, it is deleted
+ * recursively. It's not an error if the path does not exist. The
+ * second variant returns the number of bytes and blocks freed.
+ */
+void deletePath(const Path & path);
+
+void deletePath(const Path & path, uint64_t & bytesFreed);
+
+/**
+ * Create a directory and all its parents, if necessary.  Returns the
+ * list of created directories, in order of creation.
+ */
+Paths createDirs(const Path & path);
+inline Paths createDirs(PathView path)
+{
+    return createDirs(Path(path));
+}
+
+/**
+ * Create a symlink.
+ */
+void createSymlink(const Path & target, const Path & link);
+
+/**
+ * Atomically create or replace a symlink.
+ */
+void replaceSymlink(const Path & target, const Path & link);
+
+void renameFile(const Path & src, const Path & dst);
+
+/**
+ * Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst`
+ * are on a different filesystem.
+ *
+ * Beware that this might not be atomic because of the copy that happens behind
+ * the scenes
+ */
+void moveFile(const Path & src, const Path & dst);
+
+
+/**
+ * Automatic cleanup of resources.
+ */
+class AutoDelete
+{
+    Path path;
+    bool del;
+    bool recursive;
+public:
+    AutoDelete();
+    AutoDelete(const Path & p, bool recursive = true);
+    ~AutoDelete();
+    void cancel();
+    void reset(const Path & p, bool recursive = true);
+    operator Path() const { return path; }
+    operator PathView() const { return path; }
+};
+
+
+struct DIRDeleter
+{
+    void operator()(DIR * dir) const {
+        closedir(dir);
+    }
+};
+
+typedef std::unique_ptr<DIR, DIRDeleter> AutoCloseDir;
+
+
+/**
+ * Create a temporary directory.
+ */
+Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix",
+    bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755);
+
+/**
+ * Create a temporary file, returning a file handle and its path.
+ */
+std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix = "nix");
+
+
+/**
+ * Used in various places.
+ */
+typedef std::function<bool(const Path & path)> PathFilter;
+
+extern PathFilter defaultPathFilter;
+
+}
diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc
deleted file mode 100644
index 11cc0c0e7..000000000
--- a/src/libutil/filesystem.cc
+++ /dev/null
@@ -1,162 +0,0 @@
-#include <sys/time.h>
-#include <filesystem>
-#include <atomic>
-
-#include "finally.hh"
-#include "util.hh"
-#include "types.hh"
-
-namespace fs = std::filesystem;
-
-namespace nix {
-
-static Path tempName(Path tmpRoot, const Path & prefix, bool includePid,
-    std::atomic<unsigned int> & counter)
-{
-    tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true);
-    if (includePid)
-        return fmt("%1%/%2%-%3%-%4%", tmpRoot, prefix, getpid(), counter++);
-    else
-        return fmt("%1%/%2%-%3%", tmpRoot, prefix, counter++);
-}
-
-Path createTempDir(const Path & tmpRoot, const Path & prefix,
-    bool includePid, bool useGlobalCounter, mode_t mode)
-{
-    static std::atomic<unsigned int> globalCounter = 0;
-    std::atomic<unsigned int> localCounter = 0;
-    auto & counter(useGlobalCounter ? globalCounter : localCounter);
-
-    while (1) {
-        checkInterrupt();
-        Path tmpDir = tempName(tmpRoot, prefix, includePid, counter);
-        if (mkdir(tmpDir.c_str(), mode) == 0) {
-#if __FreeBSD__
-            /* Explicitly set the group of the directory.  This is to
-               work around around problems caused by BSD's group
-               ownership semantics (directories inherit the group of
-               the parent).  For instance, the group of /tmp on
-               FreeBSD is "wheel", so all directories created in /tmp
-               will be owned by "wheel"; but if the user is not in
-               "wheel", then "tar" will fail to unpack archives that
-               have the setgid bit set on directories. */
-            if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0)
-                throw SysError("setting group of directory '%1%'", tmpDir);
-#endif
-            return tmpDir;
-        }
-        if (errno != EEXIST)
-            throw SysError("creating directory '%1%'", tmpDir);
-    }
-}
-
-
-std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix)
-{
-    Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX");
-    // Strictly speaking, this is UB, but who cares...
-    // FIXME: use O_TMPFILE.
-    AutoCloseFD fd(mkstemp((char *) tmpl.c_str()));
-    if (!fd)
-        throw SysError("creating temporary file '%s'", tmpl);
-    closeOnExec(fd.get());
-    return {std::move(fd), tmpl};
-}
-
-void createSymlink(const Path & target, const Path & link)
-{
-    if (symlink(target.c_str(), link.c_str()))
-        throw SysError("creating symlink from '%1%' to '%2%'", link, target);
-}
-
-void replaceSymlink(const Path & target, const Path & link)
-{
-    for (unsigned int n = 0; true; n++) {
-        Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link)));
-
-        try {
-            createSymlink(target, tmp);
-        } catch (SysError & e) {
-            if (e.errNo == EEXIST) continue;
-            throw;
-        }
-
-        renameFile(tmp, link);
-
-        break;
-    }
-}
-
-void setWriteTime(const fs::path & p, const struct stat & st)
-{
-    struct timeval times[2];
-    times[0] = {
-        .tv_sec = st.st_atime,
-        .tv_usec = 0,
-    };
-    times[1] = {
-        .tv_sec = st.st_mtime,
-        .tv_usec = 0,
-    };
-    if (lutimes(p.c_str(), times) != 0)
-        throw SysError("changing modification time of '%s'", p);
-}
-
-void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete)
-{
-    // TODO: Rewrite the `is_*` to use `symlink_status()`
-    auto statOfFrom = lstat(from.path().c_str());
-    auto fromStatus = from.symlink_status();
-
-    // Mark the directory as writable so that we can delete its children
-    if (andDelete && fs::is_directory(fromStatus)) {
-        fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow);
-    }
-
-
-    if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) {
-        fs::copy(from.path(), to, fs::copy_options::copy_symlinks | fs::copy_options::overwrite_existing);
-    } else if (fs::is_directory(fromStatus)) {
-        fs::create_directory(to);
-        for (auto & entry : fs::directory_iterator(from.path())) {
-            copy(entry, to / entry.path().filename(), andDelete);
-        }
-    } else {
-        throw Error("file '%s' has an unsupported type", from.path());
-    }
-
-    setWriteTime(to, statOfFrom);
-    if (andDelete) {
-        if (!fs::is_symlink(fromStatus))
-            fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow);
-        fs::remove(from.path());
-    }
-}
-
-void renameFile(const Path & oldName, const Path & newName)
-{
-    fs::rename(oldName, newName);
-}
-
-void moveFile(const Path & oldName, const Path & newName)
-{
-    try {
-        renameFile(oldName, newName);
-    } catch (fs::filesystem_error & e) {
-        auto oldPath = fs::path(oldName);
-        auto newPath = fs::path(newName);
-        // For the move to be as atomic as possible, copy to a temporary
-        // directory
-        fs::path temp = createTempDir(newPath.parent_path(), "rename-tmp");
-        Finally removeTemp = [&]() { fs::remove(temp); };
-        auto tempCopyTarget = temp / "copy-target";
-        if (e.code().value() == EXDEV) {
-            fs::remove(newPath);
-            warn("Can’t rename %s as %s, copying instead", oldName, newName);
-            copy(fs::directory_entry(oldPath), tempCopyTarget, true);
-            renameFile(tempCopyTarget, newPath);
-        }
-    }
-}
-
-}
diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh
index c22edd390..bf54b7301 100644
--- a/src/libutil/fs-sink.hh
+++ b/src/libutil/fs-sink.hh
@@ -4,6 +4,7 @@
 #include "types.hh"
 #include "serialise.hh"
 #include "source-accessor.hh"
+#include "file-system.hh"
 
 namespace nix {
 
diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index e297c245b..144f7ae7e 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -9,7 +9,6 @@
 #include "hash.hh"
 #include "archive.hh"
 #include "split.hh"
-#include "util.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh
index cab3e6eca..6ade6555c 100644
--- a/src/libutil/hash.hh
+++ b/src/libutil/hash.hh
@@ -3,6 +3,7 @@
 
 #include "types.hh"
 #include "serialise.hh"
+#include "file-system.hh"
 
 
 namespace nix {
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index 9d7a141b3..60b0865bf 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -1,4 +1,7 @@
 #include "logging.hh"
+#include "file-descriptor.hh"
+#include "environment-variables.hh"
+#include "terminal.hh"
 #include "util.hh"
 #include "config.hh"
 
diff --git a/src/libutil/monitor-fd.hh b/src/libutil/monitor-fd.hh
index 86d0115fc..228fb13f8 100644
--- a/src/libutil/monitor-fd.hh
+++ b/src/libutil/monitor-fd.hh
@@ -10,6 +10,8 @@
 #include <unistd.h>
 #include <signal.h>
 
+#include "signals.hh"
+
 namespace nix {
 
 
diff --git a/src/libutil/namespaces.cc b/src/libutil/namespaces.cc
index f66accb10..a789b321e 100644
--- a/src/libutil/namespaces.cc
+++ b/src/libutil/namespaces.cc
@@ -1,13 +1,22 @@
-#if __linux__
-
-#include "namespaces.hh"
+#include "current-process.hh"
 #include "util.hh"
 #include "finally.hh"
+#include "file-system.hh"
+#include "processes.hh"
+#include "signals.hh"
+
+#if __linux__
+# include <mutex>
+# include <sys/resource.h>
+# include "cgroup.hh"
+#endif
 
 #include <sys/mount.h>
 
 namespace nix {
 
+#if __linux__
+
 bool userNamespacesSupported()
 {
     static auto res = [&]() -> bool
@@ -92,6 +101,60 @@ bool mountAndPidNamespacesSupported()
     return res;
 }
 
+#endif
+
+
+//////////////////////////////////////////////////////////////////////
+
+#if __linux__
+static AutoCloseFD fdSavedMountNamespace;
+static AutoCloseFD fdSavedRoot;
+#endif
+
+void saveMountNamespace()
+{
+#if __linux__
+    static std::once_flag done;
+    std::call_once(done, []() {
+        fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY);
+        if (!fdSavedMountNamespace)
+            throw SysError("saving parent mount namespace");
+
+        fdSavedRoot = open("/proc/self/root", O_RDONLY);
+    });
+#endif
 }
 
+void restoreMountNamespace()
+{
+#if __linux__
+    try {
+        auto savedCwd = absPath(".");
+
+        if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1)
+            throw SysError("restoring parent mount namespace");
+
+        if (fdSavedRoot) {
+            if (fchdir(fdSavedRoot.get()))
+                throw SysError("chdir into saved root");
+            if (chroot("."))
+                throw SysError("chroot into saved root");
+        }
+
+        if (chdir(savedCwd.c_str()) == -1)
+            throw SysError("restoring cwd");
+    } catch (Error & e) {
+        debug(e.msg());
+    }
 #endif
+}
+
+void unshareFilesystem()
+{
+#ifdef __linux__
+    if (unshare(CLONE_FS) != 0 && errno != EPERM)
+        throw SysError("unsharing filesystem state in download thread");
+#endif
+}
+
+}
diff --git a/src/libutil/namespaces.hh b/src/libutil/namespaces.hh
index 0b7eeb66c..7e4e921a8 100644
--- a/src/libutil/namespaces.hh
+++ b/src/libutil/namespaces.hh
@@ -1,8 +1,31 @@
 #pragma once
 ///@file
 
+#include <optional>
+
+#include "types.hh"
+
 namespace nix {
 
+/**
+ * Save the current mount namespace. Ignored if called more than
+ * once.
+ */
+void saveMountNamespace();
+
+/**
+ * Restore the mount namespace saved by saveMountNamespace(). Ignored
+ * if saveMountNamespace() was never called.
+ */
+void restoreMountNamespace();
+
+/**
+ * Cause this thread to not share any FS attributes with the main
+ * thread, because this causes setns() in restoreMountNamespace() to
+ * fail.
+ */
+void unshareFilesystem();
+
 #if __linux__
 
 bool userNamespacesSupported();
diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc
index d5e32d989..dc96f84e5 100644
--- a/src/libutil/posix-source-accessor.cc
+++ b/src/libutil/posix-source-accessor.cc
@@ -1,4 +1,5 @@
 #include "posix-source-accessor.hh"
+#include "signals.hh"
 
 namespace nix {
 
diff --git a/src/libutil/processes.cc b/src/libutil/processes.cc
new file mode 100644
index 000000000..91a0ea66f
--- /dev/null
+++ b/src/libutil/processes.cc
@@ -0,0 +1,421 @@
+#include "current-process.hh"
+#include "environment-variables.hh"
+#include "signals.hh"
+#include "processes.hh"
+#include "finally.hh"
+#include "serialise.hh"
+
+#include <cerrno>
+#include <cstdlib>
+#include <cstring>
+#include <future>
+#include <iostream>
+#include <sstream>
+#include <thread>
+
+#include <grp.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#ifdef __APPLE__
+# include <sys/syscall.h>
+#endif
+
+#ifdef __linux__
+# include <sys/prctl.h>
+# include <sys/mman.h>
+#endif
+
+
+namespace nix {
+
+Pid::Pid()
+{
+}
+
+
+Pid::Pid(pid_t pid)
+    : pid(pid)
+{
+}
+
+
+Pid::~Pid()
+{
+    if (pid != -1) kill();
+}
+
+
+void Pid::operator =(pid_t pid)
+{
+    if (this->pid != -1 && this->pid != pid) kill();
+    this->pid = pid;
+    killSignal = SIGKILL; // reset signal to default
+}
+
+
+Pid::operator pid_t()
+{
+    return pid;
+}
+
+
+int Pid::kill()
+{
+    assert(pid != -1);
+
+    debug("killing process %1%", pid);
+
+    /* Send the requested signal to the child.  If it has its own
+       process group, send the signal to every process in the child
+       process group (which hopefully includes *all* its children). */
+    if (::kill(separatePG ? -pid : pid, killSignal) != 0) {
+        /* On BSDs, killing a process group will return EPERM if all
+           processes in the group are zombies (or something like
+           that). So try to detect and ignore that situation. */
+#if __FreeBSD__ || __APPLE__
+        if (errno != EPERM || ::kill(pid, 0) != 0)
+#endif
+            logError(SysError("killing process %d", pid).info());
+    }
+
+    return wait();
+}
+
+
+int Pid::wait()
+{
+    assert(pid != -1);
+    while (1) {
+        int status;
+        int res = waitpid(pid, &status, 0);
+        if (res == pid) {
+            pid = -1;
+            return status;
+        }
+        if (errno != EINTR)
+            throw SysError("cannot get exit status of PID %d", pid);
+        checkInterrupt();
+    }
+}
+
+
+void Pid::setSeparatePG(bool separatePG)
+{
+    this->separatePG = separatePG;
+}
+
+
+void Pid::setKillSignal(int signal)
+{
+    this->killSignal = signal;
+}
+
+
+pid_t Pid::release()
+{
+    pid_t p = pid;
+    pid = -1;
+    return p;
+}
+
+
+void killUser(uid_t uid)
+{
+    debug("killing all processes running under uid '%1%'", uid);
+
+    assert(uid != 0); /* just to be safe... */
+
+    /* The system call kill(-1, sig) sends the signal `sig' to all
+       users to which the current process can send signals.  So we
+       fork a process, switch to uid, and send a mass kill. */
+
+    Pid pid = startProcess([&]() {
+
+        if (setuid(uid) == -1)
+            throw SysError("setting uid");
+
+        while (true) {
+#ifdef __APPLE__
+            /* OSX's kill syscall takes a third parameter that, among
+               other things, determines if kill(-1, signo) affects the
+               calling process. In the OSX libc, it's set to true,
+               which means "follow POSIX", which we don't want here
+                 */
+            if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break;
+#else
+            if (kill(-1, SIGKILL) == 0) break;
+#endif
+            if (errno == ESRCH || errno == EPERM) break; /* no more processes */
+            if (errno != EINTR)
+                throw SysError("cannot kill processes for uid '%1%'", uid);
+        }
+
+        _exit(0);
+    });
+
+    int status = pid.wait();
+    if (status != 0)
+        throw Error("cannot kill processes for uid '%1%': %2%", uid, statusToString(status));
+
+    /* !!! We should really do some check to make sure that there are
+       no processes left running under `uid', but there is no portable
+       way to do so (I think).  The most reliable way may be `ps -eo
+       uid | grep -q $uid'. */
+}
+
+
+//////////////////////////////////////////////////////////////////////
+
+
+/* Wrapper around vfork to prevent the child process from clobbering
+   the caller's stack frame in the parent. */
+static pid_t doFork(bool allowVfork, std::function<void()> fun) __attribute__((noinline));
+static pid_t doFork(bool allowVfork, std::function<void()> fun)
+{
+#ifdef __linux__
+    pid_t pid = allowVfork ? vfork() : fork();
+#else
+    pid_t pid = fork();
+#endif
+    if (pid != 0) return pid;
+    fun();
+    abort();
+}
+
+
+#if __linux__
+static int childEntry(void * arg)
+{
+    auto main = (std::function<void()> *) arg;
+    (*main)();
+    return 1;
+}
+#endif
+
+
+pid_t startProcess(std::function<void()> fun, const ProcessOptions & options)
+{
+    std::function<void()> wrapper = [&]() {
+        if (!options.allowVfork)
+            logger = makeSimpleLogger();
+        try {
+#if __linux__
+            if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1)
+                throw SysError("setting death signal");
+#endif
+            fun();
+        } catch (std::exception & e) {
+            try {
+                std::cerr << options.errorPrefix << e.what() << "\n";
+            } catch (...) { }
+        } catch (...) { }
+        if (options.runExitHandlers)
+            exit(1);
+        else
+            _exit(1);
+    };
+
+    pid_t pid = -1;
+
+    if (options.cloneFlags) {
+        #ifdef __linux__
+        // Not supported, since then we don't know when to free the stack.
+        assert(!(options.cloneFlags & CLONE_VM));
+
+        size_t stackSize = 1 * 1024 * 1024;
+        auto stack = (char *) mmap(0, stackSize,
+            PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
+        if (stack == MAP_FAILED) throw SysError("allocating stack");
+
+        Finally freeStack([&]() { munmap(stack, stackSize); });
+
+        pid = clone(childEntry, stack + stackSize, options.cloneFlags | SIGCHLD, &wrapper);
+        #else
+        throw Error("clone flags are only supported on Linux");
+        #endif
+    } else
+        pid = doFork(options.allowVfork, wrapper);
+
+    if (pid == -1) throw SysError("unable to fork");
+
+    return pid;
+}
+
+
+std::string runProgram(Path program, bool searchPath, const Strings & args,
+    const std::optional<std::string> & input, bool isInteractive)
+{
+    auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input, .isInteractive = isInteractive});
+
+    if (!statusOk(res.first))
+        throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first));
+
+    return res.second;
+}
+
+// Output = error code + "standard out" output stream
+std::pair<int, std::string> runProgram(RunOptions && options)
+{
+    StringSink sink;
+    options.standardOut = &sink;
+
+    int status = 0;
+
+    try {
+        runProgram2(options);
+    } catch (ExecError & e) {
+        status = e.status;
+    }
+
+    return {status, std::move(sink.s)};
+}
+
+void runProgram2(const RunOptions & options)
+{
+    checkInterrupt();
+
+    assert(!(options.standardIn && options.input));
+
+    std::unique_ptr<Source> source_;
+    Source * source = options.standardIn;
+
+    if (options.input) {
+        source_ = std::make_unique<StringSource>(*options.input);
+        source = source_.get();
+    }
+
+    /* Create a pipe. */
+    Pipe out, in;
+    if (options.standardOut) out.create();
+    if (source) in.create();
+
+    ProcessOptions processOptions;
+    // vfork implies that the environment of the main process and the fork will
+    // be shared (technically this is undefined, but in practice that's the
+    // case), so we can't use it if we alter the environment
+    processOptions.allowVfork = !options.environment;
+
+    std::optional<Finally<std::function<void()>>> resumeLoggerDefer;
+    if (options.isInteractive) {
+        logger->pause();
+        resumeLoggerDefer.emplace(
+            []() {
+                logger->resume();
+            }
+        );
+    }
+
+    /* Fork. */
+    Pid pid = startProcess([&]() {
+        if (options.environment)
+            replaceEnv(*options.environment);
+        if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
+            throw SysError("dupping stdout");
+        if (options.mergeStderrToStdout)
+            if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1)
+                throw SysError("cannot dup stdout into stderr");
+        if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1)
+            throw SysError("dupping stdin");
+
+        if (options.chdir && chdir((*options.chdir).c_str()) == -1)
+            throw SysError("chdir failed");
+        if (options.gid && setgid(*options.gid) == -1)
+            throw SysError("setgid failed");
+        /* Drop all other groups if we're setgid. */
+        if (options.gid && setgroups(0, 0) == -1)
+            throw SysError("setgroups failed");
+        if (options.uid && setuid(*options.uid) == -1)
+            throw SysError("setuid failed");
+
+        Strings args_(options.args);
+        args_.push_front(options.program);
+
+        restoreProcessContext();
+
+        if (options.searchPath)
+            execvp(options.program.c_str(), stringsToCharPtrs(args_).data());
+            // This allows you to refer to a program with a pathname relative
+            // to the PATH variable.
+        else
+            execv(options.program.c_str(), stringsToCharPtrs(args_).data());
+
+        throw SysError("executing '%1%'", options.program);
+    }, processOptions);
+
+    out.writeSide.close();
+
+    std::thread writerThread;
+
+    std::promise<void> promise;
+
+    Finally doJoin([&]() {
+        if (writerThread.joinable())
+            writerThread.join();
+    });
+
+
+    if (source) {
+        in.readSide.close();
+        writerThread = std::thread([&]() {
+            try {
+                std::vector<char> buf(8 * 1024);
+                while (true) {
+                    size_t n;
+                    try {
+                        n = source->read(buf.data(), buf.size());
+                    } catch (EndOfFile &) {
+                        break;
+                    }
+                    writeFull(in.writeSide.get(), {buf.data(), n});
+                }
+                promise.set_value();
+            } catch (...) {
+                promise.set_exception(std::current_exception());
+            }
+            in.writeSide.close();
+        });
+    }
+
+    if (options.standardOut)
+        drainFD(out.readSide.get(), *options.standardOut);
+
+    /* Wait for the child to finish. */
+    int status = pid.wait();
+
+    /* Wait for the writer thread to finish. */
+    if (source) promise.get_future().get();
+
+    if (status)
+        throw ExecError(status, "program '%1%' %2%", options.program, statusToString(status));
+}
+
+//////////////////////////////////////////////////////////////////////
+
+std::string statusToString(int status)
+{
+    if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
+        if (WIFEXITED(status))
+            return fmt("failed with exit code %1%", WEXITSTATUS(status));
+        else if (WIFSIGNALED(status)) {
+            int sig = WTERMSIG(status);
+#if HAVE_STRSIGNAL
+            const char * description = strsignal(sig);
+            return fmt("failed due to signal %1% (%2%)", sig, description);
+#else
+            return fmt("failed due to signal %1%", sig);
+#endif
+        }
+        else
+            return "died abnormally";
+    } else return "succeeded";
+}
+
+
+bool statusOk(int status)
+{
+    return WIFEXITED(status) && WEXITSTATUS(status) == 0;
+}
+
+}
diff --git a/src/libutil/processes.hh b/src/libutil/processes.hh
new file mode 100644
index 000000000..978c37105
--- /dev/null
+++ b/src/libutil/processes.hh
@@ -0,0 +1,123 @@
+#pragma once
+///@file
+
+#include "types.hh"
+#include "error.hh"
+#include "logging.hh"
+#include "ansicolor.hh"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <signal.h>
+
+#include <boost/lexical_cast.hpp>
+
+#include <atomic>
+#include <functional>
+#include <map>
+#include <sstream>
+#include <optional>
+
+namespace nix {
+
+struct Sink;
+struct Source;
+
+class Pid
+{
+    pid_t pid = -1;
+    bool separatePG = false;
+    int killSignal = SIGKILL;
+public:
+    Pid();
+    Pid(pid_t pid);
+    ~Pid();
+    void operator =(pid_t pid);
+    operator pid_t();
+    int kill();
+    int wait();
+
+    void setSeparatePG(bool separatePG);
+    void setKillSignal(int signal);
+    pid_t release();
+};
+
+
+/**
+ * Kill all processes running under the specified uid by sending them
+ * a SIGKILL.
+ */
+void killUser(uid_t uid);
+
+
+/**
+ * Fork a process that runs the given function, and return the child
+ * pid to the caller.
+ */
+struct ProcessOptions
+{
+    std::string errorPrefix = "";
+    bool dieWithParent = true;
+    bool runExitHandlers = false;
+    bool allowVfork = false;
+    /**
+     * use clone() with the specified flags (Linux only)
+     */
+    int cloneFlags = 0;
+};
+
+pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = ProcessOptions());
+
+
+/**
+ * Run a program and return its stdout in a string (i.e., like the
+ * shell backtick operator).
+ */
+std::string runProgram(Path program, bool searchPath = false,
+    const Strings & args = Strings(),
+    const std::optional<std::string> & input = {}, bool isInteractive = false);
+
+struct RunOptions
+{
+    Path program;
+    bool searchPath = true;
+    Strings args;
+    std::optional<uid_t> uid;
+    std::optional<uid_t> gid;
+    std::optional<Path> chdir;
+    std::optional<std::map<std::string, std::string>> environment;
+    std::optional<std::string> input;
+    Source * standardIn = nullptr;
+    Sink * standardOut = nullptr;
+    bool mergeStderrToStdout = false;
+    bool isInteractive = false;
+};
+
+std::pair<int, std::string> runProgram(RunOptions && options);
+
+void runProgram2(const RunOptions & options);
+
+
+class ExecError : public Error
+{
+public:
+    int status;
+
+    template<typename... Args>
+    ExecError(int status, const Args & ... args)
+        : Error(args...), status(status)
+    { }
+};
+
+
+/**
+ * Convert the exit status of a child as returned by wait() into an
+ * error string.
+ */
+std::string statusToString(int status);
+
+bool statusOk(int status);
+
+}
diff --git a/src/libutil/references.cc b/src/libutil/references.cc
index 7f59b4c09..9d75606ef 100644
--- a/src/libutil/references.cc
+++ b/src/libutil/references.cc
@@ -1,6 +1,5 @@
 #include "references.hh"
 #include "hash.hh"
-#include "util.hh"
 #include "archive.hh"
 
 #include <map>
diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index 3d5121a19..725ddbb8d 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -1,5 +1,5 @@
 #include "serialise.hh"
-#include "util.hh"
+#include "signals.hh"
 
 #include <cstring>
 #include <cerrno>
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 333c254ea..9e07226bf 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -5,6 +5,7 @@
 
 #include "types.hh"
 #include "util.hh"
+#include "file-descriptor.hh"
 
 namespace boost::context { struct stack_context; }
 
diff --git a/src/libutil/signals.cc b/src/libutil/signals.cc
new file mode 100644
index 000000000..4632aa319
--- /dev/null
+++ b/src/libutil/signals.cc
@@ -0,0 +1,188 @@
+#include "signals.hh"
+#include "util.hh"
+#include "error.hh"
+#include "sync.hh"
+#include "terminal.hh"
+
+#include <thread>
+
+namespace nix {
+
+std::atomic<bool> _isInterrupted = false;
+
+static thread_local bool interruptThrown = false;
+thread_local std::function<bool()> interruptCheck;
+
+void setInterruptThrown()
+{
+    interruptThrown = true;
+}
+
+void _interrupted()
+{
+    /* Block user interrupts while an exception is being handled.
+       Throwing an exception while another exception is being handled
+       kills the program! */
+    if (!interruptThrown && !std::uncaught_exceptions()) {
+        interruptThrown = true;
+        throw Interrupted("interrupted by the user");
+    }
+}
+
+
+//////////////////////////////////////////////////////////////////////
+
+
+/* We keep track of interrupt callbacks using integer tokens, so we can iterate
+   safely without having to lock the data structure while executing arbitrary
+   functions.
+ */
+struct InterruptCallbacks {
+    typedef int64_t Token;
+
+    /* We use unique tokens so that we can't accidentally delete the wrong
+       handler because of an erroneous double delete. */
+    Token nextToken = 0;
+
+    /* Used as a list, see InterruptCallbacks comment. */
+    std::map<Token, std::function<void()>> callbacks;
+};
+
+static Sync<InterruptCallbacks> _interruptCallbacks;
+
+static void signalHandlerThread(sigset_t set)
+{
+    while (true) {
+        int signal = 0;
+        sigwait(&set, &signal);
+
+        if (signal == SIGINT || signal == SIGTERM || signal == SIGHUP)
+            triggerInterrupt();
+
+        else if (signal == SIGWINCH) {
+            updateWindowSize();
+        }
+    }
+}
+
+void triggerInterrupt()
+{
+    _isInterrupted = true;
+
+    {
+        InterruptCallbacks::Token i = 0;
+        while (true) {
+            std::function<void()> callback;
+            {
+                auto interruptCallbacks(_interruptCallbacks.lock());
+                auto lb = interruptCallbacks->callbacks.lower_bound(i);
+                if (lb == interruptCallbacks->callbacks.end())
+                    break;
+
+                callback = lb->second;
+                i = lb->first + 1;
+            }
+
+            try {
+                callback();
+            } catch (...) {
+                ignoreException();
+            }
+        }
+    }
+}
+
+
+static sigset_t savedSignalMask;
+static bool savedSignalMaskIsSet = false;
+
+void setChildSignalMask(sigset_t * sigs)
+{
+    assert(sigs); // C style function, but think of sigs as a reference
+
+#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE
+    sigemptyset(&savedSignalMask);
+    // There's no "assign" or "copy" function, so we rely on (math) idempotence
+    // of the or operator: a or a = a.
+    sigorset(&savedSignalMask, sigs, sigs);
+#else
+    // Without sigorset, our best bet is to assume that sigset_t is a type that
+    // can be assigned directly, such as is the case for a sigset_t defined as
+    // an integer type.
+    savedSignalMask = *sigs;
+#endif
+
+    savedSignalMaskIsSet = true;
+}
+
+void saveSignalMask() {
+    if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask))
+        throw SysError("querying signal mask");
+
+    savedSignalMaskIsSet = true;
+}
+
+void startSignalHandlerThread()
+{
+    updateWindowSize();
+
+    saveSignalMask();
+
+    sigset_t set;
+    sigemptyset(&set);
+    sigaddset(&set, SIGINT);
+    sigaddset(&set, SIGTERM);
+    sigaddset(&set, SIGHUP);
+    sigaddset(&set, SIGPIPE);
+    sigaddset(&set, SIGWINCH);
+    if (pthread_sigmask(SIG_BLOCK, &set, nullptr))
+        throw SysError("blocking signals");
+
+    std::thread(signalHandlerThread, set).detach();
+}
+
+void restoreSignals()
+{
+    // If startSignalHandlerThread wasn't called, that means we're not running
+    // in a proper libmain process, but a process that presumably manages its
+    // own signal handlers. Such a process should call either
+    //  - initNix(), to be a proper libmain process
+    //  - startSignalHandlerThread(), to resemble libmain regarding signal
+    //    handling only
+    //  - saveSignalMask(), for processes that define their own signal handling
+    //    thread
+    // TODO: Warn about this? Have a default signal mask? The latter depends on
+    //       whether we should generally inherit signal masks from the caller.
+    //       I don't know what the larger unix ecosystem expects from us here.
+    if (!savedSignalMaskIsSet)
+        return;
+
+    if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr))
+        throw SysError("restoring signals");
+}
+
+
+/* RAII helper to automatically deregister a callback. */
+struct InterruptCallbackImpl : InterruptCallback
+{
+    InterruptCallbacks::Token token;
+    ~InterruptCallbackImpl() override
+    {
+        auto interruptCallbacks(_interruptCallbacks.lock());
+        interruptCallbacks->callbacks.erase(token);
+    }
+};
+
+std::unique_ptr<InterruptCallback> createInterruptCallback(std::function<void()> callback)
+{
+    auto interruptCallbacks(_interruptCallbacks.lock());
+    auto token = interruptCallbacks->nextToken++;
+    interruptCallbacks->callbacks.emplace(token, callback);
+
+    auto res = std::make_unique<InterruptCallbackImpl>();
+    res->token = token;
+
+    return std::unique_ptr<InterruptCallback>(res.release());
+}
+
+}
diff --git a/src/libutil/signals.hh b/src/libutil/signals.hh
new file mode 100644
index 000000000..7e8beff33
--- /dev/null
+++ b/src/libutil/signals.hh
@@ -0,0 +1,104 @@
+#pragma once
+///@file
+
+#include "types.hh"
+#include "error.hh"
+#include "logging.hh"
+#include "ansicolor.hh"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <signal.h>
+
+#include <boost/lexical_cast.hpp>
+
+#include <atomic>
+#include <functional>
+#include <map>
+#include <sstream>
+#include <optional>
+
+namespace nix {
+
+/* User interruption. */
+
+extern std::atomic<bool> _isInterrupted;
+
+extern thread_local std::function<bool()> interruptCheck;
+
+void setInterruptThrown();
+
+void _interrupted();
+
+void inline checkInterrupt()
+{
+    if (_isInterrupted || (interruptCheck && interruptCheck()))
+        _interrupted();
+}
+
+MakeError(Interrupted, BaseError);
+
+
+/**
+ * Start a thread that handles various signals. Also block those signals
+ * on the current thread (and thus any threads created by it).
+ * Saves the signal mask before changing the mask to block those signals.
+ * See saveSignalMask().
+ */
+void startSignalHandlerThread();
+
+/**
+ * Saves the signal mask, which is the signal mask that nix will restore
+ * before creating child processes.
+ * See setChildSignalMask() to set an arbitrary signal mask instead of the
+ * current mask.
+ */
+void saveSignalMask();
+
+/**
+ * To use in a process that already called `startSignalHandlerThread()`
+ * or `saveSignalMask()` first.
+ */
+void restoreSignals();
+
+/**
+ * Sets the signal mask. Like saveSignalMask() but for a signal set that doesn't
+ * necessarily match the current thread's mask.
+ * See saveSignalMask() to set the saved mask to the current mask.
+ */
+void setChildSignalMask(sigset_t *sigs);
+
+struct InterruptCallback
+{
+    virtual ~InterruptCallback() { };
+};
+
+/**
+ * Register a function that gets called on SIGINT (in a non-signal
+ * context).
+ */
+std::unique_ptr<InterruptCallback> createInterruptCallback(
+    std::function<void()> callback);
+
+void triggerInterrupt();
+
+/**
+ * A RAII class that causes the current thread to receive SIGUSR1 when
+ * the signal handler thread receives SIGINT. That is, this allows
+ * SIGINT to be multiplexed to multiple threads.
+ */
+struct ReceiveInterrupts
+{
+    pthread_t target;
+    std::unique_ptr<InterruptCallback> callback;
+
+    ReceiveInterrupts()
+        : target(pthread_self())
+        , callback(createInterruptCallback([&]() { pthread_kill(target, SIGUSR1); }))
+    { }
+};
+
+
+}
diff --git a/src/libutil/suggestions.cc b/src/libutil/suggestions.cc
index 9510a5f0c..e67e986fb 100644
--- a/src/libutil/suggestions.cc
+++ b/src/libutil/suggestions.cc
@@ -1,7 +1,9 @@
 #include "suggestions.hh"
 #include "ansicolor.hh"
-#include "util.hh"
+#include "terminal.hh"
+
 #include <algorithm>
+#include <sstream>
 
 namespace nix {
 
diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc
index 5060a8f24..1733c791c 100644
--- a/src/libutil/tarfile.cc
+++ b/src/libutil/tarfile.cc
@@ -3,6 +3,7 @@
 
 #include "serialise.hh"
 #include "tarfile.hh"
+#include "file-system.hh"
 
 namespace nix {
 
diff --git a/src/libutil/terminal.cc b/src/libutil/terminal.cc
new file mode 100644
index 000000000..8febc8771
--- /dev/null
+++ b/src/libutil/terminal.cc
@@ -0,0 +1,108 @@
+#include "terminal.hh"
+#include "environment-variables.hh"
+#include "sync.hh"
+
+#include <sys/ioctl.h>
+#include <unistd.h>
+
+namespace nix {
+
+bool shouldANSI()
+{
+    return isatty(STDERR_FILENO)
+        && getEnv("TERM").value_or("dumb") != "dumb"
+        && !(getEnv("NO_COLOR").has_value() || getEnv("NOCOLOR").has_value());
+}
+
+std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int width)
+{
+    std::string t, e;
+    size_t w = 0;
+    auto i = s.begin();
+
+    while (w < (size_t) width && i != s.end()) {
+
+        if (*i == '\e') {
+            std::string e;
+            e += *i++;
+            char last = 0;
+
+            if (i != s.end() && *i == '[') {
+                e += *i++;
+                // eat parameter bytes
+                while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++;
+                // eat intermediate bytes
+                while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++;
+                // eat final byte
+                if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++;
+            } else {
+                if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++;
+            }
+
+            if (!filterAll && last == 'm')
+                t += e;
+        }
+
+        else if (*i == '\t') {
+            i++; t += ' '; w++;
+            while (w < (size_t) width && w % 8) {
+                t += ' '; w++;
+            }
+        }
+
+        else if (*i == '\r' || *i == '\a')
+            // do nothing for now
+            i++;
+
+        else {
+            w++;
+            // Copy one UTF-8 character.
+            if ((*i & 0xe0) == 0xc0) {
+                t += *i++;
+                if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++;
+            } else if ((*i & 0xf0) == 0xe0) {
+                t += *i++;
+                if (i != s.end() && ((*i & 0xc0) == 0x80)) {
+                    t += *i++;
+                    if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++;
+                }
+            } else if ((*i & 0xf8) == 0xf0) {
+                t += *i++;
+                if (i != s.end() && ((*i & 0xc0) == 0x80)) {
+                    t += *i++;
+                    if (i != s.end() && ((*i & 0xc0) == 0x80)) {
+                        t += *i++;
+                        if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++;
+                    }
+                }
+            } else
+                t += *i++;
+        }
+    }
+
+    return t;
+}
+
+
+//////////////////////////////////////////////////////////////////////
+
+static Sync<std::pair<unsigned short, unsigned short>> windowSize{{0, 0}};
+
+
+void updateWindowSize()
+{
+    struct winsize ws;
+    if (ioctl(2, TIOCGWINSZ, &ws) == 0) {
+        auto windowSize_(windowSize.lock());
+        windowSize_->first = ws.ws_row;
+        windowSize_->second = ws.ws_col;
+    }
+}
+
+
+std::pair<unsigned short, unsigned short> getWindowSize()
+{
+    return *windowSize.lock();
+}
+
+}
diff --git a/src/libutil/terminal.hh b/src/libutil/terminal.hh
new file mode 100644
index 000000000..9cb191308
--- /dev/null
+++ b/src/libutil/terminal.hh
@@ -0,0 +1,38 @@
+#pragma once
+///@file
+
+#include "types.hh"
+
+namespace nix {
+/**
+ * Determine whether ANSI escape sequences are appropriate for the
+ * present output.
+ */
+bool shouldANSI();
+
+/**
+ * Truncate a string to 'width' printable characters. If 'filterAll'
+ * is true, all ANSI escape sequences are filtered out. Otherwise,
+ * some escape sequences (such as colour setting) are copied but not
+ * included in the character count. Also, tabs are expanded to
+ * spaces.
+ */
+std::string filterANSIEscapes(std::string_view s,
+    bool filterAll = false,
+    unsigned int width = std::numeric_limits<unsigned int>::max());
+
+/**
+ * Recalculate the window size, updating a global variable. Used in the
+ * `SIGWINCH` signal handler.
+ */
+void updateWindowSize();
+
+/**
+ * @return the number of rows and columns of the terminal.
+ *
+ * The value is cached so this is quick. The cached result is computed
+ * by `updateWindowSize()`.
+ */
+std::pair<unsigned short, unsigned short> getWindowSize();
+
+}
diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc
index 2ffdc2e9b..c6dfe63d3 100644
--- a/src/libutil/tests/logging.cc
+++ b/src/libutil/tests/logging.cc
@@ -2,7 +2,6 @@
 
 #include "logging.hh"
 #include "nixexpr.hh"
-#include "util.hh"
 #include <fstream>
 
 #include <gtest/gtest.h>
diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc
index f3c1e8248..568f03f70 100644
--- a/src/libutil/tests/tests.cc
+++ b/src/libutil/tests/tests.cc
@@ -1,5 +1,8 @@
 #include "util.hh"
 #include "types.hh"
+#include "file-system.hh"
+#include "processes.hh"
+#include "terminal.hh"
 
 #include <limits.h>
 #include <gtest/gtest.h>
diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc
index dc4067f1b..c5e735617 100644
--- a/src/libutil/thread-pool.cc
+++ b/src/libutil/thread-pool.cc
@@ -1,4 +1,6 @@
 #include "thread-pool.hh"
+#include "signals.hh"
+#include "util.hh"
 
 namespace nix {
 
diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh
index 0e09fae97..02765badc 100644
--- a/src/libutil/thread-pool.hh
+++ b/src/libutil/thread-pool.hh
@@ -1,8 +1,8 @@
 #pragma once
 ///@file
 
+#include "error.hh"
 #include "sync.hh"
-#include "util.hh"
 
 #include <queue>
 #include <functional>
diff --git a/src/libutil/unix-domain-socket.cc b/src/libutil/unix-domain-socket.cc
new file mode 100644
index 000000000..8949461d2
--- /dev/null
+++ b/src/libutil/unix-domain-socket.cc
@@ -0,0 +1,100 @@
+#include "file-system.hh"
+#include "processes.hh"
+#include "unix-domain-socket.hh"
+
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <unistd.h>
+
+namespace nix {
+
+AutoCloseFD createUnixDomainSocket()
+{
+    AutoCloseFD fdSocket = socket(PF_UNIX, SOCK_STREAM
+        #ifdef SOCK_CLOEXEC
+        | SOCK_CLOEXEC
+        #endif
+        , 0);
+    if (!fdSocket)
+        throw SysError("cannot create Unix domain socket");
+    closeOnExec(fdSocket.get());
+    return fdSocket;
+}
+
+
+AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode)
+{
+    auto fdSocket = nix::createUnixDomainSocket();
+
+    bind(fdSocket.get(), path);
+
+    if (chmod(path.c_str(), mode) == -1)
+        throw SysError("changing permissions on '%1%'", path);
+
+    if (listen(fdSocket.get(), 100) == -1)
+        throw SysError("cannot listen on socket '%1%'", path);
+
+    return fdSocket;
+}
+
+
+void bind(int fd, const std::string & path)
+{
+    unlink(path.c_str());
+
+    struct sockaddr_un addr;
+    addr.sun_family = AF_UNIX;
+
+    if (path.size() + 1 >= sizeof(addr.sun_path)) {
+        Pid pid = startProcess([&]() {
+            Path dir = dirOf(path);
+            if (chdir(dir.c_str()) == -1)
+                throw SysError("chdir to '%s' failed", dir);
+            std::string base(baseNameOf(path));
+            if (base.size() + 1 >= sizeof(addr.sun_path))
+                throw Error("socket path '%s' is too long", base);
+            memcpy(addr.sun_path, base.c_str(), base.size() + 1);
+            if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
+                throw SysError("cannot bind to socket '%s'", path);
+            _exit(0);
+        });
+        int status = pid.wait();
+        if (status != 0)
+            throw Error("cannot bind to socket '%s'", path);
+    } else {
+        memcpy(addr.sun_path, path.c_str(), path.size() + 1);
+        if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
+            throw SysError("cannot bind to socket '%s'", path);
+    }
+}
+
+
+void connect(int fd, const std::string & path)
+{
+    struct sockaddr_un addr;
+    addr.sun_family = AF_UNIX;
+
+    if (path.size() + 1 >= sizeof(addr.sun_path)) {
+        Pid pid = startProcess([&]() {
+            Path dir = dirOf(path);
+            if (chdir(dir.c_str()) == -1)
+                throw SysError("chdir to '%s' failed", dir);
+            std::string base(baseNameOf(path));
+            if (base.size() + 1 >= sizeof(addr.sun_path))
+                throw Error("socket path '%s' is too long", base);
+            memcpy(addr.sun_path, base.c_str(), base.size() + 1);
+            if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
+                throw SysError("cannot connect to socket at '%s'", path);
+            _exit(0);
+        });
+        int status = pid.wait();
+        if (status != 0)
+            throw Error("cannot connect to socket at '%s'", path);
+    } else {
+        memcpy(addr.sun_path, path.c_str(), path.size() + 1);
+        if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
+            throw SysError("cannot connect to socket at '%s'", path);
+    }
+}
+
+}
diff --git a/src/libutil/unix-domain-socket.hh b/src/libutil/unix-domain-socket.hh
new file mode 100644
index 000000000..b78feb454
--- /dev/null
+++ b/src/libutil/unix-domain-socket.hh
@@ -0,0 +1,31 @@
+#pragma once
+///@file
+
+#include "types.hh"
+#include "file-descriptor.hh"
+
+#include <unistd.h>
+
+namespace nix {
+
+/**
+ * Create a Unix domain socket.
+ */
+AutoCloseFD createUnixDomainSocket();
+
+/**
+ * Create a Unix domain socket in listen mode.
+ */
+AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode);
+
+/**
+ * Bind a Unix domain socket to a path.
+ */
+void bind(int fd, const std::string & path);
+
+/**
+ * Connect to a Unix domain socket.
+ */
+void connect(int fd, const std::string & path);
+
+}
diff --git a/src/libutil/users.cc b/src/libutil/users.cc
new file mode 100644
index 000000000..95a641322
--- /dev/null
+++ b/src/libutil/users.cc
@@ -0,0 +1,116 @@
+#include "util.hh"
+#include "users.hh"
+#include "environment-variables.hh"
+#include "file-system.hh"
+
+#include <pwd.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+namespace nix {
+
+std::string getUserName()
+{
+    auto pw = getpwuid(geteuid());
+    std::string name = pw ? pw->pw_name : getEnv("USER").value_or("");
+    if (name.empty())
+        throw Error("cannot figure out user name");
+    return name;
+}
+
+Path getHomeOf(uid_t userId)
+{
+    std::vector<char> buf(16384);
+    struct passwd pwbuf;
+    struct passwd * pw;
+    if (getpwuid_r(userId, &pwbuf, buf.data(), buf.size(), &pw) != 0
+        || !pw || !pw->pw_dir || !pw->pw_dir[0])
+        throw Error("cannot determine user's home directory");
+    return pw->pw_dir;
+}
+
+Path getHome()
+{
+    static Path homeDir = []()
+    {
+        std::optional<std::string> unownedUserHomeDir = {};
+        auto homeDir = getEnv("HOME");
+        if (homeDir) {
+            // Only use $HOME if doesn't exist or is owned by the current user.
+            struct stat st;
+            int result = stat(homeDir->c_str(), &st);
+            if (result != 0) {
+                if (errno != ENOENT) {
+                    warn("couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno);
+                    homeDir.reset();
+                }
+            } else if (st.st_uid != geteuid()) {
+                unownedUserHomeDir.swap(homeDir);
+            }
+        }
+        if (!homeDir) {
+            homeDir = getHomeOf(geteuid());
+            if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) {
+                warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir);
+            }
+        }
+        return *homeDir;
+    }();
+    return homeDir;
+}
+
+
+Path getCacheDir()
+{
+    auto cacheDir = getEnv("XDG_CACHE_HOME");
+    return cacheDir ? *cacheDir : getHome() + "/.cache";
+}
+
+
+Path getConfigDir()
+{
+    auto configDir = getEnv("XDG_CONFIG_HOME");
+    return configDir ? *configDir : getHome() + "/.config";
+}
+
+std::vector<Path> getConfigDirs()
+{
+    Path configHome = getConfigDir();
+    auto configDirs = getEnv("XDG_CONFIG_DIRS").value_or("/etc/xdg");
+    std::vector<Path> result = tokenizeString<std::vector<std::string>>(configDirs, ":");
+    result.insert(result.begin(), configHome);
+    return result;
+}
+
+
+Path getDataDir()
+{
+    auto dataDir = getEnv("XDG_DATA_HOME");
+    return dataDir ? *dataDir : getHome() + "/.local/share";
+}
+
+Path getStateDir()
+{
+    auto stateDir = getEnv("XDG_STATE_HOME");
+    return stateDir ? *stateDir : getHome() + "/.local/state";
+}
+
+Path createNixStateDir()
+{
+    Path dir = getStateDir() + "/nix";
+    createDirs(dir);
+    return dir;
+}
+
+
+std::string expandTilde(std::string_view path)
+{
+    // TODO: expand ~user ?
+    auto tilde = path.substr(0, 2);
+    if (tilde == "~/" || tilde == "~")
+        return getHome() + std::string(path.substr(1));
+    else
+        return std::string(path);
+}
+
+}
diff --git a/src/libutil/users.hh b/src/libutil/users.hh
new file mode 100644
index 000000000..cecbb8bfb
--- /dev/null
+++ b/src/libutil/users.hh
@@ -0,0 +1,58 @@
+#pragma once
+///@file
+
+#include "types.hh"
+
+#include <sys/types.h>
+
+namespace nix {
+
+std::string getUserName();
+
+/**
+ * @return the given user's home directory from /etc/passwd.
+ */
+Path getHomeOf(uid_t userId);
+
+/**
+ * @return $HOME or the user's home directory from /etc/passwd.
+ */
+Path getHome();
+
+/**
+ * @return $XDG_CACHE_HOME or $HOME/.cache.
+ */
+Path getCacheDir();
+
+/**
+ * @return $XDG_CONFIG_HOME or $HOME/.config.
+ */
+Path getConfigDir();
+
+/**
+ * @return the directories to search for user configuration files
+ */
+std::vector<Path> getConfigDirs();
+
+/**
+ * @return $XDG_DATA_HOME or $HOME/.local/share.
+ */
+Path getDataDir();
+
+/**
+ * @return $XDG_STATE_HOME or $HOME/.local/state.
+ */
+Path getStateDir();
+
+/**
+ * Create the Nix state directory and return the path to it.
+ */
+Path createNixStateDir();
+
+/**
+ * Perform tilde expansion on a path, replacing tilde with the user's
+ * home directory.
+ */
+std::string expandTilde(std::string_view path);
+
+}
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 3b4c181e5..ee7a22849 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1,49 +1,10 @@
 #include "util.hh"
-#include "sync.hh"
-#include "finally.hh"
-#include "serialise.hh"
-#include "cgroup.hh"
+#include "fmt.hh"
 
 #include <array>
 #include <cctype>
-#include <cerrno>
-#include <climits>
-#include <cstdio>
-#include <cstdlib>
-#include <cstring>
-#include <future>
 #include <iostream>
-#include <mutex>
-#include <sstream>
-#include <thread>
-
-#include <fcntl.h>
 #include <grp.h>
-#include <pwd.h>
-#include <sys/ioctl.h>
-#include <sys/types.h>
-#include <sys/socket.h>
-#include <sys/wait.h>
-#include <sys/time.h>
-#include <sys/un.h>
-#include <unistd.h>
-
-#ifdef __APPLE__
-#include <sys/syscall.h>
-#include <mach-o/dyld.h>
-#endif
-
-#ifdef __linux__
-#include <sys/prctl.h>
-#include <sys/resource.h>
-#include <sys/mman.h>
-
-#include <cmath>
-#endif
-
-
-extern char * * environ __attribute__((weak));
-
 
 namespace nix {
 
@@ -67,1099 +28,8 @@ void initLibUtil() {
     assert(caught);
 }
 
-std::optional<std::string> getEnv(const std::string & key)
-{
-    char * value = getenv(key.c_str());
-    if (!value) return {};
-    return std::string(value);
-}
-
-std::optional<std::string> getEnvNonEmpty(const std::string & key) {
-    auto value = getEnv(key);
-    if (value == "") return {};
-    return value;
-}
-
-std::map<std::string, std::string> getEnv()
-{
-    std::map<std::string, std::string> env;
-    for (size_t i = 0; environ[i]; ++i) {
-        auto s = environ[i];
-        auto eq = strchr(s, '=');
-        if (!eq)
-            // invalid env, just keep going
-            continue;
-        env.emplace(std::string(s, eq), std::string(eq + 1));
-    }
-    return env;
-}
-
-
-void clearEnv()
-{
-    for (auto & name : getEnv())
-        unsetenv(name.first.c_str());
-}
-
-void replaceEnv(const std::map<std::string, std::string> & newEnv)
-{
-    clearEnv();
-    for (auto & newEnvVar : newEnv)
-        setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1);
-}
-
-
-Path absPath(Path path, std::optional<PathView> dir, bool resolveSymlinks)
-{
-    if (path[0] != '/') {
-        if (!dir) {
-#ifdef __GNU__
-            /* GNU (aka. GNU/Hurd) doesn't have any limitation on path
-               lengths and doesn't define `PATH_MAX'.  */
-            char *buf = getcwd(NULL, 0);
-            if (buf == NULL)
-#else
-            char buf[PATH_MAX];
-            if (!getcwd(buf, sizeof(buf)))
-#endif
-                throw SysError("cannot get cwd");
-            path = concatStrings(buf, "/", path);
-#ifdef __GNU__
-            free(buf);
-#endif
-        } else
-            path = concatStrings(*dir, "/", path);
-    }
-    return canonPath(path, resolveSymlinks);
-}
-
-
-Path canonPath(PathView path, bool resolveSymlinks)
-{
-    assert(path != "");
-
-    std::string s;
-    s.reserve(256);
-
-    if (path[0] != '/')
-        throw Error("not an absolute path: '%1%'", path);
-
-    std::string temp;
-
-    /* Count the number of times we follow a symlink and stop at some
-       arbitrary (but high) limit to prevent infinite loops. */
-    unsigned int followCount = 0, maxFollow = 1024;
-
-    while (1) {
-
-        /* Skip slashes. */
-        while (!path.empty() && path[0] == '/') path.remove_prefix(1);
-        if (path.empty()) break;
-
-        /* Ignore `.'. */
-        if (path == "." || path.substr(0, 2) == "./")
-            path.remove_prefix(1);
-
-        /* If `..', delete the last component. */
-        else if (path == ".." || path.substr(0, 3) == "../")
-        {
-            if (!s.empty()) s.erase(s.rfind('/'));
-            path.remove_prefix(2);
-        }
-
-        /* Normal component; copy it. */
-        else {
-            s += '/';
-            if (const auto slash = path.find('/'); slash == std::string::npos) {
-                s += path;
-                path = {};
-            } else {
-                s += path.substr(0, slash);
-                path = path.substr(slash);
-            }
-
-            /* If s points to a symlink, resolve it and continue from there */
-            if (resolveSymlinks && isLink(s)) {
-                if (++followCount >= maxFollow)
-                    throw Error("infinite symlink recursion in path '%1%'", path);
-                temp = concatStrings(readLink(s), path);
-                path = temp;
-                if (!temp.empty() && temp[0] == '/') {
-                    s.clear();  /* restart for symlinks pointing to absolute path */
-                } else {
-                    s = dirOf(s);
-                    if (s == "/") {  // we don’t want trailing slashes here, which dirOf only produces if s = /
-                        s.clear();
-                    }
-                }
-            }
-        }
-    }
-
-    return s.empty() ? "/" : std::move(s);
-}
-
-
-Path dirOf(const PathView path)
-{
-    Path::size_type pos = path.rfind('/');
-    if (pos == std::string::npos)
-        return ".";
-    return pos == 0 ? "/" : Path(path, 0, pos);
-}
-
-
-std::string_view baseNameOf(std::string_view path)
-{
-    if (path.empty())
-        return "";
-
-    auto last = path.size() - 1;
-    if (path[last] == '/' && last > 0)
-        last -= 1;
-
-    auto pos = path.rfind('/', last);
-    if (pos == std::string::npos)
-        pos = 0;
-    else
-        pos += 1;
-
-    return path.substr(pos, last - pos + 1);
-}
-
-
-std::string expandTilde(std::string_view path)
-{
-    // TODO: expand ~user ?
-    auto tilde = path.substr(0, 2);
-    if (tilde == "~/" || tilde == "~")
-        return getHome() + std::string(path.substr(1));
-    else
-        return std::string(path);
-}
-
-
-bool isInDir(std::string_view path, std::string_view dir)
-{
-    return path.substr(0, 1) == "/"
-        && path.substr(0, dir.size()) == dir
-        && path.size() >= dir.size() + 2
-        && path[dir.size()] == '/';
-}
-
-
-bool isDirOrInDir(std::string_view path, std::string_view dir)
-{
-    return path == dir || isInDir(path, dir);
-}
-
-
-struct stat stat(const Path & path)
-{
-    struct stat st;
-    if (stat(path.c_str(), &st))
-        throw SysError("getting status of '%1%'", path);
-    return st;
-}
-
-
-struct stat lstat(const Path & path)
-{
-    struct stat st;
-    if (lstat(path.c_str(), &st))
-        throw SysError("getting status of '%1%'", path);
-    return st;
-}
-
-
-bool pathExists(const Path & path)
-{
-    int res;
-    struct stat st;
-    res = lstat(path.c_str(), &st);
-    if (!res) return true;
-    if (errno != ENOENT && errno != ENOTDIR)
-        throw SysError("getting status of %1%", path);
-    return false;
-}
-
-bool pathAccessible(const Path & path)
-{
-    try {
-        return pathExists(path);
-    } catch (SysError & e) {
-        // swallow EPERM
-        if (e.errNo == EPERM) return false;
-        throw;
-    }
-}
-
-
-Path readLink(const Path & path)
-{
-    checkInterrupt();
-    std::vector<char> buf;
-    for (ssize_t bufSize = PATH_MAX/4; true; bufSize += bufSize/2) {
-        buf.resize(bufSize);
-        ssize_t rlSize = readlink(path.c_str(), buf.data(), bufSize);
-        if (rlSize == -1)
-            if (errno == EINVAL)
-                throw Error("'%1%' is not a symlink", path);
-            else
-                throw SysError("reading symbolic link '%1%'", path);
-        else if (rlSize < bufSize)
-            return std::string(buf.data(), rlSize);
-    }
-}
-
-
-bool isLink(const Path & path)
-{
-    struct stat st = lstat(path);
-    return S_ISLNK(st.st_mode);
-}
-
-
-DirEntries readDirectory(DIR *dir, const Path & path)
-{
-    DirEntries entries;
-    entries.reserve(64);
-
-    struct dirent * dirent;
-    while (errno = 0, dirent = readdir(dir)) { /* sic */
-        checkInterrupt();
-        std::string name = dirent->d_name;
-        if (name == "." || name == "..") continue;
-        entries.emplace_back(name, dirent->d_ino,
-#ifdef HAVE_STRUCT_DIRENT_D_TYPE
-            dirent->d_type
-#else
-            DT_UNKNOWN
-#endif
-        );
-    }
-    if (errno) throw SysError("reading directory '%1%'", path);
-
-    return entries;
-}
-
-DirEntries readDirectory(const Path & path)
-{
-    AutoCloseDir dir(opendir(path.c_str()));
-    if (!dir) throw SysError("opening directory '%1%'", path);
-
-    return readDirectory(dir.get(), path);
-}
-
-
-unsigned char getFileType(const Path & path)
-{
-    struct stat st = lstat(path);
-    if (S_ISDIR(st.st_mode)) return DT_DIR;
-    if (S_ISLNK(st.st_mode)) return DT_LNK;
-    if (S_ISREG(st.st_mode)) return DT_REG;
-    return DT_UNKNOWN;
-}
-
-
-std::string readFile(int fd)
-{
-    struct stat st;
-    if (fstat(fd, &st) == -1)
-        throw SysError("statting file");
-
-    return drainFD(fd, true, st.st_size);
-}
-
-
-std::string readFile(const Path & path)
-{
-    AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
-    if (!fd)
-        throw SysError("opening file '%1%'", path);
-    return readFile(fd.get());
-}
-
-
-void readFile(const Path & path, Sink & sink)
-{
-    AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
-    if (!fd)
-        throw SysError("opening file '%s'", path);
-    drainFD(fd.get(), sink);
-}
-
-
-void writeFile(const Path & path, std::string_view s, mode_t mode, bool sync)
-{
-    AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode);
-    if (!fd)
-        throw SysError("opening file '%1%'", path);
-    try {
-        writeFull(fd.get(), s);
-    } catch (Error & e) {
-        e.addTrace({}, "writing file '%1%'", path);
-        throw;
-    }
-    if (sync)
-        fd.fsync();
-    // Explicitly close to make sure exceptions are propagated.
-    fd.close();
-    if (sync)
-        syncParent(path);
-}
-
-
-void writeFile(const Path & path, Source & source, mode_t mode, bool sync)
-{
-    AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode);
-    if (!fd)
-        throw SysError("opening file '%1%'", path);
-
-    std::vector<char> buf(64 * 1024);
-
-    try {
-        while (true) {
-            try {
-                auto n = source.read(buf.data(), buf.size());
-                writeFull(fd.get(), {buf.data(), n});
-            } catch (EndOfFile &) { break; }
-        }
-    } catch (Error & e) {
-        e.addTrace({}, "writing file '%1%'", path);
-        throw;
-    }
-    if (sync)
-        fd.fsync();
-    // Explicitly close to make sure exceptions are propagated.
-    fd.close();
-    if (sync)
-        syncParent(path);
-}
-
-void syncParent(const Path & path)
-{
-    AutoCloseFD fd = open(dirOf(path).c_str(), O_RDONLY, 0);
-    if (!fd)
-        throw SysError("opening file '%1%'", path);
-    fd.fsync();
-}
-
-std::string readLine(int fd)
-{
-    std::string s;
-    while (1) {
-        checkInterrupt();
-        char ch;
-        // FIXME: inefficient
-        ssize_t rd = read(fd, &ch, 1);
-        if (rd == -1) {
-            if (errno != EINTR)
-                throw SysError("reading a line");
-        } else if (rd == 0)
-            throw EndOfFile("unexpected EOF reading a line");
-        else {
-            if (ch == '\n') return s;
-            s += ch;
-        }
-    }
-}
-
-
-void writeLine(int fd, std::string s)
-{
-    s += '\n';
-    writeFull(fd, s);
-}
-
-
-static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed)
-{
-    checkInterrupt();
-
-    std::string name(baseNameOf(path));
-
-    struct stat st;
-    if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) {
-        if (errno == ENOENT) return;
-        throw SysError("getting status of '%1%'", path);
-    }
-
-    if (!S_ISDIR(st.st_mode)) {
-        /* We are about to delete a file. Will it likely free space? */
-
-        switch (st.st_nlink) {
-            /* Yes: last link. */
-            case 1:
-                bytesFreed += st.st_size;
-                break;
-            /* Maybe: yes, if 'auto-optimise-store' or manual optimisation
-               was performed. Instead of checking for real let's assume
-               it's an optimised file and space will be freed.
-
-               In worst case we will double count on freed space for files
-               with exactly two hardlinks for unoptimised packages.
-             */
-            case 2:
-                bytesFreed += st.st_size;
-                break;
-            /* No: 3+ links. */
-            default:
-                break;
-        }
-    }
-
-    if (S_ISDIR(st.st_mode)) {
-        /* Make the directory accessible. */
-        const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR;
-        if ((st.st_mode & PERM_MASK) != PERM_MASK) {
-            if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1)
-                throw SysError("chmod '%1%'", path);
-        }
-
-        int fd = openat(parentfd, path.c_str(), O_RDONLY);
-        if (fd == -1)
-            throw SysError("opening directory '%1%'", path);
-        AutoCloseDir dir(fdopendir(fd));
-        if (!dir)
-            throw SysError("opening directory '%1%'", path);
-        for (auto & i : readDirectory(dir.get(), path))
-            _deletePath(dirfd(dir.get()), path + "/" + i.name, bytesFreed);
-    }
-
-    int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0;
-    if (unlinkat(parentfd, name.c_str(), flags) == -1) {
-        if (errno == ENOENT) return;
-        throw SysError("cannot unlink '%1%'", path);
-    }
-}
-
-static void _deletePath(const Path & path, uint64_t & bytesFreed)
-{
-    Path dir = dirOf(path);
-    if (dir == "")
-        dir = "/";
-
-    AutoCloseFD dirfd{open(dir.c_str(), O_RDONLY)};
-    if (!dirfd) {
-        if (errno == ENOENT) return;
-        throw SysError("opening directory '%1%'", path);
-    }
-
-    _deletePath(dirfd.get(), path, bytesFreed);
-}
-
-
-void deletePath(const Path & path)
-{
-    uint64_t dummy;
-    deletePath(path, dummy);
-}
-
-
-void deletePath(const Path & path, uint64_t & bytesFreed)
-{
-    //Activity act(*logger, lvlDebug, "recursively deleting path '%1%'", path);
-    bytesFreed = 0;
-    _deletePath(path, bytesFreed);
-}
-
-
-std::string getUserName()
-{
-    auto pw = getpwuid(geteuid());
-    std::string name = pw ? pw->pw_name : getEnv("USER").value_or("");
-    if (name.empty())
-        throw Error("cannot figure out user name");
-    return name;
-}
-
-Path getHomeOf(uid_t userId)
-{
-    std::vector<char> buf(16384);
-    struct passwd pwbuf;
-    struct passwd * pw;
-    if (getpwuid_r(userId, &pwbuf, buf.data(), buf.size(), &pw) != 0
-        || !pw || !pw->pw_dir || !pw->pw_dir[0])
-        throw Error("cannot determine user's home directory");
-    return pw->pw_dir;
-}
-
-Path getHome()
-{
-    static Path homeDir = []()
-    {
-        std::optional<std::string> unownedUserHomeDir = {};
-        auto homeDir = getEnv("HOME");
-        if (homeDir) {
-            // Only use $HOME if doesn't exist or is owned by the current user.
-            struct stat st;
-            int result = stat(homeDir->c_str(), &st);
-            if (result != 0) {
-                if (errno != ENOENT) {
-                    warn("couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno);
-                    homeDir.reset();
-                }
-            } else if (st.st_uid != geteuid()) {
-                unownedUserHomeDir.swap(homeDir);
-            }
-        }
-        if (!homeDir) {
-            homeDir = getHomeOf(geteuid());
-            if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) {
-                warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir);
-            }
-        }
-        return *homeDir;
-    }();
-    return homeDir;
-}
-
-
-Path getCacheDir()
-{
-    auto cacheDir = getEnv("XDG_CACHE_HOME");
-    return cacheDir ? *cacheDir : getHome() + "/.cache";
-}
-
-
-Path getConfigDir()
-{
-    auto configDir = getEnv("XDG_CONFIG_HOME");
-    return configDir ? *configDir : getHome() + "/.config";
-}
-
-std::vector<Path> getConfigDirs()
-{
-    Path configHome = getConfigDir();
-    auto configDirs = getEnv("XDG_CONFIG_DIRS").value_or("/etc/xdg");
-    std::vector<Path> result = tokenizeString<std::vector<std::string>>(configDirs, ":");
-    result.insert(result.begin(), configHome);
-    return result;
-}
-
-
-Path getDataDir()
-{
-    auto dataDir = getEnv("XDG_DATA_HOME");
-    return dataDir ? *dataDir : getHome() + "/.local/share";
-}
-
-Path getStateDir()
-{
-    auto stateDir = getEnv("XDG_STATE_HOME");
-    return stateDir ? *stateDir : getHome() + "/.local/state";
-}
-
-Path createNixStateDir()
-{
-    Path dir = getStateDir() + "/nix";
-    createDirs(dir);
-    return dir;
-}
-
-
-std::optional<Path> getSelfExe()
-{
-    static auto cached = []() -> std::optional<Path>
-    {
-        #if __linux__
-        return readLink("/proc/self/exe");
-        #elif __APPLE__
-        char buf[1024];
-        uint32_t size = sizeof(buf);
-        if (_NSGetExecutablePath(buf, &size) == 0)
-            return buf;
-        else
-            return std::nullopt;
-        #else
-        return std::nullopt;
-        #endif
-    }();
-    return cached;
-}
-
-
-Paths createDirs(const Path & path)
-{
-    Paths created;
-    if (path == "/") return created;
-
-    struct stat st;
-    if (lstat(path.c_str(), &st) == -1) {
-        created = createDirs(dirOf(path));
-        if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST)
-            throw SysError("creating directory '%1%'", path);
-        st = lstat(path);
-        created.push_back(path);
-    }
-
-    if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1)
-        throw SysError("statting symlink '%1%'", path);
-
-    if (!S_ISDIR(st.st_mode)) throw Error("'%1%' is not a directory", path);
-
-    return created;
-}
-
-
-void readFull(int fd, char * buf, size_t count)
-{
-    while (count) {
-        checkInterrupt();
-        ssize_t res = read(fd, buf, count);
-        if (res == -1) {
-            if (errno == EINTR) continue;
-            throw SysError("reading from file");
-        }
-        if (res == 0) throw EndOfFile("unexpected end-of-file");
-        count -= res;
-        buf += res;
-    }
-}
-
-
-void writeFull(int fd, std::string_view s, bool allowInterrupts)
-{
-    while (!s.empty()) {
-        if (allowInterrupts) checkInterrupt();
-        ssize_t res = write(fd, s.data(), s.size());
-        if (res == -1 && errno != EINTR)
-            throw SysError("writing to file");
-        if (res > 0)
-            s.remove_prefix(res);
-    }
-}
-
-
-std::string drainFD(int fd, bool block, const size_t reserveSize)
-{
-    // the parser needs two extra bytes to append terminating characters, other users will
-    // not care very much about the extra memory.
-    StringSink sink(reserveSize + 2);
-    drainFD(fd, sink, block);
-    return std::move(sink.s);
-}
-
-
-void drainFD(int fd, Sink & sink, bool block)
-{
-    // silence GCC maybe-uninitialized warning in finally
-    int saved = 0;
-
-    if (!block) {
-        saved = fcntl(fd, F_GETFL);
-        if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1)
-            throw SysError("making file descriptor non-blocking");
-    }
-
-    Finally finally([&]() {
-        if (!block) {
-            if (fcntl(fd, F_SETFL, saved) == -1)
-                throw SysError("making file descriptor blocking");
-        }
-    });
-
-    std::vector<unsigned char> buf(64 * 1024);
-    while (1) {
-        checkInterrupt();
-        ssize_t rd = read(fd, buf.data(), buf.size());
-        if (rd == -1) {
-            if (!block && (errno == EAGAIN || errno == EWOULDBLOCK))
-                break;
-            if (errno != EINTR)
-                throw SysError("reading from file");
-        }
-        else if (rd == 0) break;
-        else sink({(char *) buf.data(), (size_t) rd});
-    }
-}
-
 //////////////////////////////////////////////////////////////////////
 
-unsigned int getMaxCPU()
-{
-    #if __linux__
-    try {
-        auto cgroupFS = getCgroupFS();
-        if (!cgroupFS) return 0;
-
-        auto cgroups = getCgroups("/proc/self/cgroup");
-        auto cgroup = cgroups[""];
-        if (cgroup == "") return 0;
-
-        auto cpuFile = *cgroupFS + "/" + cgroup + "/cpu.max";
-
-        auto cpuMax = readFile(cpuFile);
-        auto cpuMaxParts = tokenizeString<std::vector<std::string>>(cpuMax, " \n");
-        auto quota = cpuMaxParts[0];
-        auto period = cpuMaxParts[1];
-        if (quota != "max")
-                return std::ceil(std::stoi(quota) / std::stof(period));
-    } catch (Error &) { ignoreException(lvlDebug); }
-    #endif
-
-    return 0;
-}
-
-//////////////////////////////////////////////////////////////////////
-
-
-AutoDelete::AutoDelete() : del{false} {}
-
-AutoDelete::AutoDelete(const std::string & p, bool recursive) : path(p)
-{
-    del = true;
-    this->recursive = recursive;
-}
-
-AutoDelete::~AutoDelete()
-{
-    try {
-        if (del) {
-            if (recursive)
-                deletePath(path);
-            else {
-                if (remove(path.c_str()) == -1)
-                    throw SysError("cannot unlink '%1%'", path);
-            }
-        }
-    } catch (...) {
-        ignoreException();
-    }
-}
-
-void AutoDelete::cancel()
-{
-    del = false;
-}
-
-void AutoDelete::reset(const Path & p, bool recursive) {
-    path = p;
-    this->recursive = recursive;
-    del = true;
-}
-
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-AutoCloseFD::AutoCloseFD() : fd{-1} {}
-
-
-AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {}
-
-
-AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd}
-{
-    that.fd = -1;
-}
-
-
-AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that)
-{
-    close();
-    fd = that.fd;
-    that.fd = -1;
-    return *this;
-}
-
-
-AutoCloseFD::~AutoCloseFD()
-{
-    try {
-        close();
-    } catch (...) {
-        ignoreException();
-    }
-}
-
-
-int AutoCloseFD::get() const
-{
-    return fd;
-}
-
-
-void AutoCloseFD::close()
-{
-    if (fd != -1) {
-        if (::close(fd) == -1)
-            /* This should never happen. */
-            throw SysError("closing file descriptor %1%", fd);
-        fd = -1;
-    }
-}
-
-void AutoCloseFD::fsync()
-{
-  if (fd != -1) {
-      int result;
-#if __APPLE__
-      result = ::fcntl(fd, F_FULLFSYNC);
-#else
-      result = ::fsync(fd);
-#endif
-      if (result == -1)
-          throw SysError("fsync file descriptor %1%", fd);
-  }
-}
-
-
-AutoCloseFD::operator bool() const
-{
-    return fd != -1;
-}
-
-
-int AutoCloseFD::release()
-{
-    int oldFD = fd;
-    fd = -1;
-    return oldFD;
-}
-
-
-void Pipe::create()
-{
-    int fds[2];
-#if HAVE_PIPE2
-    if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe");
-#else
-    if (pipe(fds) != 0) throw SysError("creating pipe");
-    closeOnExec(fds[0]);
-    closeOnExec(fds[1]);
-#endif
-    readSide = fds[0];
-    writeSide = fds[1];
-}
-
-
-void Pipe::close()
-{
-    readSide.close();
-    writeSide.close();
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-Pid::Pid()
-{
-}
-
-
-Pid::Pid(pid_t pid)
-    : pid(pid)
-{
-}
-
-
-Pid::~Pid()
-{
-    if (pid != -1) kill();
-}
-
-
-void Pid::operator =(pid_t pid)
-{
-    if (this->pid != -1 && this->pid != pid) kill();
-    this->pid = pid;
-    killSignal = SIGKILL; // reset signal to default
-}
-
-
-Pid::operator pid_t()
-{
-    return pid;
-}
-
-
-int Pid::kill()
-{
-    assert(pid != -1);
-
-    debug("killing process %1%", pid);
-
-    /* Send the requested signal to the child.  If it has its own
-       process group, send the signal to every process in the child
-       process group (which hopefully includes *all* its children). */
-    if (::kill(separatePG ? -pid : pid, killSignal) != 0) {
-        /* On BSDs, killing a process group will return EPERM if all
-           processes in the group are zombies (or something like
-           that). So try to detect and ignore that situation. */
-#if __FreeBSD__ || __APPLE__
-        if (errno != EPERM || ::kill(pid, 0) != 0)
-#endif
-            logError(SysError("killing process %d", pid).info());
-    }
-
-    return wait();
-}
-
-
-int Pid::wait()
-{
-    assert(pid != -1);
-    while (1) {
-        int status;
-        int res = waitpid(pid, &status, 0);
-        if (res == pid) {
-            pid = -1;
-            return status;
-        }
-        if (errno != EINTR)
-            throw SysError("cannot get exit status of PID %d", pid);
-        checkInterrupt();
-    }
-}
-
-
-void Pid::setSeparatePG(bool separatePG)
-{
-    this->separatePG = separatePG;
-}
-
-
-void Pid::setKillSignal(int signal)
-{
-    this->killSignal = signal;
-}
-
-
-pid_t Pid::release()
-{
-    pid_t p = pid;
-    pid = -1;
-    return p;
-}
-
-
-void killUser(uid_t uid)
-{
-    debug("killing all processes running under uid '%1%'", uid);
-
-    assert(uid != 0); /* just to be safe... */
-
-    /* The system call kill(-1, sig) sends the signal `sig' to all
-       users to which the current process can send signals.  So we
-       fork a process, switch to uid, and send a mass kill. */
-
-    Pid pid = startProcess([&]() {
-
-        if (setuid(uid) == -1)
-            throw SysError("setting uid");
-
-        while (true) {
-#ifdef __APPLE__
-            /* OSX's kill syscall takes a third parameter that, among
-               other things, determines if kill(-1, signo) affects the
-               calling process. In the OSX libc, it's set to true,
-               which means "follow POSIX", which we don't want here
-                 */
-            if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break;
-#else
-            if (kill(-1, SIGKILL) == 0) break;
-#endif
-            if (errno == ESRCH || errno == EPERM) break; /* no more processes */
-            if (errno != EINTR)
-                throw SysError("cannot kill processes for uid '%1%'", uid);
-        }
-
-        _exit(0);
-    });
-
-    int status = pid.wait();
-    if (status != 0)
-        throw Error("cannot kill processes for uid '%1%': %2%", uid, statusToString(status));
-
-    /* !!! We should really do some check to make sure that there are
-       no processes left running under `uid', but there is no portable
-       way to do so (I think).  The most reliable way may be `ps -eo
-       uid | grep -q $uid'. */
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-/* Wrapper around vfork to prevent the child process from clobbering
-   the caller's stack frame in the parent. */
-static pid_t doFork(bool allowVfork, std::function<void()> fun) __attribute__((noinline));
-static pid_t doFork(bool allowVfork, std::function<void()> fun)
-{
-#ifdef __linux__
-    pid_t pid = allowVfork ? vfork() : fork();
-#else
-    pid_t pid = fork();
-#endif
-    if (pid != 0) return pid;
-    fun();
-    abort();
-}
-
-
-#if __linux__
-static int childEntry(void * arg)
-{
-    auto main = (std::function<void()> *) arg;
-    (*main)();
-    return 1;
-}
-#endif
-
-
-pid_t startProcess(std::function<void()> fun, const ProcessOptions & options)
-{
-    std::function<void()> wrapper = [&]() {
-        if (!options.allowVfork)
-            logger = makeSimpleLogger();
-        try {
-#if __linux__
-            if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1)
-                throw SysError("setting death signal");
-#endif
-            fun();
-        } catch (std::exception & e) {
-            try {
-                std::cerr << options.errorPrefix << e.what() << "\n";
-            } catch (...) { }
-        } catch (...) { }
-        if (options.runExitHandlers)
-            exit(1);
-        else
-            _exit(1);
-    };
-
-    pid_t pid = -1;
-
-    if (options.cloneFlags) {
-        #ifdef __linux__
-        // Not supported, since then we don't know when to free the stack.
-        assert(!(options.cloneFlags & CLONE_VM));
-
-        size_t stackSize = 1 * 1024 * 1024;
-        auto stack = (char *) mmap(0, stackSize,
-            PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
-        if (stack == MAP_FAILED) throw SysError("allocating stack");
-
-        Finally freeStack([&]() { munmap(stack, stackSize); });
-
-        pid = clone(childEntry, stack + stackSize, options.cloneFlags | SIGCHLD, &wrapper);
-        #else
-        throw Error("clone flags are only supported on Linux");
-        #endif
-    } else
-        pid = doFork(options.allowVfork, wrapper);
-
-    if (pid == -1) throw SysError("unable to fork");
-
-    return pid;
-}
-
-
 std::vector<char *> stringsToCharPtrs(const Strings & ss)
 {
     std::vector<char *> res;
@@ -1168,211 +38,6 @@ std::vector<char *> stringsToCharPtrs(const Strings & ss)
     return res;
 }
 
-std::string runProgram(Path program, bool searchPath, const Strings & args,
-    const std::optional<std::string> & input, bool isInteractive)
-{
-    auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input, .isInteractive = isInteractive});
-
-    if (!statusOk(res.first))
-        throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first));
-
-    return res.second;
-}
-
-// Output = error code + "standard out" output stream
-std::pair<int, std::string> runProgram(RunOptions && options)
-{
-    StringSink sink;
-    options.standardOut = &sink;
-
-    int status = 0;
-
-    try {
-        runProgram2(options);
-    } catch (ExecError & e) {
-        status = e.status;
-    }
-
-    return {status, std::move(sink.s)};
-}
-
-void runProgram2(const RunOptions & options)
-{
-    checkInterrupt();
-
-    assert(!(options.standardIn && options.input));
-
-    std::unique_ptr<Source> source_;
-    Source * source = options.standardIn;
-
-    if (options.input) {
-        source_ = std::make_unique<StringSource>(*options.input);
-        source = source_.get();
-    }
-
-    /* Create a pipe. */
-    Pipe out, in;
-    if (options.standardOut) out.create();
-    if (source) in.create();
-
-    ProcessOptions processOptions;
-    // vfork implies that the environment of the main process and the fork will
-    // be shared (technically this is undefined, but in practice that's the
-    // case), so we can't use it if we alter the environment
-    processOptions.allowVfork = !options.environment;
-
-    std::optional<Finally<std::function<void()>>> resumeLoggerDefer;
-    if (options.isInteractive) {
-        logger->pause();
-        resumeLoggerDefer.emplace(
-            []() {
-                logger->resume();
-            }
-        );
-    }
-
-    /* Fork. */
-    Pid pid = startProcess([&]() {
-        if (options.environment)
-            replaceEnv(*options.environment);
-        if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
-            throw SysError("dupping stdout");
-        if (options.mergeStderrToStdout)
-            if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1)
-                throw SysError("cannot dup stdout into stderr");
-        if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1)
-            throw SysError("dupping stdin");
-
-        if (options.chdir && chdir((*options.chdir).c_str()) == -1)
-            throw SysError("chdir failed");
-        if (options.gid && setgid(*options.gid) == -1)
-            throw SysError("setgid failed");
-        /* Drop all other groups if we're setgid. */
-        if (options.gid && setgroups(0, 0) == -1)
-            throw SysError("setgroups failed");
-        if (options.uid && setuid(*options.uid) == -1)
-            throw SysError("setuid failed");
-
-        Strings args_(options.args);
-        args_.push_front(options.program);
-
-        restoreProcessContext();
-
-        if (options.searchPath)
-            execvp(options.program.c_str(), stringsToCharPtrs(args_).data());
-            // This allows you to refer to a program with a pathname relative
-            // to the PATH variable.
-        else
-            execv(options.program.c_str(), stringsToCharPtrs(args_).data());
-
-        throw SysError("executing '%1%'", options.program);
-    }, processOptions);
-
-    out.writeSide.close();
-
-    std::thread writerThread;
-
-    std::promise<void> promise;
-
-    Finally doJoin([&]() {
-        if (writerThread.joinable())
-            writerThread.join();
-    });
-
-
-    if (source) {
-        in.readSide.close();
-        writerThread = std::thread([&]() {
-            try {
-                std::vector<char> buf(8 * 1024);
-                while (true) {
-                    size_t n;
-                    try {
-                        n = source->read(buf.data(), buf.size());
-                    } catch (EndOfFile &) {
-                        break;
-                    }
-                    writeFull(in.writeSide.get(), {buf.data(), n});
-                }
-                promise.set_value();
-            } catch (...) {
-                promise.set_exception(std::current_exception());
-            }
-            in.writeSide.close();
-        });
-    }
-
-    if (options.standardOut)
-        drainFD(out.readSide.get(), *options.standardOut);
-
-    /* Wait for the child to finish. */
-    int status = pid.wait();
-
-    /* Wait for the writer thread to finish. */
-    if (source) promise.get_future().get();
-
-    if (status)
-        throw ExecError(status, "program '%1%' %2%", options.program, statusToString(status));
-}
-
-
-void closeMostFDs(const std::set<int> & exceptions)
-{
-#if __linux__
-    try {
-        for (auto & s : readDirectory("/proc/self/fd")) {
-            auto fd = std::stoi(s.name);
-            if (!exceptions.count(fd)) {
-                debug("closing leaked FD %d", fd);
-                close(fd);
-            }
-        }
-        return;
-    } catch (SysError &) {
-    }
-#endif
-
-    int maxFD = 0;
-    maxFD = sysconf(_SC_OPEN_MAX);
-    for (int fd = 0; fd < maxFD; ++fd)
-        if (!exceptions.count(fd))
-            close(fd); /* ignore result */
-}
-
-
-void closeOnExec(int fd)
-{
-    int prev;
-    if ((prev = fcntl(fd, F_GETFD, 0)) == -1 ||
-        fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1)
-        throw SysError("setting close-on-exec flag");
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
-std::atomic<bool> _isInterrupted = false;
-
-static thread_local bool interruptThrown = false;
-thread_local std::function<bool()> interruptCheck;
-
-void setInterruptThrown()
-{
-    interruptThrown = true;
-}
-
-void _interrupted()
-{
-    /* Block user interrupts while an exception is being handled.
-       Throwing an exception while another exception is being handled
-       kills the program! */
-    if (!interruptThrown && !std::uncaught_exceptions()) {
-        interruptThrown = true;
-        throw Interrupted("interrupted by the user");
-    }
-}
-
 
 //////////////////////////////////////////////////////////////////////
 
@@ -1438,32 +103,6 @@ std::string rewriteStrings(std::string s, const StringMap & rewrites)
 }
 
 
-std::string statusToString(int status)
-{
-    if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
-        if (WIFEXITED(status))
-            return fmt("failed with exit code %1%", WEXITSTATUS(status));
-        else if (WIFSIGNALED(status)) {
-            int sig = WTERMSIG(status);
-#if HAVE_STRSIGNAL
-            const char * description = strsignal(sig);
-            return fmt("failed due to signal %1% (%2%)", sig, description);
-#else
-            return fmt("failed due to signal %1%", sig);
-#endif
-        }
-        else
-            return "died abnormally";
-    } else return "succeeded";
-}
-
-
-bool statusOk(int status)
-{
-    return WIFEXITED(status) && WEXITSTATUS(status) == 0;
-}
-
-
 bool hasPrefix(std::string_view s, std::string_view prefix)
 {
     return s.compare(0, prefix.size(), prefix) == 0;
@@ -1511,82 +150,6 @@ void ignoreException(Verbosity lvl)
     } catch (...) { }
 }
 
-bool shouldANSI()
-{
-    return isatty(STDERR_FILENO)
-        && getEnv("TERM").value_or("dumb") != "dumb"
-        && !(getEnv("NO_COLOR").has_value() || getEnv("NOCOLOR").has_value());
-}
-
-std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int width)
-{
-    std::string t, e;
-    size_t w = 0;
-    auto i = s.begin();
-
-    while (w < (size_t) width && i != s.end()) {
-
-        if (*i == '\e') {
-            std::string e;
-            e += *i++;
-            char last = 0;
-
-            if (i != s.end() && *i == '[') {
-                e += *i++;
-                // eat parameter bytes
-                while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++;
-                // eat intermediate bytes
-                while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++;
-                // eat final byte
-                if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++;
-            } else {
-                if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++;
-            }
-
-            if (!filterAll && last == 'm')
-                t += e;
-        }
-
-        else if (*i == '\t') {
-            i++; t += ' '; w++;
-            while (w < (size_t) width && w % 8) {
-                t += ' '; w++;
-            }
-        }
-
-        else if (*i == '\r' || *i == '\a')
-            // do nothing for now
-            i++;
-
-        else {
-            w++;
-            // Copy one UTF-8 character.
-            if ((*i & 0xe0) == 0xc0) {
-                t += *i++;
-                if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++;
-            } else if ((*i & 0xf0) == 0xe0) {
-                t += *i++;
-                if (i != s.end() && ((*i & 0xc0) == 0x80)) {
-                    t += *i++;
-                    if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++;
-                }
-            } else if ((*i & 0xf8) == 0xf0) {
-                t += *i++;
-                if (i != s.end() && ((*i & 0xc0) == 0x80)) {
-                    t += *i++;
-                    if (i != s.end() && ((*i & 0xc0) == 0x80)) {
-                        t += *i++;
-                        if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++;
-                    }
-                }
-            } else
-                t += *i++;
-        }
-    }
-
-    return t;
-}
-
 
 constexpr char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
 
@@ -1703,386 +266,9 @@ std::pair<std::string_view, std::string_view> getLine(std::string_view s)
 }
 
 
-//////////////////////////////////////////////////////////////////////
-
-static Sync<std::pair<unsigned short, unsigned short>> windowSize{{0, 0}};
-
-
-static void updateWindowSize()
-{
-    struct winsize ws;
-    if (ioctl(2, TIOCGWINSZ, &ws) == 0) {
-        auto windowSize_(windowSize.lock());
-        windowSize_->first = ws.ws_row;
-        windowSize_->second = ws.ws_col;
-    }
-}
-
-
-std::pair<unsigned short, unsigned short> getWindowSize()
-{
-    return *windowSize.lock();
-}
-
-
-/* We keep track of interrupt callbacks using integer tokens, so we can iterate
-   safely without having to lock the data structure while executing arbitrary
-   functions.
- */
-struct InterruptCallbacks {
-    typedef int64_t Token;
-
-    /* We use unique tokens so that we can't accidentally delete the wrong
-       handler because of an erroneous double delete. */
-    Token nextToken = 0;
-
-    /* Used as a list, see InterruptCallbacks comment. */
-    std::map<Token, std::function<void()>> callbacks;
-};
-
-static Sync<InterruptCallbacks> _interruptCallbacks;
-
-static void signalHandlerThread(sigset_t set)
-{
-    while (true) {
-        int signal = 0;
-        sigwait(&set, &signal);
-
-        if (signal == SIGINT || signal == SIGTERM || signal == SIGHUP)
-            triggerInterrupt();
-
-        else if (signal == SIGWINCH) {
-            updateWindowSize();
-        }
-    }
-}
-
-void triggerInterrupt()
-{
-    _isInterrupted = true;
-
-    {
-        InterruptCallbacks::Token i = 0;
-        while (true) {
-            std::function<void()> callback;
-            {
-                auto interruptCallbacks(_interruptCallbacks.lock());
-                auto lb = interruptCallbacks->callbacks.lower_bound(i);
-                if (lb == interruptCallbacks->callbacks.end())
-                    break;
-
-                callback = lb->second;
-                i = lb->first + 1;
-            }
-
-            try {
-                callback();
-            } catch (...) {
-                ignoreException();
-            }
-        }
-    }
-}
-
-static sigset_t savedSignalMask;
-static bool savedSignalMaskIsSet = false;
-
-void setChildSignalMask(sigset_t * sigs)
-{
-    assert(sigs); // C style function, but think of sigs as a reference
-
-#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE
-    sigemptyset(&savedSignalMask);
-    // There's no "assign" or "copy" function, so we rely on (math) idempotence
-    // of the or operator: a or a = a.
-    sigorset(&savedSignalMask, sigs, sigs);
-#else
-    // Without sigorset, our best bet is to assume that sigset_t is a type that
-    // can be assigned directly, such as is the case for a sigset_t defined as
-    // an integer type.
-    savedSignalMask = *sigs;
-#endif
-
-    savedSignalMaskIsSet = true;
-}
-
-void saveSignalMask() {
-    if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask))
-        throw SysError("querying signal mask");
-
-    savedSignalMaskIsSet = true;
-}
-
-void startSignalHandlerThread()
-{
-    updateWindowSize();
-
-    saveSignalMask();
-
-    sigset_t set;
-    sigemptyset(&set);
-    sigaddset(&set, SIGINT);
-    sigaddset(&set, SIGTERM);
-    sigaddset(&set, SIGHUP);
-    sigaddset(&set, SIGPIPE);
-    sigaddset(&set, SIGWINCH);
-    if (pthread_sigmask(SIG_BLOCK, &set, nullptr))
-        throw SysError("blocking signals");
-
-    std::thread(signalHandlerThread, set).detach();
-}
-
-static void restoreSignals()
-{
-    // If startSignalHandlerThread wasn't called, that means we're not running
-    // in a proper libmain process, but a process that presumably manages its
-    // own signal handlers. Such a process should call either
-    //  - initNix(), to be a proper libmain process
-    //  - startSignalHandlerThread(), to resemble libmain regarding signal
-    //    handling only
-    //  - saveSignalMask(), for processes that define their own signal handling
-    //    thread
-    // TODO: Warn about this? Have a default signal mask? The latter depends on
-    //       whether we should generally inherit signal masks from the caller.
-    //       I don't know what the larger unix ecosystem expects from us here.
-    if (!savedSignalMaskIsSet)
-        return;
-
-    if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr))
-        throw SysError("restoring signals");
-}
-
-#if __linux__
-rlim_t savedStackSize = 0;
-#endif
-
-void setStackSize(size_t stackSize)
-{
-    #if __linux__
-    struct rlimit limit;
-    if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) {
-        savedStackSize = limit.rlim_cur;
-        limit.rlim_cur = stackSize;
-        setrlimit(RLIMIT_STACK, &limit);
-    }
-    #endif
-}
-
-#if __linux__
-static AutoCloseFD fdSavedMountNamespace;
-static AutoCloseFD fdSavedRoot;
-#endif
-
-void saveMountNamespace()
-{
-#if __linux__
-    static std::once_flag done;
-    std::call_once(done, []() {
-        fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY);
-        if (!fdSavedMountNamespace)
-            throw SysError("saving parent mount namespace");
-
-        fdSavedRoot = open("/proc/self/root", O_RDONLY);
-    });
-#endif
-}
-
-void restoreMountNamespace()
-{
-#if __linux__
-    try {
-        auto savedCwd = absPath(".");
-
-        if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1)
-            throw SysError("restoring parent mount namespace");
-
-        if (fdSavedRoot) {
-            if (fchdir(fdSavedRoot.get()))
-                throw SysError("chdir into saved root");
-            if (chroot("."))
-                throw SysError("chroot into saved root");
-        }
-
-        if (chdir(savedCwd.c_str()) == -1)
-            throw SysError("restoring cwd");
-    } catch (Error & e) {
-        debug(e.msg());
-    }
-#endif
-}
-
-void unshareFilesystem()
-{
-#ifdef __linux__
-    if (unshare(CLONE_FS) != 0 && errno != EPERM)
-        throw SysError("unsharing filesystem state in download thread");
-#endif
-}
-
-void restoreProcessContext(bool restoreMounts)
-{
-    restoreSignals();
-    if (restoreMounts) {
-        restoreMountNamespace();
-    }
-
-    #if __linux__
-    if (savedStackSize) {
-        struct rlimit limit;
-        if (getrlimit(RLIMIT_STACK, &limit) == 0) {
-            limit.rlim_cur = savedStackSize;
-            setrlimit(RLIMIT_STACK, &limit);
-        }
-    }
-    #endif
-}
-
-/* RAII helper to automatically deregister a callback. */
-struct InterruptCallbackImpl : InterruptCallback
-{
-    InterruptCallbacks::Token token;
-    ~InterruptCallbackImpl() override
-    {
-        auto interruptCallbacks(_interruptCallbacks.lock());
-        interruptCallbacks->callbacks.erase(token);
-    }
-};
-
-std::unique_ptr<InterruptCallback> createInterruptCallback(std::function<void()> callback)
-{
-    auto interruptCallbacks(_interruptCallbacks.lock());
-    auto token = interruptCallbacks->nextToken++;
-    interruptCallbacks->callbacks.emplace(token, callback);
-
-    auto res = std::make_unique<InterruptCallbackImpl>();
-    res->token = token;
-
-    return std::unique_ptr<InterruptCallback>(res.release());
-}
-
-
-AutoCloseFD createUnixDomainSocket()
-{
-    AutoCloseFD fdSocket = socket(PF_UNIX, SOCK_STREAM
-        #ifdef SOCK_CLOEXEC
-        | SOCK_CLOEXEC
-        #endif
-        , 0);
-    if (!fdSocket)
-        throw SysError("cannot create Unix domain socket");
-    closeOnExec(fdSocket.get());
-    return fdSocket;
-}
-
-
-AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode)
-{
-    auto fdSocket = nix::createUnixDomainSocket();
-
-    bind(fdSocket.get(), path);
-
-    if (chmod(path.c_str(), mode) == -1)
-        throw SysError("changing permissions on '%1%'", path);
-
-    if (listen(fdSocket.get(), 100) == -1)
-        throw SysError("cannot listen on socket '%1%'", path);
-
-    return fdSocket;
-}
-
-
-void bind(int fd, const std::string & path)
-{
-    unlink(path.c_str());
-
-    struct sockaddr_un addr;
-    addr.sun_family = AF_UNIX;
-
-    if (path.size() + 1 >= sizeof(addr.sun_path)) {
-        Pid pid = startProcess([&]() {
-            Path dir = dirOf(path);
-            if (chdir(dir.c_str()) == -1)
-                throw SysError("chdir to '%s' failed", dir);
-            std::string base(baseNameOf(path));
-            if (base.size() + 1 >= sizeof(addr.sun_path))
-                throw Error("socket path '%s' is too long", base);
-            memcpy(addr.sun_path, base.c_str(), base.size() + 1);
-            if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
-                throw SysError("cannot bind to socket '%s'", path);
-            _exit(0);
-        });
-        int status = pid.wait();
-        if (status != 0)
-            throw Error("cannot bind to socket '%s'", path);
-    } else {
-        memcpy(addr.sun_path, path.c_str(), path.size() + 1);
-        if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
-            throw SysError("cannot bind to socket '%s'", path);
-    }
-}
-
-
-void connect(int fd, const std::string & path)
-{
-    struct sockaddr_un addr;
-    addr.sun_family = AF_UNIX;
-
-    if (path.size() + 1 >= sizeof(addr.sun_path)) {
-        Pid pid = startProcess([&]() {
-            Path dir = dirOf(path);
-            if (chdir(dir.c_str()) == -1)
-                throw SysError("chdir to '%s' failed", dir);
-            std::string base(baseNameOf(path));
-            if (base.size() + 1 >= sizeof(addr.sun_path))
-                throw Error("socket path '%s' is too long", base);
-            memcpy(addr.sun_path, base.c_str(), base.size() + 1);
-            if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
-                throw SysError("cannot connect to socket at '%s'", path);
-            _exit(0);
-        });
-        int status = pid.wait();
-        if (status != 0)
-            throw Error("cannot connect to socket at '%s'", path);
-    } else {
-        memcpy(addr.sun_path, path.c_str(), path.size() + 1);
-        if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
-            throw SysError("cannot connect to socket at '%s'", path);
-    }
-}
-
-
 std::string showBytes(uint64_t bytes)
 {
     return fmt("%.2f MiB", bytes / (1024.0 * 1024.0));
 }
 
-
-// FIXME: move to libstore/build
-void commonChildInit()
-{
-    logger = makeSimpleLogger();
-
-    const static std::string pathNullDevice = "/dev/null";
-    restoreProcessContext(false);
-
-    /* Put the child in a separate session (and thus a separate
-       process group) so that it has no controlling terminal (meaning
-       that e.g. ssh cannot open /dev/tty) and it doesn't receive
-       terminal signals. */
-    if (setsid() == -1)
-        throw SysError("creating a new session");
-
-    /* Dup stderr to stdout. */
-    if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1)
-        throw SysError("cannot dup stderr into stdout");
-
-    /* Reroute stdin to /dev/null. */
-    int fdDevNull = open(pathNullDevice.c_str(), O_RDWR);
-    if (fdDevNull == -1)
-        throw SysError("cannot open '%1%'", pathNullDevice);
-    if (dup2(fdDevNull, STDIN_FILENO) == -1)
-        throw SysError("cannot dup null device into stdin");
-    close(fdDevNull);
-}
-
 }
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 75683f8fe..5f730eaf6 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -4,485 +4,18 @@
 #include "types.hh"
 #include "error.hh"
 #include "logging.hh"
-#include "ansicolor.hh"
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <dirent.h>
-#include <unistd.h>
-#include <signal.h>
 
 #include <boost/lexical_cast.hpp>
 
-#include <atomic>
 #include <functional>
 #include <map>
 #include <sstream>
 #include <optional>
 
-#ifndef HAVE_STRUCT_DIRENT_D_TYPE
-#define DT_UNKNOWN 0
-#define DT_REG 1
-#define DT_LNK 2
-#define DT_DIR 3
-#endif
-
 namespace nix {
 
-struct Sink;
-struct Source;
-
 void initLibUtil();
 
-/**
- * @return an environment variable.
- */
-std::optional<std::string> getEnv(const std::string & key);
-
-/**
- * @return a non empty environment variable. Returns nullopt if the env
- * variable is set to ""
- */
-std::optional<std::string> getEnvNonEmpty(const std::string & key);
-
-/**
- * Get the entire environment.
- */
-std::map<std::string, std::string> getEnv();
-
-/**
- * Clear the environment.
- */
-void clearEnv();
-
-/**
- * @return An absolutized path, resolving paths relative to the
- * specified directory, or the current directory otherwise.  The path
- * is also canonicalised.
- */
-Path absPath(Path path,
-    std::optional<PathView> dir = {},
-    bool resolveSymlinks = false);
-
-/**
- * Canonicalise a path by removing all `.` or `..` components and
- * double or trailing slashes.  Optionally resolves all symlink
- * components such that each component of the resulting path is *not*
- * a symbolic link.
- */
-Path canonPath(PathView path, bool resolveSymlinks = false);
-
-/**
- * @return The directory part of the given canonical path, i.e.,
- * everything before the final `/`.  If the path is the root or an
- * immediate child thereof (e.g., `/foo`), this means `/`
- * is returned.
- */
-Path dirOf(const PathView path);
-
-/**
- * @return the base name of the given canonical path, i.e., everything
- * following the final `/` (trailing slashes are removed).
- */
-std::string_view baseNameOf(std::string_view path);
-
-/**
- * Perform tilde expansion on a path.
- */
-std::string expandTilde(std::string_view path);
-
-/**
- * Check whether 'path' is a descendant of 'dir'. Both paths must be
- * canonicalized.
- */
-bool isInDir(std::string_view path, std::string_view dir);
-
-/**
- * Check whether 'path' is equal to 'dir' or a descendant of
- * 'dir'. Both paths must be canonicalized.
- */
-bool isDirOrInDir(std::string_view path, std::string_view dir);
-
-/**
- * Get status of `path`.
- */
-struct stat stat(const Path & path);
-struct stat lstat(const Path & path);
-
-/**
- * @return true iff the given path exists.
- */
-bool pathExists(const Path & path);
-
-/**
- * A version of pathExists that returns false on a permission error.
- * Useful for inferring default paths across directories that might not
- * be readable.
- * @return true iff the given path can be accessed and exists
- */
-bool pathAccessible(const Path & path);
-
-/**
- * Read the contents (target) of a symbolic link.  The result is not
- * in any way canonicalised.
- */
-Path readLink(const Path & path);
-
-bool isLink(const Path & path);
-
-/**
- * Read the contents of a directory.  The entries `.` and `..` are
- * removed.
- */
-struct DirEntry
-{
-    std::string name;
-    ino_t ino;
-    /**
-     * one of DT_*
-     */
-    unsigned char type;
-    DirEntry(std::string name, ino_t ino, unsigned char type)
-        : name(std::move(name)), ino(ino), type(type) { }
-};
-
-typedef std::vector<DirEntry> DirEntries;
-
-DirEntries readDirectory(const Path & path);
-
-unsigned char getFileType(const Path & path);
-
-/**
- * Read the contents of a file into a string.
- */
-std::string readFile(int fd);
-std::string readFile(const Path & path);
-void readFile(const Path & path, Sink & sink);
-
-/**
- * Write a string to a file.
- */
-void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false);
-
-void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false);
-
-/**
- * Flush a file's parent directory to disk
- */
-void syncParent(const Path & path);
-
-/**
- * Read a line from a file descriptor.
- */
-std::string readLine(int fd);
-
-/**
- * Write a line to a file descriptor.
- */
-void writeLine(int fd, std::string s);
-
-/**
- * Delete a path; i.e., in the case of a directory, it is deleted
- * recursively. It's not an error if the path does not exist. The
- * second variant returns the number of bytes and blocks freed.
- */
-void deletePath(const Path & path);
-
-void deletePath(const Path & path, uint64_t & bytesFreed);
-
-std::string getUserName();
-
-/**
- * @return the given user's home directory from /etc/passwd.
- */
-Path getHomeOf(uid_t userId);
-
-/**
- * @return $HOME or the user's home directory from /etc/passwd.
- */
-Path getHome();
-
-/**
- * @return $XDG_CACHE_HOME or $HOME/.cache.
- */
-Path getCacheDir();
-
-/**
- * @return $XDG_CONFIG_HOME or $HOME/.config.
- */
-Path getConfigDir();
-
-/**
- * @return the directories to search for user configuration files
- */
-std::vector<Path> getConfigDirs();
-
-/**
- * @return $XDG_DATA_HOME or $HOME/.local/share.
- */
-Path getDataDir();
-
-/**
- * @return the path of the current executable.
- */
-std::optional<Path> getSelfExe();
-
-/**
- * @return $XDG_STATE_HOME or $HOME/.local/state.
- */
-Path getStateDir();
-
-/**
- * Create the Nix state directory and return the path to it.
- */
-Path createNixStateDir();
-
-/**
- * Create a directory and all its parents, if necessary.  Returns the
- * list of created directories, in order of creation.
- */
-Paths createDirs(const Path & path);
-inline Paths createDirs(PathView path)
-{
-    return createDirs(Path(path));
-}
-
-/**
- * Create a symlink.
- */
-void createSymlink(const Path & target, const Path & link);
-
-/**
- * Atomically create or replace a symlink.
- */
-void replaceSymlink(const Path & target, const Path & link);
-
-void renameFile(const Path & src, const Path & dst);
-
-/**
- * Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst`
- * are on a different filesystem.
- *
- * Beware that this might not be atomic because of the copy that happens behind
- * the scenes
- */
-void moveFile(const Path & src, const Path & dst);
-
-
-/**
- * Wrappers arount read()/write() that read/write exactly the
- * requested number of bytes.
- */
-void readFull(int fd, char * buf, size_t count);
-void writeFull(int fd, std::string_view s, bool allowInterrupts = true);
-
-MakeError(EndOfFile, Error);
-
-
-/**
- * Read a file descriptor until EOF occurs.
- */
-std::string drainFD(int fd, bool block = true, const size_t reserveSize=0);
-
-void drainFD(int fd, Sink & sink, bool block = true);
-
-/**
- * If cgroups are active, attempt to calculate the number of CPUs available.
- * If cgroups are unavailable or if cpu.max is set to "max", return 0.
- */
-unsigned int getMaxCPU();
-
-/**
- * Automatic cleanup of resources.
- */
-
-
-class AutoDelete
-{
-    Path path;
-    bool del;
-    bool recursive;
-public:
-    AutoDelete();
-    AutoDelete(const Path & p, bool recursive = true);
-    ~AutoDelete();
-    void cancel();
-    void reset(const Path & p, bool recursive = true);
-    operator Path() const { return path; }
-    operator PathView() const { return path; }
-};
-
-
-class AutoCloseFD
-{
-    int fd;
-public:
-    AutoCloseFD();
-    AutoCloseFD(int fd);
-    AutoCloseFD(const AutoCloseFD & fd) = delete;
-    AutoCloseFD(AutoCloseFD&& fd);
-    ~AutoCloseFD();
-    AutoCloseFD& operator =(const AutoCloseFD & fd) = delete;
-    AutoCloseFD& operator =(AutoCloseFD&& fd);
-    int get() const;
-    explicit operator bool() const;
-    int release();
-    void close();
-    void fsync();
-};
-
-
-/**
- * Create a temporary directory.
- */
-Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix",
-    bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755);
-
-/**
- * Create a temporary file, returning a file handle and its path.
- */
-std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix = "nix");
-
-
-class Pipe
-{
-public:
-    AutoCloseFD readSide, writeSide;
-    void create();
-    void close();
-};
-
-
-struct DIRDeleter
-{
-    void operator()(DIR * dir) const {
-        closedir(dir);
-    }
-};
-
-typedef std::unique_ptr<DIR, DIRDeleter> AutoCloseDir;
-
-
-class Pid
-{
-    pid_t pid = -1;
-    bool separatePG = false;
-    int killSignal = SIGKILL;
-public:
-    Pid();
-    Pid(pid_t pid);
-    ~Pid();
-    void operator =(pid_t pid);
-    operator pid_t();
-    int kill();
-    int wait();
-
-    void setSeparatePG(bool separatePG);
-    void setKillSignal(int signal);
-    pid_t release();
-};
-
-
-/**
- * Kill all processes running under the specified uid by sending them
- * a SIGKILL.
- */
-void killUser(uid_t uid);
-
-
-/**
- * Fork a process that runs the given function, and return the child
- * pid to the caller.
- */
-struct ProcessOptions
-{
-    std::string errorPrefix = "";
-    bool dieWithParent = true;
-    bool runExitHandlers = false;
-    bool allowVfork = false;
-    /**
-     * use clone() with the specified flags (Linux only)
-     */
-    int cloneFlags = 0;
-};
-
-pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = ProcessOptions());
-
-
-/**
- * Run a program and return its stdout in a string (i.e., like the
- * shell backtick operator).
- */
-std::string runProgram(Path program, bool searchPath = false,
-    const Strings & args = Strings(),
-    const std::optional<std::string> & input = {}, bool isInteractive = false);
-
-struct RunOptions
-{
-    Path program;
-    bool searchPath = true;
-    Strings args;
-    std::optional<uid_t> uid;
-    std::optional<uid_t> gid;
-    std::optional<Path> chdir;
-    std::optional<std::map<std::string, std::string>> environment;
-    std::optional<std::string> input;
-    Source * standardIn = nullptr;
-    Sink * standardOut = nullptr;
-    bool mergeStderrToStdout = false;
-    bool isInteractive = false;
-};
-
-std::pair<int, std::string> runProgram(RunOptions && options);
-
-void runProgram2(const RunOptions & options);
-
-
-/**
- * Change the stack size.
- */
-void setStackSize(size_t stackSize);
-
-
-/**
- * Restore the original inherited Unix process context (such as signal
- * masks, stack size).
-
- * See startSignalHandlerThread(), saveSignalMask().
- */
-void restoreProcessContext(bool restoreMounts = true);
-
-/**
- * Save the current mount namespace. Ignored if called more than
- * once.
- */
-void saveMountNamespace();
-
-/**
- * Restore the mount namespace saved by saveMountNamespace(). Ignored
- * if saveMountNamespace() was never called.
- */
-void restoreMountNamespace();
-
-/**
- * Cause this thread to not share any FS attributes with the main
- * thread, because this causes setns() in restoreMountNamespace() to
- * fail.
- */
-void unshareFilesystem();
-
-
-class ExecError : public Error
-{
-public:
-    int status;
-
-    template<typename... Args>
-    ExecError(int status, const Args & ... args)
-        : Error(args...), status(status)
-    { }
-};
-
 /**
  * Convert a list of strings to a null-terminated vector of `char
  * *`s. The result must not be accessed beyond the lifetime of the
@@ -490,36 +23,6 @@ public:
  */
 std::vector<char *> stringsToCharPtrs(const Strings & ss);
 
-/**
- * Close all file descriptors except those listed in the given set.
- * Good practice in child processes.
- */
-void closeMostFDs(const std::set<int> & exceptions);
-
-/**
- * Set the close-on-exec flag for the given file descriptor.
- */
-void closeOnExec(int fd);
-
-
-/* User interruption. */
-
-extern std::atomic<bool> _isInterrupted;
-
-extern thread_local std::function<bool()> interruptCheck;
-
-void setInterruptThrown();
-
-void _interrupted();
-
-void inline checkInterrupt()
-{
-    if (_isInterrupted || (interruptCheck && interruptCheck()))
-        _interrupted();
-}
-
-MakeError(Interrupted, BaseError);
-
 
 MakeError(FormatError, Error);
 
@@ -595,15 +98,6 @@ std::string replaceStrings(
 std::string rewriteStrings(std::string s, const StringMap & rewrites);
 
 
-/**
- * Convert the exit status of a child as returned by wait() into an
- * error string.
- */
-std::string statusToString(int status);
-
-bool statusOk(int status);
-
-
 /**
  * Parse a string into an integer.
  */
@@ -711,23 +205,6 @@ constexpr char treeLast[] = "└───";
 constexpr char treeLine[] = "│   ";
 constexpr char treeNull[] = "    ";
 
-/**
- * Determine whether ANSI escape sequences are appropriate for the
- * present output.
- */
-bool shouldANSI();
-
-/**
- * Truncate a string to 'width' printable characters. If 'filterAll'
- * is true, all ANSI escape sequences are filtered out. Otherwise,
- * some escape sequences (such as colour setting) are copied but not
- * included in the character count. Also, tabs are expanded to
- * spaces.
- */
-std::string filterANSIEscapes(std::string_view s,
-    bool filterAll = false,
-    unsigned int width = std::numeric_limits<unsigned int>::max());
-
 
 /**
  * Base64 encoding/decoding.
@@ -815,61 +292,6 @@ template<typename T>
 class Callback;
 
 
-/**
- * Start a thread that handles various signals. Also block those signals
- * on the current thread (and thus any threads created by it).
- * Saves the signal mask before changing the mask to block those signals.
- * See saveSignalMask().
- */
-void startSignalHandlerThread();
-
-/**
- * Saves the signal mask, which is the signal mask that nix will restore
- * before creating child processes.
- * See setChildSignalMask() to set an arbitrary signal mask instead of the
- * current mask.
- */
-void saveSignalMask();
-
-/**
- * Sets the signal mask. Like saveSignalMask() but for a signal set that doesn't
- * necessarily match the current thread's mask.
- * See saveSignalMask() to set the saved mask to the current mask.
- */
-void setChildSignalMask(sigset_t *sigs);
-
-struct InterruptCallback
-{
-    virtual ~InterruptCallback() { };
-};
-
-/**
- * Register a function that gets called on SIGINT (in a non-signal
- * context).
- */
-std::unique_ptr<InterruptCallback> createInterruptCallback(
-    std::function<void()> callback);
-
-void triggerInterrupt();
-
-/**
- * A RAII class that causes the current thread to receive SIGUSR1 when
- * the signal handler thread receives SIGINT. That is, this allows
- * SIGINT to be multiplexed to multiple threads.
- */
-struct ReceiveInterrupts
-{
-    pthread_t target;
-    std::unique_ptr<InterruptCallback> callback;
-
-    ReceiveInterrupts()
-        : target(pthread_self())
-        , callback(createInterruptCallback([&]() { pthread_kill(target, SIGUSR1); }))
-    { }
-};
-
-
-
 /**
  * A RAII helper that increments a counter on construction and
  * decrements it on destruction.
@@ -884,45 +306,6 @@ struct MaintainCount
 };
 
 
-/**
- * @return the number of rows and columns of the terminal.
- */
-std::pair<unsigned short, unsigned short> getWindowSize();
-
-
-/**
- * Used in various places.
- */
-typedef std::function<bool(const Path & path)> PathFilter;
-
-extern PathFilter defaultPathFilter;
-
-/**
- * Common initialisation performed in child processes.
- */
-void commonChildInit();
-
-/**
- * Create a Unix domain socket.
- */
-AutoCloseFD createUnixDomainSocket();
-
-/**
- * Create a Unix domain socket in listen mode.
- */
-AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode);
-
-/**
- * Bind a Unix domain socket to a path.
- */
-void bind(int fd, const std::string & path);
-
-/**
- * Connect to a Unix domain socket.
- */
-void connect(int fd, const std::string & path);
-
-
 /**
  * A Rust/Python-like enumerate() iterator adapter.
  *
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 60bc08146..75ce12a8c 100644
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -9,12 +9,12 @@
 
 #include <nlohmann/json.hpp>
 
+#include "current-process.hh"
 #include "parsed-derivations.hh"
 #include "store-api.hh"
 #include "local-fs-store.hh"
 #include "globals.hh"
 #include "derivations.hh"
-#include "util.hh"
 #include "shared.hh"
 #include "path-with-outputs.hh"
 #include "eval.hh"
diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index 4504441fa..79db78236 100644
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -5,7 +5,7 @@
 #include "store-api.hh"
 #include "legacy.hh"
 #include "eval-settings.hh" // for defexpr
-#include "util.hh"
+#include "users.hh"
 #include "tarball.hh"
 
 #include <fcntl.h>
diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc
index 70af53b28..bb3f1bc6a 100644
--- a/src/nix-collect-garbage/nix-collect-garbage.cc
+++ b/src/nix-collect-garbage/nix-collect-garbage.cc
@@ -1,3 +1,5 @@
+#include "file-system.hh"
+#include "signals.hh"
 #include "store-api.hh"
 #include "store-cast.hh"
 #include "gc-store.hh"
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 25068f801..213a20d93 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -1,3 +1,4 @@
+#include "users.hh"
 #include "attr-path.hh"
 #include "common-eval-args.hh"
 #include "derivations.hh"
@@ -11,7 +12,6 @@
 #include "store-api.hh"
 #include "local-fs-store.hh"
 #include "user-env.hh"
-#include "util.hh"
 #include "value-to-json.hh"
 #include "xml-writer.hh"
 #include "legacy.hh"
diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc
index d12d70f33..250224e7d 100644
--- a/src/nix-env/user-env.cc
+++ b/src/nix-env/user-env.cc
@@ -1,5 +1,4 @@
 #include "user-env.hh"
-#include "util.hh"
 #include "derivations.hh"
 #include "store-api.hh"
 #include "path-with-outputs.hh"
diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc
index d40196497..c67409e89 100644
--- a/src/nix-instantiate/nix-instantiate.cc
+++ b/src/nix-instantiate/nix-instantiate.cc
@@ -6,7 +6,6 @@
 #include "attr-path.hh"
 #include "value-to-xml.hh"
 #include "value-to-json.hh"
-#include "util.hh"
 #include "store-api.hh"
 #include "local-fs-store.hh"
 #include "common-eval-args.hh"
diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc
index 577cadceb..2c530999b 100644
--- a/src/nix-store/dotgraph.cc
+++ b/src/nix-store/dotgraph.cc
@@ -1,5 +1,4 @@
 #include "dotgraph.hh"
-#include "util.hh"
 #include "store-api.hh"
 
 #include <iostream>
diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc
index 439557658..3e789a2d8 100644
--- a/src/nix-store/graphml.cc
+++ b/src/nix-store/graphml.cc
@@ -1,5 +1,4 @@
 #include "graphml.hh"
-#include "util.hh"
 #include "store-api.hh"
 #include "derivations.hh"
 
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index e4dd94585..123283dfe 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -11,7 +11,6 @@
 #include "serve-protocol.hh"
 #include "serve-protocol-impl.hh"
 #include "shared.hh"
-#include "util.hh"
 #include "graphml.hh"
 #include "legacy.hh"
 #include "path-with-outputs.hh"
diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc
index af428018a..373dedf7c 100644
--- a/src/nix/daemon.cc
+++ b/src/nix/daemon.cc
@@ -1,11 +1,12 @@
 ///@file
 
+#include "signals.hh"
+#include "unix-domain-socket.hh"
 #include "command.hh"
 #include "shared.hh"
 #include "local-store.hh"
 #include "remote-store.hh"
 #include "remote-store-connection.hh"
-#include "util.hh"
 #include "serialise.hh"
 #include "archive.hh"
 #include "globals.hh"
diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index b080a3939..38482ed42 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -8,7 +8,6 @@
 #include "derivations.hh"
 #include "progress-bar.hh"
 #include "run.hh"
-#include "util.hh"
 
 #include <iterator>
 #include <memory>
diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc
index 1aa6831d3..59f9e3e5d 100644
--- a/src/nix/doctor.cc
+++ b/src/nix/doctor.cc
@@ -6,7 +6,6 @@
 #include "shared.hh"
 #include "store-api.hh"
 #include "local-fs-store.hh"
-#include "util.hh"
 #include "worker-protocol.hh"
 
 using namespace nix;
diff --git a/src/nix/edit.cc b/src/nix/edit.cc
index 66629fab0..9cbab230b 100644
--- a/src/nix/edit.cc
+++ b/src/nix/edit.cc
@@ -1,3 +1,4 @@
+#include "current-process.hh"
 #include "command-installable-value.hh"
 #include "shared.hh"
 #include "eval.hh"
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index e8906a252..38938f09e 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -15,6 +15,7 @@
 #include "registry.hh"
 #include "eval-cache.hh"
 #include "markdown.hh"
+#include "users.hh"
 
 #include <nlohmann/json.hpp>
 #include <queue>
diff --git a/src/nix/main.cc b/src/nix/main.cc
index d20bc1f8a..b582fc166 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -1,6 +1,8 @@
 #include <algorithm>
 
 #include "args/root.hh"
+#include "current-process.hh"
+#include "namespaces.hh"
 #include "command.hh"
 #include "common-args.hh"
 #include "eval.hh"
diff --git a/src/nix/run.cc b/src/nix/run.cc
index 1465e8cde..ea0a17897 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -1,3 +1,4 @@
+#include "current-process.hh"
 #include "run.hh"
 #include "command-installable-value.hh"
 #include "common-args.hh"
diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc
index 45cd2e1a6..a68616355 100644
--- a/src/nix/sigs.cc
+++ b/src/nix/sigs.cc
@@ -1,3 +1,4 @@
+#include "signals.hh"
 #include "command.hh"
 #include "shared.hh"
 #include "store-api.hh"
diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc
index d238456db..c529c2363 100644
--- a/src/nix/upgrade-nix.cc
+++ b/src/nix/upgrade-nix.cc
@@ -1,3 +1,4 @@
+#include "processes.hh"
 #include "command.hh"
 #include "common-args.hh"
 #include "store-api.hh"
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index adaa33c0c..78cb765ce 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -4,6 +4,7 @@
 #include "sync.hh"
 #include "thread-pool.hh"
 #include "references.hh"
+#include "signals.hh"
 
 #include <atomic>
 

From 6472c3bf0d4b529f28f9e50834e1fc3dd101c409 Mon Sep 17 00:00:00 2001
From: ThinkChaos <ThinkChaos@users.noreply.github.com>
Date: Sat, 4 Nov 2023 12:08:00 -0400
Subject: [PATCH 063/294] fix(ssh): extraneous master processes

---
 src/libstore/ssh.cc | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc
index 03b2f0be9..300eb391c 100644
--- a/src/libstore/ssh.cc
+++ b/src/libstore/ssh.cc
@@ -132,7 +132,6 @@ Path SSHMaster::startMaster()
 
     if (state->sshMaster != -1) return state->socketPath;
 
-
     state->socketPath = (Path) *state->tmpDir + "/ssh.sock";
 
     Pipe out;
@@ -144,7 +143,8 @@ Path SSHMaster::startMaster()
     logger->pause();
     Finally cleanup = [&]() { logger->resume(); };
 
-    bool wasMasterRunning = isMasterRunning();
+    if (isMasterRunning())
+        return state->socketPath;
 
     state->sshMaster = startProcess([&]() {
         restoreProcessContext();
@@ -165,14 +165,13 @@ Path SSHMaster::startMaster()
 
     out.writeSide = -1;
 
-    if (!wasMasterRunning) {
-        std::string reply;
-        try {
-            reply = readLine(out.readSide.get());
-        } catch (EndOfFile & e) { }
+    std::string reply;
+    try {
+        reply = readLine(out.readSide.get());
+    } catch (EndOfFile & e) { }
 
-        if (reply != "started")
-            throw Error("failed to start SSH master connection to '%s'", host);
+    if (reply != "started") {
+        throw Error("failed to start SSH master connection to '%s'", host);
     }
 
     return state->socketPath;

From 2fb49759b8307838dd1208d8ce756a60d41e4ebf Mon Sep 17 00:00:00 2001
From: ThinkChaos <ThinkChaos@users.noreply.github.com>
Date: Sat, 4 Nov 2023 12:32:57 -0400
Subject: [PATCH 064/294] fix(ssh): log first line of stdout

Spent a while debugging why `nix-copy-closure` wasn't working anymore
and it was my shell RC printing something I added for debug.
Hopefully this can save someone else some time.
---
 src/libstore/ssh.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc
index 300eb391c..5c8d6a504 100644
--- a/src/libstore/ssh.cc
+++ b/src/libstore/ssh.cc
@@ -114,8 +114,10 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
             reply = readLine(out.readSide.get());
         } catch (EndOfFile & e) { }
 
-        if (reply != "started")
+        if (reply != "started") {
+            printTalkative("SSH stdout first line: %s", reply);
             throw Error("failed to start SSH connection to '%s'", host);
+        }
     }
 
     conn->out = std::move(out.readSide);
@@ -171,6 +173,7 @@ Path SSHMaster::startMaster()
     } catch (EndOfFile & e) { }
 
     if (reply != "started") {
+        printTalkative("SSH master stdout first line: %s", reply);
         throw Error("failed to start SSH master connection to '%s'", host);
     }
 

From 0b0d1b521449e7a66e7fa33ca7afe292d88aa14b Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 26 Oct 2023 19:39:21 -0400
Subject: [PATCH 065/294] Add comparison functions for `NarInfo`

We will need these for tests.
---
 src/libstore/nar-info.cc | 9 +++++++++
 src/libstore/nar-info.hh | 2 ++
 2 files changed, 11 insertions(+)

diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc
index ee2ddfd81..2b77c6ab7 100644
--- a/src/libstore/nar-info.cc
+++ b/src/libstore/nar-info.cc
@@ -4,6 +4,15 @@
 
 namespace nix {
 
+GENERATE_CMP_EXT(
+    ,
+    NarInfo,
+    me->url,
+    me->compression,
+    me->fileHash,
+    me->fileSize,
+    static_cast<const ValidPathInfo &>(*me));
+
 NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence)
     : ValidPathInfo(StorePath(StorePath::dummy), Hash(Hash::dummy)) // FIXME: hack
 {
diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh
index 5dbdafac3..1b3551106 100644
--- a/src/libstore/nar-info.hh
+++ b/src/libstore/nar-info.hh
@@ -24,6 +24,8 @@ struct NarInfo : ValidPathInfo
     NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { }
     NarInfo(const Store & store, const std::string & s, const std::string & whence);
 
+    DECLARE_CMP(NarInfo);
+
     std::string to_string(const Store & store) const;
 };
 

From 07ac53732b8989758c264d4e847c94a5d28072cf Mon Sep 17 00:00:00 2001
From: Jacek Galowicz <jacek@galowicz.de>
Date: Sun, 5 Nov 2023 15:27:25 +0100
Subject: [PATCH 066/294] Fix moves in appendOrSet

---
 src/libutil/config-impl.hh | 10 +++++-----
 src/libutil/config.cc      | 20 +++++++++-----------
 src/libutil/config.hh      |  2 +-
 3 files changed, 15 insertions(+), 17 deletions(-)

diff --git a/src/libutil/config-impl.hh b/src/libutil/config-impl.hh
index b9639e761..9f69e8444 100644
--- a/src/libutil/config-impl.hh
+++ b/src/libutil/config-impl.hh
@@ -45,13 +45,13 @@ bool BaseSetting<T>::isAppendable()
     return trait::appendable;
 }
 
-template<> void BaseSetting<Strings>::appendOrSet(Strings && newValue, bool append);
-template<> void BaseSetting<StringSet>::appendOrSet(StringSet && newValue, bool append);
-template<> void BaseSetting<StringMap>::appendOrSet(StringMap && newValue, bool append);
-template<> void BaseSetting<std::set<ExperimentalFeature>>::appendOrSet(std::set<ExperimentalFeature> && newValue, bool append);
+template<> void BaseSetting<Strings>::appendOrSet(Strings newValue, bool append);
+template<> void BaseSetting<StringSet>::appendOrSet(StringSet newValue, bool append);
+template<> void BaseSetting<StringMap>::appendOrSet(StringMap newValue, bool append);
+template<> void BaseSetting<std::set<ExperimentalFeature>>::appendOrSet(std::set<ExperimentalFeature> newValue, bool append);
 
 template<typename T>
-void BaseSetting<T>::appendOrSet(T && newValue, bool append)
+void BaseSetting<T>::appendOrSet(T newValue, bool append)
 {
     static_assert(
         !trait::appendable,
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 0bf36c987..5b510b69e 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -301,10 +301,11 @@ template<> Strings BaseSetting<Strings>::parse(const std::string & str) const
     return tokenizeString<Strings>(str);
 }
 
-template<> void BaseSetting<Strings>::appendOrSet(Strings && newValue, bool append)
+template<> void BaseSetting<Strings>::appendOrSet(Strings newValue, bool append)
 {
     if (!append) value.clear();
-    for (auto && s : std::move(newValue)) value.push_back(std::move(s));
+    value.insert(value.end(), std::make_move_iterator(newValue.begin()),
+                              std::make_move_iterator(newValue.end()));
 }
 
 template<> std::string BaseSetting<Strings>::to_string() const
@@ -317,11 +318,10 @@ template<> StringSet BaseSetting<StringSet>::parse(const std::string & str) cons
     return tokenizeString<StringSet>(str);
 }
 
-template<> void BaseSetting<StringSet>::appendOrSet(StringSet && newValue, bool append)
+template<> void BaseSetting<StringSet>::appendOrSet(StringSet newValue, bool append)
 {
     if (!append) value.clear();
-    for (auto && s : std::move(newValue))
-        value.insert(s);
+    value.insert(std::make_move_iterator(newValue.begin()), std::make_move_iterator(newValue.end()));
 }
 
 template<> std::string BaseSetting<StringSet>::to_string() const
@@ -342,11 +342,10 @@ template<> std::set<ExperimentalFeature> BaseSetting<std::set<ExperimentalFeatur
     return res;
 }
 
-template<> void BaseSetting<std::set<ExperimentalFeature>>::appendOrSet(std::set<ExperimentalFeature> && newValue, bool append)
+template<> void BaseSetting<std::set<ExperimentalFeature>>::appendOrSet(std::set<ExperimentalFeature> newValue, bool append)
 {
     if (!append) value.clear();
-    for (auto && s : std::move(newValue))
-        value.insert(s);
+    value.insert(std::make_move_iterator(newValue.begin()), std::make_move_iterator(newValue.end()));
 }
 
 template<> std::string BaseSetting<std::set<ExperimentalFeature>>::to_string() const
@@ -369,11 +368,10 @@ template<> StringMap BaseSetting<StringMap>::parse(const std::string & str) cons
     return res;
 }
 
-template<> void BaseSetting<StringMap>::appendOrSet(StringMap && newValue, bool append)
+template<> void BaseSetting<StringMap>::appendOrSet(StringMap newValue, bool append)
 {
     if (!append) value.clear();
-    for (auto && [k, v] : std::move(newValue))
-        value.emplace(std::move(k), std::move(v));
+    value.insert(std::make_move_iterator(newValue.begin()), std::make_move_iterator(newValue.end()));
 }
 
 template<> std::string BaseSetting<StringMap>::to_string() const
diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index 38c3ce0c4..d9441fb63 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -247,7 +247,7 @@ protected:
      *
      * @param append Whether to append or overwrite.
      */
-    virtual void appendOrSet(T && newValue, bool append);
+    virtual void appendOrSet(T newValue, bool append);
 
 public:
 

From ad385f9ec44f8d845e994764c45876042c715946 Mon Sep 17 00:00:00 2001
From: Jacek Galowicz <jacek@galowicz.de>
Date: Sun, 5 Nov 2023 15:27:48 +0100
Subject: [PATCH 067/294] Minor improvements

---
 src/libutil/config.hh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index d9441fb63..3f2522c38 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -150,7 +150,7 @@ public:
         AbstractSetting * setting;
     };
 
-    typedef std::map<std::string, SettingData> Settings;
+    using Settings = std::map<std::string, SettingData>;
 
 private:
 
@@ -316,7 +316,7 @@ std::ostream & operator <<(std::ostream & str, const BaseSetting<T> & opt)
 }
 
 template<typename T>
-bool operator ==(const T & v1, const BaseSetting<T> & v2) { return v1 == (const T &) v2; }
+bool operator ==(const T & v1, const BaseSetting<T> & v2) { return v1 == static_cast<const T &>(v2); }
 
 template<typename T>
 class Setting : public BaseSetting<T>
@@ -329,7 +329,7 @@ public:
         const std::set<std::string> & aliases = {},
         const bool documentDefault = true,
         std::optional<ExperimentalFeature> experimentalFeature = std::nullopt)
-        : BaseSetting<T>(def, documentDefault, name, description, aliases, experimentalFeature)
+        : BaseSetting<T>(def, documentDefault, name, description, aliases, std::move(experimentalFeature))
     {
         options->addSetting(this);
     }

From f404e9b3b362a054219797df02bbe277de249f80 Mon Sep 17 00:00:00 2001
From: Jacek Galowicz <jacek@galowicz.de>
Date: Sun, 5 Nov 2023 16:12:20 +0100
Subject: [PATCH 068/294] Make toJSONObject const

---
 src/libutil/abstract-setting-to-json.hh | 2 +-
 src/libutil/config.cc                   | 2 +-
 src/libutil/config.hh                   | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/libutil/abstract-setting-to-json.hh b/src/libutil/abstract-setting-to-json.hh
index d506dfb74..eea687d8a 100644
--- a/src/libutil/abstract-setting-to-json.hh
+++ b/src/libutil/abstract-setting-to-json.hh
@@ -7,7 +7,7 @@
 
 namespace nix {
 template<typename T>
-std::map<std::string, nlohmann::json> BaseSetting<T>::toJSONObject()
+std::map<std::string, nlohmann::json> BaseSetting<T>::toJSONObject() const
 {
     auto obj = AbstractSetting::toJSONObject();
     obj.emplace("value", value);
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 5b510b69e..2a5cf6212 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -221,7 +221,7 @@ nlohmann::json AbstractSetting::toJSON()
     return nlohmann::json(toJSONObject());
 }
 
-std::map<std::string, nlohmann::json> AbstractSetting::toJSONObject()
+std::map<std::string, nlohmann::json> AbstractSetting::toJSONObject() const
 {
     std::map<std::string, nlohmann::json> obj;
     obj.emplace("description", description);
diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index 3f2522c38..5d7bd8e0c 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -213,7 +213,7 @@ protected:
 
     nlohmann::json toJSON();
 
-    virtual std::map<std::string, nlohmann::json> toJSONObject();
+    virtual std::map<std::string, nlohmann::json> toJSONObject() const;
 
     virtual void convertToArg(Args & args, const std::string & category);
 
@@ -306,7 +306,7 @@ public:
 
     void convertToArg(Args & args, const std::string & category) override;
 
-    std::map<std::string, nlohmann::json> toJSONObject() override;
+    std::map<std::string, nlohmann::json> toJSONObject() const override;
 };
 
 template<typename T>

From a4b7df7bfaee7d27a152be2445886c81881daf94 Mon Sep 17 00:00:00 2001
From: Jacek Galowicz <jacek@galowicz.de>
Date: Mon, 6 Nov 2023 15:47:25 +0100
Subject: [PATCH 069/294] More const, scope reductions, move fixes

---
 src/libutil/config.cc | 60 +++++++++++++++++++------------------------
 1 file changed, 27 insertions(+), 33 deletions(-)

diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 2a5cf6212..8e7901133 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -36,27 +36,25 @@ bool Config::set(const std::string & name, const std::string & value)
 void Config::addSetting(AbstractSetting * setting)
 {
     _settings.emplace(setting->name, Config::SettingData{false, setting});
-    for (auto & alias : setting->aliases)
+    for (const auto & alias : setting->aliases)
         _settings.emplace(alias, Config::SettingData{true, setting});
 
     bool set = false;
 
-    auto i = unknownSettings.find(setting->name);
-    if (i != unknownSettings.end()) {
-        setting->set(i->second);
+    if (auto i = unknownSettings.find(setting->name); i != unknownSettings.end()) {
+        setting->set(std::move(i->second));
         setting->overridden = true;
         unknownSettings.erase(i);
         set = true;
     }
 
     for (auto & alias : setting->aliases) {
-        auto i = unknownSettings.find(alias);
-        if (i != unknownSettings.end()) {
+        if (auto i = unknownSettings.find(alias); i != unknownSettings.end()) {
             if (set)
                 warn("setting '%s' is set, but it's an alias of '%s' which is also set",
                     alias, setting->name);
             else {
-                setting->set(i->second);
+                setting->set(std::move(i->second));
                 setting->overridden = true;
                 unknownSettings.erase(i);
                 set = true;
@@ -71,7 +69,7 @@ AbstractConfig::AbstractConfig(StringMap initials)
 
 void AbstractConfig::warnUnknownSettings()
 {
-    for (auto & s : unknownSettings)
+    for (const auto & s : unknownSettings)
         warn("unknown setting '%s'", s.first);
 }
 
@@ -85,7 +83,7 @@ void AbstractConfig::reapplyUnknownSettings()
 
 void Config::getSettings(std::map<std::string, SettingInfo> & res, bool overriddenOnly)
 {
-    for (auto & opt : _settings)
+    for (const auto & opt : _settings)
         if (!opt.second.isAlias && (!overriddenOnly || opt.second.setting->overridden))
             res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description});
 }
@@ -101,8 +99,7 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string
             line += contents[pos++];
         pos++;
 
-        auto hash = line.find('#');
-        if (hash != std::string::npos)
+        if (auto hash = line.find('#'); hash != line.npos)
             line = std::string(line, 0, hash);
 
         auto tokens = tokenizeString<std::vector<std::string>>(line);
@@ -135,24 +132,24 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string
         if (tokens[1] != "=")
             throw UsageError("illegal configuration line '%1%' in '%2%'", line, path);
 
-        std::string name = tokens[0];
+        std::string name = std::move(tokens[0]);
 
         auto i = tokens.begin();
         advance(i, 2);
 
         parsedContents.push_back({
-            name,
+            std::move(name),
             concatStringsSep(" ", Strings(i, tokens.end())),
         });
     };
 
     // First apply experimental-feature related settings
-    for (auto & [name, value] : parsedContents)
+    for (const auto & [name, value] : parsedContents)
         if (name == "experimental-features" || name == "extra-experimental-features")
             set(name, value);
 
     // Then apply other settings
-    for (auto & [name, value] : parsedContents)
+    for (const auto & [name, value] : parsedContents)
         if (name != "experimental-features" && name != "extra-experimental-features")
             set(name, value);
 }
@@ -174,7 +171,7 @@ void Config::resetOverridden()
 nlohmann::json Config::toJSON()
 {
     auto res = nlohmann::json::object();
-    for (auto & s : _settings)
+    for (const auto & s : _settings)
         if (!s.second.isAlias)
             res.emplace(s.first, s.second.setting->toJSON());
     return res;
@@ -182,8 +179,8 @@ nlohmann::json Config::toJSON()
 
 std::string Config::toKeyValue()
 {
-    auto res = std::string();
-    for (auto & s : _settings)
+    std::string res;
+    for (const auto & s : _settings)
         if (s.second.isAlias)
             res += fmt("%s = %s\n", s.first, s.second.setting->to_string());
     return res;
@@ -205,7 +202,7 @@ AbstractSetting::AbstractSetting(
     : name(name)
     , description(stripIndentation(description))
     , aliases(aliases)
-    , experimentalFeature(experimentalFeature)
+    , experimentalFeature(std::move(experimentalFeature))
 {
 }
 
@@ -284,14 +281,14 @@ template<> void BaseSetting<bool>::convertToArg(Args & args, const std::string &
         .longName = name,
         .description = fmt("Enable the `%s` setting.", name),
         .category = category,
-        .handler = {[this]() { override(true); }},
+        .handler = {[this] { override(true); }},
         .experimentalFeature = experimentalFeature,
     });
     args.addFlag({
         .longName = "no-" + name,
         .description = fmt("Disable the `%s` setting.", name),
         .category = category,
-        .handler = {[this]() { override(false); }},
+        .handler = {[this] { override(false); }},
         .experimentalFeature = experimentalFeature,
     });
 }
@@ -333,8 +330,7 @@ template<> std::set<ExperimentalFeature> BaseSetting<std::set<ExperimentalFeatur
 {
     std::set<ExperimentalFeature> res;
     for (auto & s : tokenizeString<StringSet>(str)) {
-        auto thisXpFeature = parseExperimentalFeature(s);
-        if (thisXpFeature)
+        if (auto thisXpFeature = parseExperimentalFeature(s); thisXpFeature)
             res.insert(thisXpFeature.value());
         else
             warn("unknown experimental feature '%s'", s);
@@ -351,7 +347,7 @@ template<> void BaseSetting<std::set<ExperimentalFeature>>::appendOrSet(std::set
 template<> std::string BaseSetting<std::set<ExperimentalFeature>>::to_string() const
 {
     StringSet stringifiedXpFeatures;
-    for (auto & feature : value)
+    for (const auto & feature : value)
         stringifiedXpFeatures.insert(std::string(showExperimentalFeature(feature)));
     return concatStringsSep(" ", stringifiedXpFeatures);
 }
@@ -359,9 +355,8 @@ template<> std::string BaseSetting<std::set<ExperimentalFeature>>::to_string() c
 template<> StringMap BaseSetting<StringMap>::parse(const std::string & str) const
 {
     StringMap res;
-    for (auto & s : tokenizeString<Strings>(str)) {
-        auto eq = s.find_first_of('=');
-        if (std::string::npos != eq)
+    for (const auto & s : tokenizeString<Strings>(str)) {
+        if (auto eq = s.find_first_of('='); s.npos != eq)
             res.emplace(std::string(s, 0, eq), std::string(s, eq + 1));
         // else ignored
     }
@@ -376,10 +371,9 @@ template<> void BaseSetting<StringMap>::appendOrSet(StringMap newValue, bool app
 
 template<> std::string BaseSetting<StringMap>::to_string() const
 {
-    Strings kvstrs;
-    std::transform(value.begin(), value.end(), back_inserter(kvstrs),
-        [&](auto kvpair){ return kvpair.first + "=" + kvpair.second; });
-    return concatStringsSep(" ", kvstrs);
+    return std::transform_reduce(value.cbegin(), value.cend(), std::string{},
+        [](const auto & l, const auto  &r) { return l + " " + r; },
+        [](const auto & kvpair){ return kvpair.first + "=" + kvpair.second; });
 }
 
 template class BaseSetting<int>;
@@ -468,7 +462,7 @@ void GlobalConfig::resetOverridden()
 nlohmann::json GlobalConfig::toJSON()
 {
     auto res = nlohmann::json::object();
-    for (auto & config : *configRegistrations)
+    for (const auto & config : *configRegistrations)
         res.update(config->toJSON());
     return res;
 }
@@ -478,7 +472,7 @@ std::string GlobalConfig::toKeyValue()
     std::string res;
     std::map<std::string, Config::SettingInfo> settings;
     globalConfig.getSettings(settings);
-    for (auto & s : settings)
+    for (const auto & s : settings)
         res += fmt("%s = %s\n", s.first, s.second.value);
     return res;
 }

From 937e02e7b9538fd4500ade184eb4f0a888a9967d Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 22 Oct 2023 21:12:54 -0400
Subject: [PATCH 070/294] Shuffle `ValidPathInfo` JSON rendering

`Store::pathInfoToJSON` was a rather baroque functions, being full of
parameters to support both parsed derivations and `nix path-info`. The
common core of each, a simple `dValidPathInfo::toJSON` function, is
factored out, but the rest of the logic is just duplicated and then
specialized to its use-case (at which point it is no longer that
duplicated).

This keeps the human oriented CLI logic (which is currently unstable)
and the core domain logic (export reference graphs with structured
attrs, which is stable), separate, which I think is better.
---
 src/libstore/nar-info.cc                      | 46 +++++++++
 src/libstore/nar-info.hh                      |  9 ++
 src/libstore/parsed-derivations.cc            | 34 ++++++-
 src/libstore/path-info.cc                     | 99 +++++++++++++++++++
 src/libstore/path-info.hh                     | 12 +++
 src/libstore/store-api.cc                     | 90 -----------------
 src/libstore/store-api.hh                     | 23 -----
 src/libstore/tests/nar-info.cc                | 84 ++++++++++++++++
 src/libstore/tests/path-info.cc               | 79 +++++++++++++++
 src/libutil/tests/characterization.hh         |  1 +
 src/nix/path-info.cc                          | 80 ++++++++++++++-
 unit-test-data/libstore/nar-info/impure.json  | 21 ++++
 unit-test-data/libstore/nar-info/pure.json    | 11 +++
 unit-test-data/libstore/path-info/impure.json | 18 ++++
 unit-test-data/libstore/path-info/pure.json   | 11 +++
 15 files changed, 499 insertions(+), 119 deletions(-)
 create mode 100644 src/libstore/tests/nar-info.cc
 create mode 100644 src/libstore/tests/path-info.cc
 create mode 100644 unit-test-data/libstore/nar-info/impure.json
 create mode 100644 unit-test-data/libstore/nar-info/pure.json
 create mode 100644 unit-test-data/libstore/path-info/impure.json
 create mode 100644 unit-test-data/libstore/path-info/pure.json

diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc
index 2b77c6ab7..a90812ff9 100644
--- a/src/libstore/nar-info.cc
+++ b/src/libstore/nar-info.cc
@@ -134,4 +134,50 @@ std::string NarInfo::to_string(const Store & store) const
     return res;
 }
 
+nlohmann::json NarInfo::toJSON(
+    const Store & store,
+    bool includeImpureInfo,
+    HashFormat hashFormat) const
+{
+    using nlohmann::json;
+
+    auto jsonObject = ValidPathInfo::toJSON(store, includeImpureInfo, hashFormat);
+
+    if (includeImpureInfo) {
+        if (!url.empty())
+            jsonObject["url"] = url;
+        if (fileHash)
+            jsonObject["downloadHash"] = fileHash->to_string(hashFormat, true);
+        if (fileSize)
+            jsonObject["downloadSize"] = fileSize;
+    }
+
+    return jsonObject;
+}
+
+NarInfo NarInfo::fromJSON(
+    const Store & store,
+    const StorePath & path,
+    const nlohmann::json & json)
+{
+    using nlohmann::detail::value_t;
+
+    NarInfo res { ValidPathInfo::fromJSON(store, json) };
+    res.path = path;
+
+    if (json.contains("url"))
+        res.url = ensureType(valueAt(json, "url"), value_t::string);
+
+    if (json.contains("downloadHash"))
+        res.fileHash = Hash::parseAny(
+            static_cast<const std::string &>(
+                ensureType(valueAt(json, "downloadHash"), value_t::string)),
+            std::nullopt);
+
+    if (json.contains("downloadSize"))
+        res.fileSize = ensureType(valueAt(json, "downloadSize"), value_t::number_integer);
+
+    return res;
+}
+
 }
diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh
index 1b3551106..cec65ff70 100644
--- a/src/libstore/nar-info.hh
+++ b/src/libstore/nar-info.hh
@@ -27,6 +27,15 @@ struct NarInfo : ValidPathInfo
     DECLARE_CMP(NarInfo);
 
     std::string to_string(const Store & store) const;
+
+    nlohmann::json toJSON(
+        const Store & store,
+        bool includeImpureInfo,
+        HashFormat hashFormat) const override;
+    static NarInfo fromJSON(
+        const Store & store,
+        const StorePath & path,
+        const nlohmann::json & json);
 };
 
 }
diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc
index 1d900c272..45629dc7f 100644
--- a/src/libstore/parsed-derivations.cc
+++ b/src/libstore/parsed-derivations.cc
@@ -132,6 +132,36 @@ bool ParsedDerivation::useUidRange() const
 
 static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
 
+/**
+ * Write a JSON representation of store object metadata, such as the
+ * hash and the references.
+ */
+static nlohmann::json pathInfoToJSON(
+    Store & store,
+    const StorePathSet & storePaths)
+{
+    nlohmann::json::array_t jsonList = nlohmann::json::array();
+
+    for (auto & storePath : storePaths) {
+        auto info = store.queryPathInfo(storePath);
+
+        auto & jsonPath = jsonList.emplace_back(
+            info->toJSON(store, false, HashFormat::Base32));
+
+        jsonPath["closureSize"] = ({
+            uint64_t totalNarSize = 0;
+            StorePathSet closure;
+            store.computeFSClosure(info->path, closure, false, false);
+            for (auto & p : closure) {
+                auto info = store.queryPathInfo(p);
+                totalNarSize += info->narSize;
+            }
+            totalNarSize;
+        });
+    }
+    return jsonList;
+}
+
 std::optional<nlohmann::json> ParsedDerivation::prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths)
 {
     auto structuredAttrs = getStructuredAttrs();
@@ -152,8 +182,8 @@ std::optional<nlohmann::json> ParsedDerivation::prepareStructuredAttrs(Store & s
             StorePathSet storePaths;
             for (auto & p : *i)
                 storePaths.insert(store.parseStorePath(p.get<std::string>()));
-            json[i.key()] = store.pathInfoToJSON(
-                store.exportReferences(storePaths, inputPaths), false, true);
+            json[i.key()] = pathInfoToJSON(store,
+                store.exportReferences(storePaths, inputPaths));
         }
     }
 
diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc
index ab39e71f4..e5d5205f4 100644
--- a/src/libstore/path-info.cc
+++ b/src/libstore/path-info.cc
@@ -1,5 +1,8 @@
+#include <nlohmann/json.hpp>
+
 #include "path-info.hh"
 #include "store-api.hh"
+#include "json-utils.hh"
 
 namespace nix {
 
@@ -144,4 +147,100 @@ ValidPathInfo::ValidPathInfo(
     }, std::move(ca).raw);
 }
 
+
+nlohmann::json ValidPathInfo::toJSON(
+    const Store & store,
+    bool includeImpureInfo,
+    HashFormat hashFormat) const
+{
+    using nlohmann::json;
+
+    auto jsonObject = json::object();
+
+    jsonObject["path"] = store.printStorePath(path);
+    jsonObject["valid"] = true;
+    jsonObject["narHash"] = narHash.to_string(hashFormat, true);
+    jsonObject["narSize"] = narSize;
+
+    {
+        auto& jsonRefs = (jsonObject["references"] = json::array());
+        for (auto & ref : references)
+            jsonRefs.emplace_back(store.printStorePath(ref));
+    }
+
+    if (ca)
+        jsonObject["ca"] = renderContentAddress(ca);
+
+    if (includeImpureInfo) {
+        if (deriver)
+            jsonObject["deriver"] = store.printStorePath(*deriver);
+
+        if (registrationTime)
+            jsonObject["registrationTime"] = registrationTime;
+
+        if (ultimate)
+            jsonObject["ultimate"] = ultimate;
+
+        if (!sigs.empty()) {
+            for (auto & sig : sigs)
+                jsonObject["signatures"].push_back(sig);
+        }
+    }
+
+    return jsonObject;
+}
+
+ValidPathInfo ValidPathInfo::fromJSON(
+    const Store & store,
+    const nlohmann::json & json)
+{
+    using nlohmann::detail::value_t;
+
+    ValidPathInfo res {
+        StorePath(StorePath::dummy),
+        Hash(Hash::dummy),
+    };
+
+    ensureType(json, value_t::object);
+    res.path = store.parseStorePath(
+        static_cast<const std::string &>(
+            ensureType(valueAt(json, "path"), value_t::string)));
+    res.narHash = Hash::parseAny(
+        static_cast<const std::string &>(
+            ensureType(valueAt(json, "narHash"), value_t::string)),
+        std::nullopt);
+    res.narSize = ensureType(valueAt(json, "narSize"), value_t::number_integer);
+
+    try {
+        auto & references = ensureType(valueAt(json, "references"), value_t::array);
+        for (auto & input : references)
+            res.references.insert(store.parseStorePath(static_cast<const std::string &>
+(input)));
+    } catch (Error & e) {
+        e.addTrace({}, "while reading key 'references'");
+        throw;
+    }
+
+    if (json.contains("ca"))
+        res.ca = ContentAddress::parse(
+            static_cast<const std::string &>(
+                ensureType(valueAt(json, "ca"), value_t::string)));
+
+    if (json.contains("deriver"))
+        res.deriver = store.parseStorePath(
+            static_cast<const std::string &>(
+                ensureType(valueAt(json, "deriver"), value_t::string)));
+
+    if (json.contains("registrationTime"))
+        res.registrationTime = ensureType(valueAt(json, "registrationTime"), value_t::number_integer);
+
+    if (json.contains("ultimate"))
+        res.ultimate = ensureType(valueAt(json, "ultimate"), value_t::boolean);
+
+    if (json.contains("signatures"))
+        res.sigs = valueAt(json, "signatures");
+
+    return res;
+}
+
 }
diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh
index c4c4a6366..feeda6c27 100644
--- a/src/libstore/path-info.hh
+++ b/src/libstore/path-info.hh
@@ -125,6 +125,18 @@ struct ValidPathInfo : UnkeyedValidPathInfo {
 
     Strings shortRefs() const;
 
+    /**
+     * @param includeImpureInfo If true, variable elements such as the
+     * registration time are included.
+     */
+    virtual nlohmann::json toJSON(
+        const Store & store,
+        bool includeImpureInfo,
+        HashFormat hashFormat) const;
+    static ValidPathInfo fromJSON(
+        const Store & store,
+        const nlohmann::json & json);
+
     ValidPathInfo(const ValidPathInfo & other) = default;
 
     ValidPathInfo(StorePath && path, UnkeyedValidPathInfo info) : UnkeyedValidPathInfo(info), path(std::move(path)) { };
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index c9ebb6c14..0f88d9b92 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -951,96 +951,6 @@ StorePathSet Store::exportReferences(const StorePathSet & storePaths, const Stor
     return paths;
 }
 
-json Store::pathInfoToJSON(const StorePathSet & storePaths,
-    bool includeImpureInfo, bool showClosureSize,
-    HashFormat hashFormat,
-    AllowInvalidFlag allowInvalid)
-{
-    json::array_t jsonList = json::array();
-
-    for (auto & storePath : storePaths) {
-        auto& jsonPath = jsonList.emplace_back(json::object());
-
-        try {
-            auto info = queryPathInfo(storePath);
-
-            jsonPath["path"] = printStorePath(info->path);
-            jsonPath["valid"] = true;
-            jsonPath["narHash"] = info->narHash.to_string(hashFormat, true);
-            jsonPath["narSize"] = info->narSize;
-
-            {
-                auto& jsonRefs = (jsonPath["references"] = json::array());
-                for (auto & ref : info->references)
-                    jsonRefs.emplace_back(printStorePath(ref));
-            }
-
-            if (info->ca)
-                jsonPath["ca"] = renderContentAddress(info->ca);
-
-            std::pair<uint64_t, uint64_t> closureSizes;
-
-            if (showClosureSize) {
-                closureSizes = getClosureSize(info->path);
-                jsonPath["closureSize"] = closureSizes.first;
-            }
-
-            if (includeImpureInfo) {
-
-                if (info->deriver)
-                    jsonPath["deriver"] = printStorePath(*info->deriver);
-
-                if (info->registrationTime)
-                    jsonPath["registrationTime"] = info->registrationTime;
-
-                if (info->ultimate)
-                    jsonPath["ultimate"] = info->ultimate;
-
-                if (!info->sigs.empty()) {
-                    for (auto & sig : info->sigs)
-                        jsonPath["signatures"].push_back(sig);
-                }
-
-                auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
-                    std::shared_ptr<const ValidPathInfo>(info));
-
-                if (narInfo) {
-                    if (!narInfo->url.empty())
-                        jsonPath["url"] = narInfo->url;
-                    if (narInfo->fileHash)
-                        jsonPath["downloadHash"] = narInfo->fileHash->to_string(hashFormat, true);
-                    if (narInfo->fileSize)
-                        jsonPath["downloadSize"] = narInfo->fileSize;
-                    if (showClosureSize)
-                        jsonPath["closureDownloadSize"] = closureSizes.second;
-                }
-            }
-
-        } catch (InvalidPath &) {
-            jsonPath["path"] = printStorePath(storePath);
-            jsonPath["valid"] = false;
-        }
-    }
-    return jsonList;
-}
-
-
-std::pair<uint64_t, uint64_t> Store::getClosureSize(const StorePath & storePath)
-{
-    uint64_t totalNarSize = 0, totalDownloadSize = 0;
-    StorePathSet closure;
-    computeFSClosure(storePath, closure, false, false);
-    for (auto & p : closure) {
-        auto info = queryPathInfo(p);
-        totalNarSize += info->narSize;
-        auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
-            std::shared_ptr<const ValidPathInfo>(info));
-        if (narInfo)
-            totalDownloadSize += narInfo->fileSize;
-    }
-    return {totalNarSize, totalDownloadSize};
-}
-
 
 const Store::Stats & Store::getStats()
 {
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 6aa317e3d..32ad2aa44 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -80,7 +80,6 @@ typedef std::map<std::string, StorePath> OutputPathMap;
 
 enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true };
 enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true };
-enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true };
 
 /**
  * Magic header of exportPath() output (obsolete).
@@ -665,28 +664,6 @@ public:
     std::string makeValidityRegistration(const StorePathSet & paths,
         bool showDerivers, bool showHash);
 
-    /**
-     * Write a JSON representation of store path metadata, such as the
-     * hash and the references.
-     *
-     * @param includeImpureInfo If true, variable elements such as the
-     * registration time are included.
-     *
-     * @param showClosureSize If true, the closure size of each path is
-     * included.
-     */
-    nlohmann::json pathInfoToJSON(const StorePathSet & storePaths,
-        bool includeImpureInfo, bool showClosureSize,
-        HashFormat hashFormat = HashFormat::Base32,
-        AllowInvalidFlag allowInvalid = DisallowInvalid);
-
-    /**
-     * @return the size of the closure of the specified path, that is,
-     * the sum of the size of the NAR serialisation of each path in the
-     * closure.
-     */
-    std::pair<uint64_t, uint64_t> getClosureSize(const StorePath & storePath);
-
     /**
      * Optimise the disk space usage of the Nix store by hard-linking files
      * with the same contents.
diff --git a/src/libstore/tests/nar-info.cc b/src/libstore/tests/nar-info.cc
new file mode 100644
index 000000000..cb92f3a28
--- /dev/null
+++ b/src/libstore/tests/nar-info.cc
@@ -0,0 +1,84 @@
+#include <nlohmann/json.hpp>
+#include <gtest/gtest.h>
+
+#include "path-info.hh"
+
+#include "tests/characterization.hh"
+#include "tests/libstore.hh"
+
+namespace nix {
+
+using nlohmann::json;
+
+class NarInfoTest : public CharacterizationTest, public LibStoreTest
+{
+    Path unitTestData = getUnitTestData() + "/libstore/nar-info";
+
+    Path goldenMaster(PathView testStem) const override {
+        return unitTestData + "/" + testStem + ".json";
+    }
+};
+
+static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) {
+    NarInfo info = ValidPathInfo {
+        store,
+        "foo",
+        FixedOutputInfo {
+            .method = FileIngestionMethod::Recursive,
+            .hash = hashString(HashType::htSHA256, "(...)"),
+
+            .references = {
+                .others = {
+                    StorePath {
+                        "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar",
+                    },
+                },
+                .self = true,
+            },
+        },
+        Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
+    };
+    info.narSize = 34878;
+    if (includeImpureInfo) {
+        info.deriver = StorePath {
+            "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
+        };
+        info.registrationTime = 23423;
+        info.ultimate = true;
+        info.sigs = { "asdf", "qwer" };
+
+        info.url = "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz";
+        info.fileHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=");
+        info.fileSize = 4029176;
+    }
+    return info;
+}
+
+#define JSON_TEST(STEM, PURE)                                             \
+    TEST_F(NarInfoTest, NarInfo_ ## STEM ## _from_json) {                 \
+        readTest(#STEM, [&](const auto & encoded_) {                      \
+            auto encoded = json::parse(encoded_);                         \
+            auto expected = makeNarInfo(*store, PURE);                    \
+            NarInfo got = NarInfo::fromJSON(                              \
+                *store,                                                   \
+                expected.path,                                            \
+                encoded);                                                 \
+            ASSERT_EQ(got, expected);                                     \
+        });                                                               \
+    }                                                                     \
+                                                                          \
+    TEST_F(NarInfoTest, NarInfo_ ## STEM ## _to_json) {                   \
+        writeTest(#STEM, [&]() -> json {                                  \
+            return makeNarInfo(*store, PURE)                              \
+                .toJSON(*store, PURE, HashFormat::SRI);                   \
+        }, [](const auto & file) {                                        \
+            return json::parse(readFile(file));                           \
+        }, [](const auto & file, const auto & got) {                      \
+            return writeFile(file, got.dump(2) + "\n");                   \
+        });                                                               \
+    }
+
+JSON_TEST(pure, false)
+JSON_TEST(impure, true)
+
+}
diff --git a/src/libstore/tests/path-info.cc b/src/libstore/tests/path-info.cc
new file mode 100644
index 000000000..fbee751c6
--- /dev/null
+++ b/src/libstore/tests/path-info.cc
@@ -0,0 +1,79 @@
+#include <nlohmann/json.hpp>
+#include <gtest/gtest.h>
+
+#include "path-info.hh"
+
+#include "tests/characterization.hh"
+#include "tests/libstore.hh"
+
+namespace nix {
+
+using nlohmann::json;
+
+class PathInfoTest : public CharacterizationTest, public LibStoreTest
+{
+    Path unitTestData = getUnitTestData() + "/libstore/path-info";
+
+    Path goldenMaster(PathView testStem) const override {
+        return unitTestData + "/" + testStem + ".json";
+    }
+};
+
+static ValidPathInfo makePathInfo(const Store & store, bool includeImpureInfo) {
+    ValidPathInfo info {
+        store,
+        "foo",
+        FixedOutputInfo {
+            .method = FileIngestionMethod::Recursive,
+            .hash = hashString(HashType::htSHA256, "(...)"),
+
+            .references = {
+                .others = {
+                    StorePath {
+                        "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar",
+                    },
+                },
+                .self = true,
+            },
+        },
+        Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
+    };
+    info.narSize = 34878;
+    if (includeImpureInfo) {
+        info.deriver = StorePath {
+            "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
+        };
+        info.registrationTime = 23423;
+        info.ultimate = true;
+        info.sigs = { "asdf", "qwer" };
+    }
+    return info;
+}
+
+#define JSON_TEST(STEM, PURE)                                             \
+    TEST_F(PathInfoTest, PathInfo_ ## STEM ## _from_json) {               \
+        readTest(#STEM, [&](const auto & encoded_) {                      \
+            auto encoded = json::parse(encoded_);                         \
+            ValidPathInfo got = ValidPathInfo::fromJSON(                  \
+                *store,                                                   \
+                encoded);                                                 \
+            auto expected = makePathInfo(*store, PURE);                   \
+            ASSERT_EQ(got, expected);                                     \
+        });                                                               \
+    }                                                                     \
+                                                                          \
+    TEST_F(PathInfoTest, PathInfo_ ## STEM ## _to_json) {                 \
+        writeTest(#STEM, [&]() -> json {                                  \
+            return makePathInfo(*store, PURE)                             \
+                .toJSON(*store, PURE, HashFormat::SRI);                   \
+        }, [](const auto & file) {                                        \
+            return json::parse(readFile(file));                           \
+        }, [](const auto & file, const auto & got) {                      \
+            return writeFile(file, got.dump(2) + "\n");                   \
+        });                                                               \
+    }
+
+JSON_TEST(pure, false)
+JSON_TEST(impure, true)
+
+}
diff --git a/src/libutil/tests/characterization.hh b/src/libutil/tests/characterization.hh
index 6698c5239..6eb513d68 100644
--- a/src/libutil/tests/characterization.hh
+++ b/src/libutil/tests/characterization.hh
@@ -4,6 +4,7 @@
 #include <gtest/gtest.h>
 
 #include "types.hh"
+#include "environment-variables.hh"
 
 namespace nix {
 
diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index c16864d30..b4bdd15ba 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -9,6 +9,74 @@
 #include <nlohmann/json.hpp>
 
 using namespace nix;
+using nlohmann::json;
+
+/**
+ * @return the total size of a set of store objects (specified by path),
+ * that is, the sum of the size of the NAR serialisation of each object
+ * in the set.
+ */
+static uint64_t getStoreObjectsTotalSize(Store & store, const StorePathSet & closure)
+{
+    uint64_t totalNarSize = 0;
+    for (auto & p : closure) {
+        totalNarSize += store.queryPathInfo(p)->narSize;
+    }
+    return totalNarSize;
+}
+
+
+/**
+ * Write a JSON representation of store object metadata, such as the
+ * hash and the references.
+ *
+ * @param showClosureSize If true, the closure size of each path is
+ * included.
+ */
+static json pathInfoToJSON(
+    Store & store,
+    const StorePathSet & storePaths,
+    bool showClosureSize)
+{
+    json::array_t jsonList = json::array();
+
+    for (auto & storePath : storePaths) {
+        try {
+            auto info = store.queryPathInfo(storePath);
+
+            auto & jsonPath = jsonList.emplace_back(
+                info->toJSON(store, true, HashFormat::SRI));
+
+            if (showClosureSize) {
+                StorePathSet closure;
+                store.computeFSClosure(storePath, closure, false, false);
+
+                jsonPath["closureSize"] = getStoreObjectsTotalSize(store, closure);
+
+                if (auto * narInfo = dynamic_cast<const NarInfo *>(&*info)) {
+                    uint64_t totalDownloadSize = 0;
+                    for (auto & p : closure) {
+                        auto depInfo = store.queryPathInfo(p);
+                        if (auto * depNarInfo = dynamic_cast<const NarInfo *>(&*depInfo))
+                            totalDownloadSize += depNarInfo->fileSize;
+                        else
+                            throw Error("Missing .narinfo for dep %s of %s",
+                                store.printStorePath(p),
+                                store.printStorePath(storePath));
+                    }
+                    jsonPath["closureDownloadSize"] = totalDownloadSize;
+                }
+            }
+
+        } catch (InvalidPath &) {
+            auto & jsonPath = jsonList.emplace_back(json::object());
+            jsonPath["path"] = store.printStorePath(storePath);
+            jsonPath["valid"] = false;
+        }
+    }
+    return jsonList;
+}
+
 
 struct CmdPathInfo : StorePathsCommand, MixJSON
 {
@@ -87,10 +155,11 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
             pathLen = std::max(pathLen, store->printStorePath(storePath).size());
 
         if (json) {
-            std::cout << store->pathInfoToJSON(
+            std::cout << pathInfoToJSON(
+                *store,
                 // FIXME: preserve order?
                 StorePathSet(storePaths.begin(), storePaths.end()),
-                true, showClosureSize, HashFormat::SRI, AllowInvalid).dump();
+                showClosureSize).dump();
         }
 
         else {
@@ -107,8 +176,11 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
                 if (showSize)
                     printSize(info->narSize);
 
-                if (showClosureSize)
-                    printSize(store->getClosureSize(info->path).first);
+                if (showClosureSize) {
+                    StorePathSet closure;
+                    store->computeFSClosure(storePath, closure, false, false);
+                    printSize(getStoreObjectsTotalSize(*store, closure));
+                }
 
                 if (showSigs) {
                     std::cout << '\t';
diff --git a/unit-test-data/libstore/nar-info/impure.json b/unit-test-data/libstore/nar-info/impure.json
new file mode 100644
index 000000000..093f25025
--- /dev/null
+++ b/unit-test-data/libstore/nar-info/impure.json
@@ -0,0 +1,21 @@
+{
+  "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh",
+  "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
+  "downloadHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=",
+  "downloadSize": 4029176,
+  "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=",
+  "narSize": 34878,
+  "path": "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo",
+  "references": [
+    "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar",
+    "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo"
+  ],
+  "registrationTime": 23423,
+  "signatures": [
+    "asdf",
+    "qwer"
+  ],
+  "ultimate": true,
+  "url": "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz",
+  "valid": true
+}
diff --git a/unit-test-data/libstore/nar-info/pure.json b/unit-test-data/libstore/nar-info/pure.json
new file mode 100644
index 000000000..62005d414
--- /dev/null
+++ b/unit-test-data/libstore/nar-info/pure.json
@@ -0,0 +1,11 @@
+{
+  "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh",
+  "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=",
+  "narSize": 34878,
+  "path": "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo",
+  "references": [
+    "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar",
+    "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo"
+  ],
+  "valid": true
+}
diff --git a/unit-test-data/libstore/path-info/impure.json b/unit-test-data/libstore/path-info/impure.json
new file mode 100644
index 000000000..c477c768c
--- /dev/null
+++ b/unit-test-data/libstore/path-info/impure.json
@@ -0,0 +1,18 @@
+{
+  "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh",
+  "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
+  "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=",
+  "narSize": 34878,
+  "path": "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo",
+  "references": [
+    "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar",
+    "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo"
+  ],
+  "registrationTime": 23423,
+  "signatures": [
+    "asdf",
+    "qwer"
+  ],
+  "ultimate": true,
+  "valid": true
+}
diff --git a/unit-test-data/libstore/path-info/pure.json b/unit-test-data/libstore/path-info/pure.json
new file mode 100644
index 000000000..62005d414
--- /dev/null
+++ b/unit-test-data/libstore/path-info/pure.json
@@ -0,0 +1,11 @@
+{
+  "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh",
+  "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=",
+  "narSize": 34878,
+  "path": "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo",
+  "references": [
+    "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar",
+    "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo"
+  ],
+  "valid": true
+}

From a7212e169b7204f80ea67f60c855d05b72b5d4f7 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 26 Oct 2023 20:01:36 -0400
Subject: [PATCH 071/294] Include `compression` in the `NarInfo` JSON format

It was forgotten before.
---
 src/libstore/nar-info.cc                     | 5 +++++
 src/libstore/tests/nar-info.cc               | 1 +
 unit-test-data/libstore/nar-info/impure.json | 1 +
 3 files changed, 7 insertions(+)

diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc
index a90812ff9..708cc7341 100644
--- a/src/libstore/nar-info.cc
+++ b/src/libstore/nar-info.cc
@@ -146,6 +146,8 @@ nlohmann::json NarInfo::toJSON(
     if (includeImpureInfo) {
         if (!url.empty())
             jsonObject["url"] = url;
+        if (!compression.empty())
+            jsonObject["compression"] = compression;
         if (fileHash)
             jsonObject["downloadHash"] = fileHash->to_string(hashFormat, true);
         if (fileSize)
@@ -168,6 +170,9 @@ NarInfo NarInfo::fromJSON(
     if (json.contains("url"))
         res.url = ensureType(valueAt(json, "url"), value_t::string);
 
+    if (json.contains("compression"))
+        res.compression = ensureType(valueAt(json, "compression"), value_t::string);
+
     if (json.contains("downloadHash"))
         res.fileHash = Hash::parseAny(
             static_cast<const std::string &>(
diff --git a/src/libstore/tests/nar-info.cc b/src/libstore/tests/nar-info.cc
index cb92f3a28..c5b21d56b 100644
--- a/src/libstore/tests/nar-info.cc
+++ b/src/libstore/tests/nar-info.cc
@@ -48,6 +48,7 @@ static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) {
         info.sigs = { "asdf", "qwer" };
 
         info.url = "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz";
+        info.compression = "xz";
         info.fileHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=");
         info.fileSize = 4029176;
     }
diff --git a/unit-test-data/libstore/nar-info/impure.json b/unit-test-data/libstore/nar-info/impure.json
index 093f25025..3f16667c9 100644
--- a/unit-test-data/libstore/nar-info/impure.json
+++ b/unit-test-data/libstore/nar-info/impure.json
@@ -1,5 +1,6 @@
 {
   "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh",
+  "compression": "xz",
   "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
   "downloadHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=",
   "downloadSize": 4029176,

From cc46ea163024254d0b74646e1b38b19896d40040 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 22 Oct 2023 21:12:54 -0400
Subject: [PATCH 072/294] Make `nix path-info --json` return an object not
 array

Before it returned a list of JSON objects with store object information,
including the path in each object. Now, it maps the paths to JSON
objects with the metadata sans path.

This matches how `nix derivation show` works.

Quite hillariously, none of our existing functional tests caught this
change to `path-info --json` though they did use it. So just new
functional tests need to be added.
---
 doc/manual/src/release-notes/rl-next.md       | 36 ++++++++++++++++++-
 src/libstore/nar-info.cc                      |  8 +++--
 src/libstore/parsed-derivations.cc            |  5 +++
 src/libstore/path-info.cc                     | 12 ++-----
 src/libstore/path-info.hh                     | 24 ++++++-------
 src/libstore/tests/path-info.cc               |  6 ++--
 src/nix/path-info.cc                          | 19 +++++-----
 src/nix/path-info.md                          | 12 +++----
 tests/functional/local.mk                     |  1 +
 tests/functional/path-info.sh                 | 23 ++++++++++++
 unit-test-data/libstore/nar-info/impure.json  |  4 +--
 unit-test-data/libstore/nar-info/pure.json    |  4 +--
 unit-test-data/libstore/path-info/impure.json |  4 +--
 unit-test-data/libstore/path-info/pure.json   |  4 +--
 14 files changed, 108 insertions(+), 54 deletions(-)
 create mode 100644 tests/functional/path-info.sh

diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
index 8cd69f8fd..85e180e37 100644
--- a/doc/manual/src/release-notes/rl-next.md
+++ b/doc/manual/src/release-notes/rl-next.md
@@ -28,5 +28,39 @@
 
   - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables.
     They are superceded by `nix flake update`.
-  
+
 - Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches).
+
+- [`nix path-info --json`](@docroot@/command-ref/new-cli/nix3-path-info.md)
+  (experimental) now returns a JSON map rather than JSON list.
+  The `path` field of each object has instead become the key in th outer map, since it is unique.
+  The `valid` field also goes away because we just use null instead.
+
+  - Old way:
+
+    ```json5
+    [
+      {
+        "path": "/nix/store/8fv91097mbh5049i9rglc73dx6kjg3qk-bash-5.2-p15",
+        "valid": true,
+        // ...
+      },
+      {
+        "path": "/nix/store/wffw7l0alvs3iw94cbgi1gmmbmw99sqb-home-manager-path",
+        "valid": false
+      }
+    ]
+    ```
+
+  - New way
+
+    ```json5
+    {
+      "/nix/store/8fv91097mbh5049i9rglc73dx6kjg3qk-bash-5.2-p15": {
+        // ...
+      },
+      "/nix/store/wffw7l0alvs3iw94cbgi1gmmbmw99sqb-home-manager-path": null,
+    }
+    ```
+
+  This makes it match `nix derivation show`, which also maps store paths to information.
diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc
index 708cc7341..ae2223fb0 100644
--- a/src/libstore/nar-info.cc
+++ b/src/libstore/nar-info.cc
@@ -164,8 +164,12 @@ NarInfo NarInfo::fromJSON(
 {
     using nlohmann::detail::value_t;
 
-    NarInfo res { ValidPathInfo::fromJSON(store, json) };
-    res.path = path;
+    NarInfo res {
+        ValidPathInfo {
+            path,
+            UnkeyedValidPathInfo::fromJSON(store, json),
+        }
+    };
 
     if (json.contains("url"))
         res.url = ensureType(valueAt(json, "url"), value_t::string);
diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc
index 45629dc7f..73e55a96c 100644
--- a/src/libstore/parsed-derivations.cc
+++ b/src/libstore/parsed-derivations.cc
@@ -148,6 +148,11 @@ static nlohmann::json pathInfoToJSON(
         auto & jsonPath = jsonList.emplace_back(
             info->toJSON(store, false, HashFormat::Base32));
 
+        // Add the path to the object whose metadata we are including.
+        jsonPath["path"] = store.printStorePath(storePath);
+
+        jsonPath["valid"] = true;
+
         jsonPath["closureSize"] = ({
             uint64_t totalNarSize = 0;
             StorePathSet closure;
diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc
index e5d5205f4..2d7dc972f 100644
--- a/src/libstore/path-info.cc
+++ b/src/libstore/path-info.cc
@@ -148,7 +148,7 @@ ValidPathInfo::ValidPathInfo(
 }
 
 
-nlohmann::json ValidPathInfo::toJSON(
+nlohmann::json UnkeyedValidPathInfo::toJSON(
     const Store & store,
     bool includeImpureInfo,
     HashFormat hashFormat) const
@@ -157,8 +157,6 @@ nlohmann::json ValidPathInfo::toJSON(
 
     auto jsonObject = json::object();
 
-    jsonObject["path"] = store.printStorePath(path);
-    jsonObject["valid"] = true;
     jsonObject["narHash"] = narHash.to_string(hashFormat, true);
     jsonObject["narSize"] = narSize;
 
@@ -190,21 +188,17 @@ nlohmann::json ValidPathInfo::toJSON(
     return jsonObject;
 }
 
-ValidPathInfo ValidPathInfo::fromJSON(
+UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON(
     const Store & store,
     const nlohmann::json & json)
 {
     using nlohmann::detail::value_t;
 
-    ValidPathInfo res {
-        StorePath(StorePath::dummy),
+    UnkeyedValidPathInfo res {
         Hash(Hash::dummy),
     };
 
     ensureType(json, value_t::object);
-    res.path = store.parseStorePath(
-        static_cast<const std::string &>(
-            ensureType(valueAt(json, "path"), value_t::string)));
     res.narHash = Hash::parseAny(
         static_cast<const std::string &>(
             ensureType(valueAt(json, "narHash"), value_t::string)),
diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh
index feeda6c27..077abc7e1 100644
--- a/src/libstore/path-info.hh
+++ b/src/libstore/path-info.hh
@@ -78,6 +78,18 @@ struct UnkeyedValidPathInfo
     DECLARE_CMP(UnkeyedValidPathInfo);
 
     virtual ~UnkeyedValidPathInfo() { }
+
+    /**
+     * @param includeImpureInfo If true, variable elements such as the
+     * registration time are included.
+     */
+    virtual nlohmann::json toJSON(
+        const Store & store,
+        bool includeImpureInfo,
+        HashFormat hashFormat) const;
+    static UnkeyedValidPathInfo fromJSON(
+        const Store & store,
+        const nlohmann::json & json);
 };
 
 struct ValidPathInfo : UnkeyedValidPathInfo {
@@ -125,18 +137,6 @@ struct ValidPathInfo : UnkeyedValidPathInfo {
 
     Strings shortRefs() const;
 
-    /**
-     * @param includeImpureInfo If true, variable elements such as the
-     * registration time are included.
-     */
-    virtual nlohmann::json toJSON(
-        const Store & store,
-        bool includeImpureInfo,
-        HashFormat hashFormat) const;
-    static ValidPathInfo fromJSON(
-        const Store & store,
-        const nlohmann::json & json);
-
     ValidPathInfo(const ValidPathInfo & other) = default;
 
     ValidPathInfo(StorePath && path, UnkeyedValidPathInfo info) : UnkeyedValidPathInfo(info), path(std::move(path)) { };
diff --git a/src/libstore/tests/path-info.cc b/src/libstore/tests/path-info.cc
index fbee751c6..49bf623bd 100644
--- a/src/libstore/tests/path-info.cc
+++ b/src/libstore/tests/path-info.cc
@@ -19,8 +19,8 @@ class PathInfoTest : public CharacterizationTest, public LibStoreTest
     }
 };
 
-static ValidPathInfo makePathInfo(const Store & store, bool includeImpureInfo) {
-    ValidPathInfo info {
+static UnkeyedValidPathInfo makePathInfo(const Store & store, bool includeImpureInfo) {
+    UnkeyedValidPathInfo info = ValidPathInfo {
         store,
         "foo",
         FixedOutputInfo {
@@ -54,7 +54,7 @@ static ValidPathInfo makePathInfo(const Store & store, bool includeImpureInfo) {
     TEST_F(PathInfoTest, PathInfo_ ## STEM ## _from_json) {               \
         readTest(#STEM, [&](const auto & encoded_) {                      \
             auto encoded = json::parse(encoded_);                         \
-            ValidPathInfo got = ValidPathInfo::fromJSON(                  \
+            UnkeyedValidPathInfo got = UnkeyedValidPathInfo::fromJSON(    \
                 *store,                                                   \
                 encoded);                                                 \
             auto expected = makePathInfo(*store, PURE);                   \
diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index b4bdd15ba..23198a120 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -38,20 +38,21 @@ static json pathInfoToJSON(
     const StorePathSet & storePaths,
     bool showClosureSize)
 {
-    json::array_t jsonList = json::array();
+    json::object_t jsonAllObjects = json::object();
 
     for (auto & storePath : storePaths) {
+        json jsonObject;
+
         try {
             auto info = store.queryPathInfo(storePath);
 
-            auto & jsonPath = jsonList.emplace_back(
-                info->toJSON(store, true, HashFormat::SRI));
+            jsonObject = info->toJSON(store, true, HashFormat::SRI);
 
             if (showClosureSize) {
                 StorePathSet closure;
                 store.computeFSClosure(storePath, closure, false, false);
 
-                jsonPath["closureSize"] = getStoreObjectsTotalSize(store, closure);
+                jsonObject["closureSize"] = getStoreObjectsTotalSize(store, closure);
 
                 if (auto * narInfo = dynamic_cast<const NarInfo *>(&*info)) {
                     uint64_t totalDownloadSize = 0;
@@ -64,17 +65,17 @@ static json pathInfoToJSON(
                                 store.printStorePath(p),
                                 store.printStorePath(storePath));
                     }
-                    jsonPath["closureDownloadSize"] = totalDownloadSize;
+                    jsonObject["closureDownloadSize"] = totalDownloadSize;
                 }
             }
 
         } catch (InvalidPath &) {
-            auto & jsonPath = jsonList.emplace_back(json::object());
-            jsonPath["path"] = store.printStorePath(storePath);
-            jsonPath["valid"] = false;
+            jsonObject = nullptr;
         }
+
+        jsonAllObjects[store.printStorePath(storePath)] = std::move(jsonObject);
     }
-    return jsonList;
+    return jsonAllObjects;
 }
 
 
diff --git a/src/nix/path-info.md b/src/nix/path-info.md
index 2dda866d0..4594854eb 100644
--- a/src/nix/path-info.md
+++ b/src/nix/path-info.md
@@ -43,7 +43,7 @@ R""(
   command):
 
   ```console
-  # nix path-info --json --all | jq -r 'sort_by(.registrationTime)[-11:-1][].path'
+  # nix path-info --json --all | jq -r 'to_entries | sort_by(.value.registrationTime) | .[-11:-1][] | .key'
   ```
 
 * Show the size of the entire Nix store:
@@ -58,13 +58,13 @@ R""(
 
   ```console
   # nix path-info --json --all --closure-size \
-    | jq 'map(select(.closureSize > 1e9)) | sort_by(.closureSize) | map([.path, .closureSize])'
+    | jq 'map_values(.closureSize | select(. < 1e9)) | to_entries | sort_by(.value)'
   [
     …,
-    [
-      "/nix/store/zqamz3cz4dbzfihki2mk7a63mbkxz9xq-nixos-system-machine-20.09.20201112.3090c65",
-      5887562256
-    ]
+    {
+      .key = "/nix/store/zqamz3cz4dbzfihki2mk7a63mbkxz9xq-nixos-system-machine-20.09.20201112.3090c65",
+      .value = 5887562256,
+    }
   ]
   ```
 
diff --git a/tests/functional/local.mk b/tests/functional/local.mk
index fe0d0c4ed..21dabca88 100644
--- a/tests/functional/local.mk
+++ b/tests/functional/local.mk
@@ -120,6 +120,7 @@ nix_tests = \
   flakes/show.sh \
   impure-derivations.sh \
   path-from-hash-part.sh \
+  path-info.sh \
   toString-path.sh \
   read-only-store.sh \
   nested-sandboxing.sh \
diff --git a/tests/functional/path-info.sh b/tests/functional/path-info.sh
new file mode 100644
index 000000000..763935eb7
--- /dev/null
+++ b/tests/functional/path-info.sh
@@ -0,0 +1,23 @@
+source common.sh
+
+echo foo > $TEST_ROOT/foo
+foo=$(nix store add-file $TEST_ROOT/foo)
+
+echo bar > $TEST_ROOT/bar
+bar=$(nix store add-file $TEST_ROOT/bar)
+
+echo baz > $TEST_ROOT/baz
+baz=$(nix store add-file $TEST_ROOT/baz)
+nix-store --delete "$baz"
+
+diff --unified --color=always \
+    <(nix path-info --json "$foo" "$bar" "$baz" |
+        jq --sort-keys 'map_values(.narHash)') \
+    <(jq --sort-keys <<-EOF
+        {
+          "$foo": "sha256-QvtAMbUl/uvi+LCObmqOhvNOapHdA2raiI4xG5zI5pA=",
+          "$bar": "sha256-9fhYGu9fqxcQC2Kc81qh2RMo1QcLBUBo8U+pPn+jthQ=",
+          "$baz": null
+        }
+EOF
+    )
diff --git a/unit-test-data/libstore/nar-info/impure.json b/unit-test-data/libstore/nar-info/impure.json
index 3f16667c9..bb9791a6a 100644
--- a/unit-test-data/libstore/nar-info/impure.json
+++ b/unit-test-data/libstore/nar-info/impure.json
@@ -6,7 +6,6 @@
   "downloadSize": 4029176,
   "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=",
   "narSize": 34878,
-  "path": "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo",
   "references": [
     "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar",
     "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo"
@@ -17,6 +16,5 @@
     "qwer"
   ],
   "ultimate": true,
-  "url": "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz",
-  "valid": true
+  "url": "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz"
 }
diff --git a/unit-test-data/libstore/nar-info/pure.json b/unit-test-data/libstore/nar-info/pure.json
index 62005d414..955baec31 100644
--- a/unit-test-data/libstore/nar-info/pure.json
+++ b/unit-test-data/libstore/nar-info/pure.json
@@ -2,10 +2,8 @@
   "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh",
   "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=",
   "narSize": 34878,
-  "path": "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo",
   "references": [
     "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar",
     "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo"
-  ],
-  "valid": true
+  ]
 }
diff --git a/unit-test-data/libstore/path-info/impure.json b/unit-test-data/libstore/path-info/impure.json
index c477c768c..0c452cc49 100644
--- a/unit-test-data/libstore/path-info/impure.json
+++ b/unit-test-data/libstore/path-info/impure.json
@@ -3,7 +3,6 @@
   "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
   "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=",
   "narSize": 34878,
-  "path": "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo",
   "references": [
     "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar",
     "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo"
@@ -13,6 +12,5 @@
     "asdf",
     "qwer"
   ],
-  "ultimate": true,
-  "valid": true
+  "ultimate": true
 }
diff --git a/unit-test-data/libstore/path-info/pure.json b/unit-test-data/libstore/path-info/pure.json
index 62005d414..955baec31 100644
--- a/unit-test-data/libstore/path-info/pure.json
+++ b/unit-test-data/libstore/path-info/pure.json
@@ -2,10 +2,8 @@
   "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh",
   "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=",
   "narSize": 34878,
-  "path": "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo",
   "references": [
     "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar",
     "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo"
-  ],
-  "valid": true
+  ]
 }

From 61d6fe059e959455e156c1d57bb91155d363e983 Mon Sep 17 00:00:00 2001
From: Mel Zuser <mel@mzero.dev>
Date: Mon, 6 Nov 2023 14:13:40 -0500
Subject: [PATCH 073/294] Fix `boost::bad_format_string` exception in
 `builtins.addErrorContext` (#9291)

* Fix boost::bad_format_string exception in builtins.addErrorContext

The message passed to addTrace was incorrectly being used as a format
string and this this would cause an exception when the string contained
a '%', which can be hit in places where arbitrary file paths are
interpolated.

* add test
---
 src/libexpr/primops.cc   | 2 +-
 tests/functional/lang.sh | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 36340d0f9..8d3a18526 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -825,7 +825,7 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value * *
         auto message = state.coerceToString(pos, *args[0], context,
                 "while evaluating the error message passed to builtins.addErrorContext",
                 false, false).toOwned();
-        e.addTrace(nullptr, message, true);
+        e.addTrace(nullptr, hintfmt(message), true);
         throw;
     }
 }
diff --git a/tests/functional/lang.sh b/tests/functional/lang.sh
index c3acef5ee..12df32c87 100755
--- a/tests/functional/lang.sh
+++ b/tests/functional/lang.sh
@@ -23,6 +23,7 @@ nix-instantiate --trace-verbose --eval -E 'builtins.traceVerbose "Hello" 123' 2>
 nix-instantiate --eval -E 'builtins.traceVerbose "Hello" 123' 2>&1 | grepQuietInverse Hello
 nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 | grepQuietInverse Hello
 expectStderr 1 nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" (throw "Foo")' | grepQuiet Hello
+expectStderr 1 nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello %" (throw "Foo")' | grepQuiet 'Hello %'
 
 nix-instantiate --eval -E 'let x = builtins.trace { x = x; } true; in x' \
   2>&1 | grepQuiet -E 'trace: { x = «potential infinite recursion»; }'

From 867f894289437a96630579592a46a4253151f079 Mon Sep 17 00:00:00 2001
From: Ana Hobden <operator@hoverbear.org>
Date: Fri, 15 Sep 2023 10:49:30 -0700
Subject: [PATCH 074/294] Populate $XDG_DATA_DIRS with appropriate folder from
 Nix profile

On non-NixOS systems, the default `nix` install does not populate the
`$XDG_DATA_DIRS`. This populates it and enables things like bash-completion
and `.desktop` file detection for `nix` profile installed packages.

Signed-off-by: Ana Hobden <operator@hoverbear.org>
---
 scripts/nix-profile-daemon.fish.in | 8 ++++++++
 scripts/nix-profile-daemon.sh.in   | 8 ++++++++
 scripts/nix-profile.fish.in        | 8 ++++++++
 scripts/nix-profile.sh.in          | 8 ++++++++
 4 files changed, 32 insertions(+)

diff --git a/scripts/nix-profile-daemon.fish.in b/scripts/nix-profile-daemon.fish.in
index 400696812..e7b394d56 100644
--- a/scripts/nix-profile-daemon.fish.in
+++ b/scripts/nix-profile-daemon.fish.in
@@ -19,6 +19,14 @@ set __ETC_PROFILE_NIX_SOURCED 1
 
 set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile"
 
+# Populate bash completions, .desktop files, etc
+if test -n "$NIX_SSH_CERT_FILE"
+    # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default
+    set --export XDG_DATA_DIRS "/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
+else
+    set --export XDG_DATA_DIRS "$XDG_DATA_DIRS:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
+end
+
 # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
 if test -n "$NIX_SSH_CERT_FILE"
   : # Allow users to override the NIX_SSL_CERT_FILE
diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in
index 8cfd3149e..3089cec66 100644
--- a/scripts/nix-profile-daemon.sh.in
+++ b/scripts/nix-profile-daemon.sh.in
@@ -30,6 +30,14 @@ fi
 
 export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK"
 
+# Populate bash completions, .desktop files, etc
+if [ -n "${XDG_DATA_DIRS-}" ]; then
+    # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default
+    export XDG_DATA_DIRS="/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
+else
+    export XDG_DATA_DIRS="$XDG_DATA_DIRS:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
+fi
+
 # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
 if [ -n "${NIX_SSL_CERT_FILE:-}" ]; then
     : # Allow users to override the NIX_SSL_CERT_FILE
diff --git a/scripts/nix-profile.fish.in b/scripts/nix-profile.fish.in
index 731498c76..fc8fe4e97 100644
--- a/scripts/nix-profile.fish.in
+++ b/scripts/nix-profile.fish.in
@@ -20,6 +20,14 @@ if test -n "$HOME" && test -n "$USER"
     # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix
     set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile"
 
+    # Populate bash completions, .desktop files, etc
+    if test -n "$NIX_SSH_CERT_FILE"
+        # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default
+        set --export XDG_DATA_DIRS "/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
+    else
+        set --export XDG_DATA_DIRS "$XDG_DATA_DIRS:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
+    end
+
     # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
     if test -n "$NIX_SSH_CERT_FILE"
         : # Allow users to override the NIX_SSL_CERT_FILE
diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in
index c4d60cf37..a0d098588 100644
--- a/scripts/nix-profile.sh.in
+++ b/scripts/nix-profile.sh.in
@@ -32,6 +32,14 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
     # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix
     export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK"
 
+    # Populate bash completions, .desktop files, etc
+    if [ -n "${XDG_DATA_DIRS-}" ]; then
+        # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default
+        export XDG_DATA_DIRS="/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
+    else
+        export XDG_DATA_DIRS="$XDG_DATA_DIRS:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
+    fi
+
     # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
     if [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch
         export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt

From 896013ec0c0d4633349ff0373bdae626667adc77 Mon Sep 17 00:00:00 2001
From: Ana Hobden <operator@hoverbear.org>
Date: Thu, 21 Sep 2023 09:27:35 -0700
Subject: [PATCH 075/294] Fix bad copy-paste

---
 scripts/nix-profile-daemon.fish.in | 2 +-
 scripts/nix-profile.fish.in        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/nix-profile-daemon.fish.in b/scripts/nix-profile-daemon.fish.in
index e7b394d56..5f5a53141 100644
--- a/scripts/nix-profile-daemon.fish.in
+++ b/scripts/nix-profile-daemon.fish.in
@@ -20,7 +20,7 @@ set __ETC_PROFILE_NIX_SOURCED 1
 set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile"
 
 # Populate bash completions, .desktop files, etc
-if test -n "$NIX_SSH_CERT_FILE"
+if test -n "$XDG_DATA_DIRS"
     # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default
     set --export XDG_DATA_DIRS "/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
 else
diff --git a/scripts/nix-profile.fish.in b/scripts/nix-profile.fish.in
index fc8fe4e97..2523594f2 100644
--- a/scripts/nix-profile.fish.in
+++ b/scripts/nix-profile.fish.in
@@ -21,7 +21,7 @@ if test -n "$HOME" && test -n "$USER"
     set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile"
 
     # Populate bash completions, .desktop files, etc
-    if test -n "$NIX_SSH_CERT_FILE"
+    if test -n "$XDG_DATA_DIRS"
         # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default
         set --export XDG_DATA_DIRS "/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
     else

From 150b5aba509d169a50c6ad62100c3ad7bf00242b Mon Sep 17 00:00:00 2001
From: Ana Hobden <operator@hoverbear.org>
Date: Mon, 6 Nov 2023 10:07:53 -0800
Subject: [PATCH 076/294] Update scripts/nix-profile-daemon.fish.in

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
---
 scripts/nix-profile-daemon.fish.in | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/nix-profile-daemon.fish.in b/scripts/nix-profile-daemon.fish.in
index 5f5a53141..3fe9e782a 100644
--- a/scripts/nix-profile-daemon.fish.in
+++ b/scripts/nix-profile-daemon.fish.in
@@ -22,9 +22,9 @@ set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profi
 # Populate bash completions, .desktop files, etc
 if test -n "$XDG_DATA_DIRS"
     # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default
-    set --export XDG_DATA_DIRS "/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
+    set --export XDG_DATA_DIRS "/usr/local/share:/usr/share:/nix/var/nix/profiles/default/share"
 else
-    set --export XDG_DATA_DIRS "$XDG_DATA_DIRS:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
+    set --export XDG_DATA_DIRS "$XDG_DATA_DIRS:/nix/var/nix/profiles/default/share"
 end
 
 # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.

From c60eba3276d7417a7f51ef606e5b9ca580cf5e5b Mon Sep 17 00:00:00 2001
From: Felix Uhl <iFreilicht@users.noreply.github.com>
Date: Mon, 6 Nov 2023 21:43:18 +0100
Subject: [PATCH 077/294] Add release note on XDG_DATA_DIRS change

Follow-up to https://github.com/NixOS/nix/pull/8985
---
 doc/manual/src/release-notes/rl-next.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
index 85e180e37..73ba03fc4 100644
--- a/doc/manual/src/release-notes/rl-next.md
+++ b/doc/manual/src/release-notes/rl-next.md
@@ -64,3 +64,8 @@
     ```
 
   This makes it match `nix derivation show`, which also maps store paths to information.
+
+- When Nix is installed using the [binary installer](@docroot@/installation/installing-binary.md), in supported shells (Bash, Zsh, Fish)
+  [`XDG_DATA_DIRS`](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html#variables) is now populated with the path to the `/share` subdirectory of the current profile.
+  This means that command completion scripts, `.desktop` files, and similar artifacts installed via [`nix-env`](@docroot@/command-ref/nix-env.md) or [`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md)
+  (experimental) can be found by any program that follows the [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html).

From 9fec62a10044629ad4758ec95f9b1e67d7aefff5 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 6 Nov 2023 22:21:07 +0000
Subject: [PATCH 078/294] build(deps): bump zeebe-io/backport-action from 2.0.0
 to 2.1.0

Bumps [zeebe-io/backport-action](https://github.com/zeebe-io/backport-action) from 2.0.0 to 2.1.0.
- [Release notes](https://github.com/zeebe-io/backport-action/releases)
- [Commits](https://github.com/zeebe-io/backport-action/compare/v2.0.0...v2.1.0)

---
updated-dependencies:
- dependency-name: zeebe-io/backport-action
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/backport.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml
index 312c211dd..893f4a56f 100644
--- a/.github/workflows/backport.yml
+++ b/.github/workflows/backport.yml
@@ -21,7 +21,7 @@ jobs:
           fetch-depth: 0
       - name: Create backport PRs
         # should be kept in sync with `version`
-        uses: zeebe-io/backport-action@v2.0.0
+        uses: zeebe-io/backport-action@v2.1.0
         with:
           # Config README: https://github.com/zeebe-io/backport-action#backport-action
           github_token: ${{ secrets.GITHUB_TOKEN }}

From b733f4ab29cec07cf17e1fe6580c9d2f8a4362a0 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Tue, 7 Nov 2023 01:12:39 +0100
Subject: [PATCH 079/294] maintainers: refine the mission statement phrasing

setting a direction falls short of what we're already doing: guide contributors.

the direction aspect is still important, as that is the authoritative part. guidance is the supportive part.
---
 maintainers/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/maintainers/README.md b/maintainers/README.md
index 5be4f9d04..ee97c1195 100644
--- a/maintainers/README.md
+++ b/maintainers/README.md
@@ -2,7 +2,7 @@
 
 ## Motivation
 
-The team's main responsibility is to set a direction for the development of Nix and ensure that the code is in good shape.
+The team's main responsibility is to guide and direct the development of Nix and ensure that the code is in good shape.
 
 We aim to achieve this by improving the contributor experience and attracting more maintainers – that is, by helping other people contributing to Nix and eventually taking responsibility – in order to scale the development process to match users' needs.
 

From 1362a0a55aaddccef5a525e3b1179239d650bb07 Mon Sep 17 00:00:00 2001
From: Felix Uhl <felix.uhl@outlook.com>
Date: Mon, 6 Nov 2023 23:16:05 +0100
Subject: [PATCH 080/294] Fix logic for default XDG_DATA_DIRS value

The [POSIX test manpage](https://pubs.opengroup.org/onlinepubs/9699919799/utilities/test.html)
as well as the [fish test manpage](https://fishshell.com/docs/current/cmds/test.html#operators-for-text-strings)
specify that `-z` will be "True if the length of string string is zero;
otherwise, false."

The `-n` was likely a mixup and not caught during testing of
https://github.com/NixOS/nix/pull/8985 due to a lack of missing
conflicting entries in `XDG_DATA_DIRS`.
---
 scripts/nix-profile-daemon.fish.in | 2 +-
 scripts/nix-profile-daemon.sh.in   | 2 +-
 scripts/nix-profile.fish.in        | 2 +-
 scripts/nix-profile.sh.in          | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/scripts/nix-profile-daemon.fish.in b/scripts/nix-profile-daemon.fish.in
index 3fe9e782a..c23aa64f0 100644
--- a/scripts/nix-profile-daemon.fish.in
+++ b/scripts/nix-profile-daemon.fish.in
@@ -20,7 +20,7 @@ set __ETC_PROFILE_NIX_SOURCED 1
 set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile"
 
 # Populate bash completions, .desktop files, etc
-if test -n "$XDG_DATA_DIRS"
+if test -z "$XDG_DATA_DIRS"
     # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default
     set --export XDG_DATA_DIRS "/usr/local/share:/usr/share:/nix/var/nix/profiles/default/share"
 else
diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in
index 3089cec66..c63db4648 100644
--- a/scripts/nix-profile-daemon.sh.in
+++ b/scripts/nix-profile-daemon.sh.in
@@ -31,7 +31,7 @@ fi
 export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK"
 
 # Populate bash completions, .desktop files, etc
-if [ -n "${XDG_DATA_DIRS-}" ]; then
+if [ -z "$XDG_DATA_DIRS" ]; then
     # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default
     export XDG_DATA_DIRS="/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
 else
diff --git a/scripts/nix-profile.fish.in b/scripts/nix-profile.fish.in
index 2523594f2..619df52b8 100644
--- a/scripts/nix-profile.fish.in
+++ b/scripts/nix-profile.fish.in
@@ -21,7 +21,7 @@ if test -n "$HOME" && test -n "$USER"
     set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile"
 
     # Populate bash completions, .desktop files, etc
-    if test -n "$XDG_DATA_DIRS"
+    if test -z "$XDG_DATA_DIRS"
         # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default
         set --export XDG_DATA_DIRS "/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
     else
diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in
index a0d098588..56e070ae1 100644
--- a/scripts/nix-profile.sh.in
+++ b/scripts/nix-profile.sh.in
@@ -33,7 +33,7 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
     export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK"
 
     # Populate bash completions, .desktop files, etc
-    if [ -n "${XDG_DATA_DIRS-}" ]; then
+    if [ -z "$XDG_DATA_DIRS" ]; then
         # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default
         export XDG_DATA_DIRS="/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
     else

From 74210c12feccc6c6b717c5f39c28d7ce86614e60 Mon Sep 17 00:00:00 2001
From: Tom Bereknyei <tomberek@gmail.com>
Date: Sat, 28 Aug 2021 16:26:53 -0400
Subject: [PATCH 081/294] Shellbang support with flakes

Enables shebang usage of nix shell. All arguments with `#! nix` get
added to the nix invocation. This implementation does NOT set any
additional arguments other than placing the script path itself as the
first argument such that the interpreter can utilize it.

Example below:

```
    #!/usr/bin/env nix
    #! nix shell --quiet
    #! nix nixpkgs#bash
    #! nix nixpkgs#shellcheck
    #! nix nixpkgs#hello
    #! nix --ignore-environment --command bash
    # shellcheck shell=bash
    set -eu
    shellcheck "$0" || exit 1
    function main {
        hello
        echo 0:"$0" 1:"$1" 2:"$2"
    }
    "$@"
```

fix: include programName usage

EDIT: For posterity I've changed shellwords to shellwords2 in order
      not to interfere with other changes during a rebase.
      shellwords2 is removed in a later commit. -- roberth
---
 src/libutil/args.cc               | 37 +++++++++++++++++++++++++
 src/libutil/args.hh               |  8 +++++-
 src/libutil/util.cc               | 45 +++++++++++++++++++++++++++++++
 src/libutil/util.hh               | 11 +++++---
 src/nix/main.cc                   |  2 +-
 tests/functional/flakes/common.sh |  5 ++--
 tests/functional/flakes/flakes.sh | 15 ++++++++++-
 7 files changed, 114 insertions(+), 9 deletions(-)

diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 0b65519a3..7106491fd 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -6,6 +6,7 @@
 #include "users.hh"
 #include "json-utils.hh"
 
+#include <regex>
 #include <glob.h>
 
 namespace nix {
@@ -78,6 +79,12 @@ std::optional<std::string> RootArgs::needsCompletion(std::string_view s)
 }
 
 void RootArgs::parseCmdline(const Strings & _cmdline)
+{
+    // Default via 5.1.2.2.1 in C standard
+    Args::parseCmdline("", _cmdline);
+}
+
+void Args::parseCmdline(const std::string & programName, const Strings & _cmdline)
 {
     Strings pendingArgs;
     bool dashDash = false;
@@ -93,6 +100,36 @@ void RootArgs::parseCmdline(const Strings & _cmdline)
     }
 
     bool argsSeen = false;
+
+    // Heuristic to see if we're invoked as a shebang script, namely,
+    // if we have at least one argument, it's the name of an
+    // executable file, and it starts with "#!".
+    Strings savedArgs;
+    auto isNixCommand = std::regex_search(programName, std::regex("nix$"));
+    if (isNixCommand && cmdline.size() > 0) {
+        auto script = *cmdline.begin();
+        try {
+            auto lines = tokenizeString<Strings>(readFile(script), "\n");
+            if (std::regex_search(lines.front(), std::regex("^#!"))) {
+                lines.pop_front();
+                for (auto pos = std::next(cmdline.begin()); pos != cmdline.end();pos++)
+                    savedArgs.push_back(*pos);
+                cmdline.clear();
+
+                for (auto line : lines) {
+                    line = chomp(line);
+
+                    std::smatch match;
+                    if (std::regex_match(line, match, std::regex("^#!\\s*nix\\s(.*)$")))
+                        for (const auto & word : shellwords(match[1].str()))
+                            cmdline.push_back(word);
+                }
+                cmdline.push_back(script);
+                for (auto pos = savedArgs.begin(); pos != savedArgs.end();pos++)
+                    cmdline.push_back(*pos);
+            }
+        } catch (SysError &) { }
+    }
     for (auto pos = cmdline.begin(); pos != cmdline.end(); ) {
 
         auto arg = *pos;
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 45fd678e7..1d056678d 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -27,8 +27,14 @@ class Args
 public:
 
     /**
-     * Return a short one-line description of the command.
+     * Parse the command line with argv0, throwing a UsageError if something
+       goes wrong.
      */
+    void parseCmdline(const std::string & argv0, const Strings & cmdline);
+
+    /**
+     * Return a short one-line description of the command.
+    */
     virtual std::string description() { return ""; }
 
     virtual bool forceImpureByDefault() { return false; }
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index ee7a22849..6ca1dbd7a 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -5,6 +5,8 @@
 #include <cctype>
 #include <iostream>
 #include <grp.h>
+#include <regex>
+
 
 namespace nix {
 
@@ -136,6 +138,49 @@ std::string shellEscape(const std::string_view s)
     return r;
 }
 
+/* Recreate the effect of the perl shellwords function, breaking up a
+ * string into arguments like a shell word, including escapes
+ */
+std::vector<std::string> shellwords2(const std::string & s)
+{
+    std::regex whitespace("^(\\s+).*");
+    auto begin = s.cbegin();
+    std::vector<std::string> res;
+    std::string cur;
+    enum state {
+        sBegin,
+        sQuote
+    };
+    state st = sBegin;
+    auto it = begin;
+    for (; it != s.cend(); ++it) {
+        if (st == sBegin) {
+            std::smatch match;
+            if (regex_search(it, s.cend(), match, whitespace)) {
+                cur.append(begin, it);
+                res.push_back(cur);
+                cur.clear();
+                it = match[1].second;
+                begin = it;
+            }
+        }
+        switch (*it) {
+            case '"':
+                cur.append(begin, it);
+                begin = it + 1;
+                st = st == sBegin ? sQuote : sBegin;
+                break;
+            case '\\':
+                /* perl shellwords mostly just treats the next char as part of the string with no special processing */
+                cur.append(begin, it);
+                begin = ++it;
+                break;
+        }
+    }
+    cur.append(begin, it);
+    if (!cur.empty()) res.push_back(cur);
+    return res;
+}
 
 void ignoreException(Verbosity lvl)
 {
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 5f730eaf6..bcd0c1769 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -189,10 +189,13 @@ std::string toLower(const std::string & s);
 std::string shellEscape(const std::string_view s);
 
 
-/**
- * Exception handling in destructors: print an error message, then
- * ignore the exception.
- */
+/* Recreate the effect of the perl shellwords function, breaking up a
+ * string into arguments like a shell word, including escapes */
+std::vector<std::string> shellwords2(const std::string & s);
+
+
+/* Exception handling in destructors: print an error message, then
+   ignore the exception. */
 void ignoreException(Verbosity lvl = lvlError);
 
 
diff --git a/src/nix/main.cc b/src/nix/main.cc
index b582fc166..16fb50806 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -428,7 +428,7 @@ void mainWrapped(int argc, char * * argv)
     });
 
     try {
-        args.parseCmdline(argvToStrings(argc, argv));
+        args.parseCmdline(programName, argvToStrings(argc, argv));
     } catch (UsageError &) {
         if (!args.helpRequested && !args.completions) throw;
     }
diff --git a/tests/functional/flakes/common.sh b/tests/functional/flakes/common.sh
index 8aed296e6..fc45cf7bf 100644
--- a/tests/functional/flakes/common.sh
+++ b/tests/functional/flakes/common.sh
@@ -11,6 +11,7 @@ writeSimpleFlake() {
   outputs = inputs: rec {
     packages.$system = rec {
       foo = import ./simple.nix;
+      fooScript = (import ./shell.nix {}).foo;
       default = foo;
     };
     packages.someOtherSystem = rec {
@@ -24,13 +25,13 @@ writeSimpleFlake() {
 }
 EOF
 
-    cp ../simple.nix ../simple.builder.sh ../config.nix $flakeDir/
+    cp ../simple.nix ../shell.nix ../simple.builder.sh ../config.nix $flakeDir/
 }
 
 createSimpleGitFlake() {
     local flakeDir="$1"
     writeSimpleFlake $flakeDir
-    git -C $flakeDir add flake.nix simple.nix simple.builder.sh config.nix
+    git -C $flakeDir add flake.nix simple.nix shell.nix simple.builder.sh config.nix
     git -C $flakeDir commit -m 'Initial'
 }
 
diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh
index b0038935c..c4b18a21b 100644
--- a/tests/functional/flakes/flakes.sh
+++ b/tests/functional/flakes/flakes.sh
@@ -66,7 +66,17 @@ cat > "$nonFlakeDir/README.md" <<EOF
 FNORD
 EOF
 
-git -C "$nonFlakeDir" add README.md
+cat > "$nonFlakeDir/shebang.sh" <<EOF
+#! $(type -P env) nix
+#! nix --offline shell
+#! nix flake1#fooScript
+#! nix --no-write-lock-file --command bash
+set -e
+foo
+EOF
+chmod +x "$nonFlakeDir/shebang.sh"
+
+git -C "$nonFlakeDir" add README.md shebang.sh
 git -C "$nonFlakeDir" commit -m 'Initial'
 
 # Construct a custom registry, additionally test the --registry flag
@@ -511,3 +521,6 @@ nix flake metadata "$flake2Dir" --reference-lock-file $TEST_ROOT/flake2-overridd
 
 # reference-lock-file can only be used if allow-dirty is set.
 expectStderr 1 nix flake metadata "$flake2Dir" --no-allow-dirty --reference-lock-file $TEST_ROOT/flake2-overridden.lock
+
+# Test shebang
+[[ $($nonFlakeDir/shebang.sh) = "foo" ]]

From 5f9b5758b6f8b0d2c4bea5bcc11c64eb623e3650 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 14 Nov 2022 17:04:19 +0100
Subject: [PATCH 082/294] src/libutil/util.hh: Formatting

---
 src/libutil/util.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index bcd0c1769..b7d3ac504 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -190,7 +190,7 @@ std::string shellEscape(const std::string_view s);
 
 
 /* Recreate the effect of the perl shellwords function, breaking up a
- * string into arguments like a shell word, including escapes */
+   string into arguments like a shell word, including escapes. */
 std::vector<std::string> shellwords2(const std::string & s);
 
 

From eea5a003d99094d8488fd0d1ecd97f98d3573133 Mon Sep 17 00:00:00 2001
From: Tom Bereknyei <tomberek@gmail.com>
Date: Mon, 14 Nov 2022 19:40:01 -0500
Subject: [PATCH 083/294] fix: test to ensure arguments are passed

---
 tests/functional/flakes/flakes.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh
index c4b18a21b..e7dffde07 100644
--- a/tests/functional/flakes/flakes.sh
+++ b/tests/functional/flakes/flakes.sh
@@ -71,8 +71,9 @@ cat > "$nonFlakeDir/shebang.sh" <<EOF
 #! nix --offline shell
 #! nix flake1#fooScript
 #! nix --no-write-lock-file --command bash
-set -e
+set -ex
 foo
+echo "\$@"
 EOF
 chmod +x "$nonFlakeDir/shebang.sh"
 
@@ -524,3 +525,4 @@ expectStderr 1 nix flake metadata "$flake2Dir" --no-allow-dirty --reference-lock
 
 # Test shebang
 [[ $($nonFlakeDir/shebang.sh) = "foo" ]]
+[[ $($nonFlakeDir/shebang.sh "bar") = "foo"$'\n'"bar" ]]

From bfcbf3b5bff21de2634f2713491f818702bea2a1 Mon Sep 17 00:00:00 2001
From: Tom Bereknyei <tomberek@gmail.com>
Date: Mon, 14 Nov 2022 23:58:58 -0500
Subject: [PATCH 084/294] doc: shebang release notes, docs, tests

fix: release notes
---
 doc/manual/src/release-notes/rl-next.md |  44 +++++++++
 src/nix/shell.md                        | 117 ++++++++++++++++++++++++
 tests/functional/flakes/flakes.sh       |  13 +++
 3 files changed, 174 insertions(+)

diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
index 73ba03fc4..93d4f432b 100644
--- a/doc/manual/src/release-notes/rl-next.md
+++ b/doc/manual/src/release-notes/rl-next.md
@@ -1,5 +1,49 @@
 # Release X.Y (202?-??-??)
 
+- The experimental nix command is now a `#!-interpreter` by appending the
+  contents of any `#! nix` lines and the script's location to a single call.
+  Some examples:
+  ```
+  #!/usr/bin/env nix
+  #! nix shell --file "<nixpkgs>" hello --command bash
+
+  hello | cowsay
+  ```
+  or with flakes:
+  ```
+  #!/usr/bin/env nix
+  #! nix shell nixpkgs#bash nixpkgs#hello nixpkgs#cowsay --command bash
+
+  hello | cowsay
+  ```
+  or
+  ```bash
+  #! /usr/bin/env nix
+  #! nix shell --impure --expr
+  #! nix "with (import (builtins.getFlake ''nixpkgs'') {}); terraform.withPlugins (plugins: [ plugins.openstack ])"
+  #! nix --command bash
+
+  terraform "$@"
+  ```
+  or
+  ```
+  #!/usr/bin/env nix
+  //! ```cargo
+  //! [dependencies]
+  //! time = "0.1.25"
+  //! ```
+  /*
+  #!nix shell nixpkgs#rustc nixpkgs#rust-script nixpkgs#cargo --command rust-script
+  */
+  fn main() {
+      for argument in std::env::args().skip(1) {
+          println!("{}", argument);
+      };
+      println!("{}", std::env::var("HOME").expect(""));
+      println!("{}", time::now().rfc822z());
+  }
+  // vim: ft=rust
+  ```
 - [URL flake references](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) now support [percent-encoded](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1) characters.
 
 - [Path-like flake references](@docroot@/command-ref/new-cli/nix3-flake.md#path-like-syntax) now accept arbitrary unicode characters (except `#` and `?`).
diff --git a/src/nix/shell.md b/src/nix/shell.md
index f36919575..b0bfa1609 100644
--- a/src/nix/shell.md
+++ b/src/nix/shell.md
@@ -51,4 +51,121 @@ R""(
 provides the specified [*installables*](./nix.md#installable). If no command is specified, it starts the
 default shell of your user account specified by `$SHELL`.
 
+# Use as a `#!`-interpreter
+
+You can use `nix` as a script interpreter to allow scripts written
+in arbitrary languages to obtain their own dependencies via Nix. This is
+done by starting the script with the following lines:
+
+```bash
+#! /usr/bin/env nix
+#! nix shell installables --command real-interpreter
+```
+
+where *real-interpreter* is the “real” script interpreter that will be
+invoked by `nix shell` after it has obtained the dependencies and
+initialised the environment, and *installables* are the attribute names of
+the dependencies in Nixpkgs.
+
+The lines starting with `#! nix` specify options (see above). Note that you
+cannot write `#! /usr/bin/env nix shell -i ...` because many operating systems
+only allow one argument in `#!` lines.
+
+For example, here is a Python script that depends on Python and the
+`prettytable` package:
+
+```python
+#! /usr/bin/env nix
+#! nix shell github:tomberek/-#python3With.prettytable --command python
+
+import prettytable
+
+# Print a simple table.
+t = prettytable.PrettyTable(["N", "N^2"])
+for n in range(1, 10): t.add_row([n, n * n])
+print t
+```
+
+Similarly, the following is a Perl script that specifies that it
+requires Perl and the `HTML::TokeParser::Simple` and `LWP` packages:
+
+```perl
+#! /usr/bin/env nix
+#! nix shell github:tomberek/-#perlWith.HTMLTokeParserSimple.LWP --command perl -x
+
+use HTML::TokeParser::Simple;
+
+# Fetch nixos.org and print all hrefs.
+my $p = HTML::TokeParser::Simple->new(url => 'http://nixos.org/');
+
+while (my $token = $p->get_tag("a")) {
+    my $href = $token->get_attr("href");
+    print "$href\n" if $href;
+}
+```
+
+Sometimes you need to pass a simple Nix expression to customize a
+package like Terraform:
+
+```bash
+#! /usr/bin/env nix
+#! nix shell --impure --expr
+#! nix "with (import (builtins.getFlake ''nixpkgs'') {}); terraform.withPlugins (plugins: [ plugins.openstack ])"
+#! nix --command bash
+
+terraform "$@"
+```
+
+> **Note**
+>
+> You must use double quotes (`"`) when passing a simple Nix expression
+> in a nix shell shebang.
+
+Finally, using the merging of multiple nix shell shebangs the following
+Haskell script uses a specific branch of Nixpkgs/NixOS (the 21.11 stable
+branch):
+
+```haskell
+#!/usr/bin/env nix
+#!nix shell --override-input nixpkgs github:NixOS/nixpkgs/nixos-21.11
+#!nix github:tomberek/-#haskellWith.download-curl.tagsoup --command runghc
+
+import Network.Curl.Download
+import Text.HTML.TagSoup
+import Data.Either
+import Data.ByteString.Char8 (unpack)
+
+-- Fetch nixos.org and print all hrefs.
+main = do
+  resp <- openURI "https://nixos.org/"
+  let tags = filter (isTagOpenName "a") $ parseTags $ unpack $ fromRight undefined resp
+  let tags' = map (fromAttrib "href") tags
+  mapM_ putStrLn $ filter (/= "") tags'
+```
+
+If you want to be even more precise, you can specify a specific revision
+of Nixpkgs:
+
+    #!nix shell --override-input nixpkgs github:NixOS/nixpkgs/eabc38219184cc3e04a974fe31857d8e0eac098d
+
+The examples above all used `-p` to get dependencies from Nixpkgs. You
+can also use a Nix expression to build your own dependencies. For
+example, the Python example could have been written as:
+
+```python
+#! /usr/bin/env nix
+#! nix shell --impure --file deps.nix -i python
+```
+
+where the file `deps.nix` in the same directory as the `#!`-script
+contains:
+
+```nix
+with import <nixpkgs> {};
+python3.withPackages (ps: with ps; [ prettytable ])
+```
+
+
+
+
 )""
diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh
index e7dffde07..f27925493 100644
--- a/tests/functional/flakes/flakes.sh
+++ b/tests/functional/flakes/flakes.sh
@@ -80,6 +80,18 @@ chmod +x "$nonFlakeDir/shebang.sh"
 git -C "$nonFlakeDir" add README.md shebang.sh
 git -C "$nonFlakeDir" commit -m 'Initial'
 
+cat > $nonFlakeDir/shebang-perl.sh <<EOF
+#! $(type -P env) nix
+#! nix run nixpkgs#perl
+## Modules used
+use strict;
+use warnings;
+
+# Print function
+print("Hello World\n");
+EOF
+chmod +x $nonFlakeDir/shebang-perl.sh
+
 # Construct a custom registry, additionally test the --registry flag
 nix registry add --registry "$registry" flake1 "git+file://$flake1Dir"
 nix registry add --registry "$registry" flake2 "git+file://$percentEncodedFlake2Dir"
@@ -526,3 +538,4 @@ expectStderr 1 nix flake metadata "$flake2Dir" --no-allow-dirty --reference-lock
 # Test shebang
 [[ $($nonFlakeDir/shebang.sh) = "foo" ]]
 [[ $($nonFlakeDir/shebang.sh "bar") = "foo"$'\n'"bar" ]]
+[[ $($nonFlakeDir/shebang-perl.sh ) = "Hello World!" ]]

From 01f61cefcb77f0177afd9d17589eb45c2ebf6cee Mon Sep 17 00:00:00 2001
From: Tom Bereknyei <tomberek@gmail.com>
Date: Wed, 1 Sep 2021 02:19:51 -0400
Subject: [PATCH 085/294] Read file incrementally

---
 src/libutil/args.cc | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 7106491fd..80216e7ad 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -6,6 +6,8 @@
 #include "users.hh"
 #include "json-utils.hh"
 
+#include <fstream>
+#include <string>
 #include <regex>
 #include <glob.h>
 
@@ -109,14 +111,17 @@ void Args::parseCmdline(const std::string & programName, const Strings & _cmdlin
     if (isNixCommand && cmdline.size() > 0) {
         auto script = *cmdline.begin();
         try {
-            auto lines = tokenizeString<Strings>(readFile(script), "\n");
-            if (std::regex_search(lines.front(), std::regex("^#!"))) {
-                lines.pop_front();
+            std::ifstream stream(script);
+            char shebang[3]={0,0,0};
+            stream.get(shebang,3);
+            if (strncmp(shebang,"#!",2) == 0){
                 for (auto pos = std::next(cmdline.begin()); pos != cmdline.end();pos++)
                     savedArgs.push_back(*pos);
                 cmdline.clear();
 
-                for (auto line : lines) {
+                std::string line;
+                std::getline(stream,line);
+                while (std::getline(stream,line) && !line.empty()){
                     line = chomp(line);
 
                     std::smatch match;

From 06f3583b1c860b24f2f704f216f4db8fd1dcae9c Mon Sep 17 00:00:00 2001
From: Tom Bereknyei <tomberek@gmail.com>
Date: Sat, 26 Nov 2022 09:06:39 -0500
Subject: [PATCH 086/294] feat: break out of shebang processing for
 non-comments

---
 src/libutil/args.cc               |  3 ++-
 tests/functional/flakes/flakes.sh | 20 ++++++++++----------
 2 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 80216e7ad..d90374adc 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -121,7 +121,8 @@ void Args::parseCmdline(const std::string & programName, const Strings & _cmdlin
 
                 std::string line;
                 std::getline(stream,line);
-                while (std::getline(stream,line) && !line.empty()){
+                std::string commentChars("#/\\%@*-");
+                while (std::getline(stream,line) && !line.empty() && commentChars.find(line[0]) != std::string::npos){
                     line = chomp(line);
 
                     std::smatch match;
diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh
index f27925493..28b5e4e0f 100644
--- a/tests/functional/flakes/flakes.sh
+++ b/tests/functional/flakes/flakes.sh
@@ -80,17 +80,17 @@ chmod +x "$nonFlakeDir/shebang.sh"
 git -C "$nonFlakeDir" add README.md shebang.sh
 git -C "$nonFlakeDir" commit -m 'Initial'
 
-cat > $nonFlakeDir/shebang-perl.sh <<EOF
+cat > $nonFlakeDir/shebang-comments.sh <<EOF
 #! $(type -P env) nix
-#! nix run nixpkgs#perl
-## Modules used
-use strict;
-use warnings;
-
-# Print function
-print("Hello World\n");
+# some comments
+# some comments
+# some comments
+#! nix --offline shell
+#! nix flake1#fooScript
+#! nix --no-write-lock-file --command bash
+foo
 EOF
-chmod +x $nonFlakeDir/shebang-perl.sh
+chmod +x $nonFlakeDir/shebang-comments.sh
 
 # Construct a custom registry, additionally test the --registry flag
 nix registry add --registry "$registry" flake1 "git+file://$flake1Dir"
@@ -538,4 +538,4 @@ expectStderr 1 nix flake metadata "$flake2Dir" --no-allow-dirty --reference-lock
 # Test shebang
 [[ $($nonFlakeDir/shebang.sh) = "foo" ]]
 [[ $($nonFlakeDir/shebang.sh "bar") = "foo"$'\n'"bar" ]]
-[[ $($nonFlakeDir/shebang-perl.sh ) = "Hello World!" ]]
+[[ $($nonFlakeDir/shebang-comments.sh ) = "foo" ]]

From e6ed7292433d2cf4c9d887d450d2fa9c4e5377e3 Mon Sep 17 00:00:00 2001
From: Tom Bereknyei <tomberek@gmail.com>
Date: Tue, 3 Jan 2023 05:55:06 -0500
Subject: [PATCH 087/294] doc: remove reference to nix-shell

---
 src/nix/shell.md | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/src/nix/shell.md b/src/nix/shell.md
index b0bfa1609..7e0e5f213 100644
--- a/src/nix/shell.md
+++ b/src/nix/shell.md
@@ -148,9 +148,8 @@ of Nixpkgs:
 
     #!nix shell --override-input nixpkgs github:NixOS/nixpkgs/eabc38219184cc3e04a974fe31857d8e0eac098d
 
-The examples above all used `-p` to get dependencies from Nixpkgs. You
-can also use a Nix expression to build your own dependencies. For
-example, the Python example could have been written as:
+You can also use a Nix expression to build your own dependencies. For example,
+the Python example could have been written as:
 
 ```python
 #! /usr/bin/env nix
@@ -166,6 +165,4 @@ python3.withPackages (ps: with ps; [ prettytable ])
 ```
 
 
-
-
 )""

From bbeddf06027424dc08742c1d54bf2fdc85ff6e8e Mon Sep 17 00:00:00 2001
From: Tom Bereknyei <tomberek@gmail.com>
Date: Fri, 12 May 2023 07:44:25 -0400
Subject: [PATCH 088/294] fix: refactor parseCmdline interface

---
 src/libutil/args.cc | 9 ++++-----
 src/libutil/args.hh | 2 +-
 src/nix/main.cc     | 5 ++++-
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index d90374adc..481ed33ff 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -83,10 +83,10 @@ std::optional<std::string> RootArgs::needsCompletion(std::string_view s)
 void RootArgs::parseCmdline(const Strings & _cmdline)
 {
     // Default via 5.1.2.2.1 in C standard
-    Args::parseCmdline("", _cmdline);
+    Args::parseCmdline(_cmdline, false);
 }
 
-void Args::parseCmdline(const std::string & programName, const Strings & _cmdline)
+void Args::parseCmdline(const Strings & _cmdline, bool allowShebang)
 {
     Strings pendingArgs;
     bool dashDash = false;
@@ -107,8 +107,7 @@ void Args::parseCmdline(const std::string & programName, const Strings & _cmdlin
     // if we have at least one argument, it's the name of an
     // executable file, and it starts with "#!".
     Strings savedArgs;
-    auto isNixCommand = std::regex_search(programName, std::regex("nix$"));
-    if (isNixCommand && cmdline.size() > 0) {
+    if (allowShebang){
         auto script = *cmdline.begin();
         try {
             std::ifstream stream(script);
@@ -121,7 +120,7 @@ void Args::parseCmdline(const std::string & programName, const Strings & _cmdlin
 
                 std::string line;
                 std::getline(stream,line);
-                std::string commentChars("#/\\%@*-");
+                static const std::string commentChars("#/\\%@*-");
                 while (std::getline(stream,line) && !line.empty() && commentChars.find(line[0]) != std::string::npos){
                     line = chomp(line);
 
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 1d056678d..e753dcaf6 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -30,7 +30,7 @@ public:
      * Parse the command line with argv0, throwing a UsageError if something
        goes wrong.
      */
-    void parseCmdline(const std::string & argv0, const Strings & cmdline);
+    void parseCmdline(const Strings & _cmdline, bool allowShebang);
 
     /**
      * Return a short one-line description of the command.
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 16fb50806..73641f6d2 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -22,6 +22,7 @@
 #include <ifaddrs.h>
 #include <netdb.h>
 #include <netinet/in.h>
+#include <regex>
 
 #include <nlohmann/json.hpp>
 
@@ -428,7 +429,9 @@ void mainWrapped(int argc, char * * argv)
     });
 
     try {
-        args.parseCmdline(programName, argvToStrings(argc, argv));
+        auto isNixCommand = std::regex_search(programName, std::regex("nix$"));
+        auto allowShebang = isNixCommand && argc > 1;
+        args.parseCmdline(argvToStrings(argc, argv),allowShebang);
     } catch (UsageError &) {
         if (!args.helpRequested && !args.completions) throw;
     }

From cc68ed8ff7b9e3898308a39dfdad2660bacc153f Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 12 May 2023 19:42:49 +0200
Subject: [PATCH 089/294] libcmd: lookupFileArg(): add baseDir

This will allow a different base directory to be used, matching
a shebang script location instead of the working directory.
---
 src/libcmd/common-eval-args.cc | 4 ++--
 src/libcmd/common-eval-args.hh | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc
index 91fa881b1..401acc38e 100644
--- a/src/libcmd/common-eval-args.cc
+++ b/src/libcmd/common-eval-args.cc
@@ -164,7 +164,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
     return res.finish();
 }
 
-SourcePath lookupFileArg(EvalState & state, std::string_view s)
+SourcePath lookupFileArg(EvalState & state, std::string_view s, CanonPath baseDir)
 {
     if (EvalSettings::isPseudoUrl(s)) {
         auto storePath = fetchers::downloadTarball(
@@ -185,7 +185,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s)
     }
 
     else
-        return state.rootPath(CanonPath::fromCwd(s));
+        return state.rootPath(CanonPath(s, baseDir));
 }
 
 }
diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh
index 6359b2579..4b403d936 100644
--- a/src/libcmd/common-eval-args.hh
+++ b/src/libcmd/common-eval-args.hh
@@ -2,6 +2,7 @@
 ///@file
 
 #include "args.hh"
+#include "canon-path.hh"
 #include "common-args.hh"
 #include "search-path.hh"
 
@@ -28,6 +29,6 @@ private:
     std::map<std::string, std::string> autoArgs;
 };
 
-SourcePath lookupFileArg(EvalState & state, std::string_view s);
+SourcePath lookupFileArg(EvalState & state, std::string_view s, CanonPath baseDir = CanonPath::fromCwd());
 
 }

From 20ff61ab252fc1d2bd69987f51a000739b24c670 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 12 May 2023 19:46:37 +0200
Subject: [PATCH 090/294] nix: Reserve shebang line syntax and only parse
 double backtick quotes

Being restrictive about syntax leaves opportunity to improve the
syntax and functionality later.
---
 doc/manual/src/release-notes/rl-next.md |  11 +-
 src/libutil/args.cc                     | 152 +++++++++++++++++++++++-
 src/libutil/util.cc                     |  43 -------
 src/libutil/util.hh                     |   5 -
 src/nix/shell.md                        |   8 +-
 tests/functional/flakes/flakes.sh       |  16 ++-
 6 files changed, 177 insertions(+), 58 deletions(-)

diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
index 93d4f432b..4bff3c685 100644
--- a/doc/manual/src/release-notes/rl-next.md
+++ b/doc/manual/src/release-notes/rl-next.md
@@ -2,10 +2,13 @@
 
 - The experimental nix command is now a `#!-interpreter` by appending the
   contents of any `#! nix` lines and the script's location to a single call.
+
+  Verbatim strings may be passed in double backtick (```` `` ````) quotes.
+
   Some examples:
   ```
   #!/usr/bin/env nix
-  #! nix shell --file "<nixpkgs>" hello --command bash
+  #! nix shell --file ``<nixpkgs>`` hello --command bash
 
   hello | cowsay
   ```
@@ -19,8 +22,10 @@
   or
   ```bash
   #! /usr/bin/env nix
-  #! nix shell --impure --expr
-  #! nix "with (import (builtins.getFlake ''nixpkgs'') {}); terraform.withPlugins (plugins: [ plugins.openstack ])"
+  #! nix shell --impure --expr ``
+  #! nix with (import (builtins.getFlake "nixpkgs") {});
+  #! nix terraform.withPlugins (plugins: [ plugins.openstack ])
+  #! nix ``
   #! nix --command bash
 
   terraform "$@"
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 481ed33ff..ab6e0e266 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -86,6 +86,147 @@ void RootArgs::parseCmdline(const Strings & _cmdline)
     Args::parseCmdline(_cmdline, false);
 }
 
+/**
+ * Basically this is `typedef std::optional<Parser> Parser(std::string_view s, Strings & r);`
+ *
+ * Except we can't recursively reference the Parser typedef, so we have to write a class.
+ */
+struct Parser {
+    std::string_view remaining;
+
+    /**
+     * @brief Parse the next character(s)
+     *
+     * @param r
+     * @return std::shared_ptr<Parser>
+     */
+    virtual void operator()(std::shared_ptr<Parser> & state, Strings & r) = 0;
+
+    Parser(std::string_view s) : remaining(s) {};
+};
+
+struct ParseQuoted : public Parser {
+    /**
+     * @brief Accumulated string
+     *
+     * Parsed argument up to this point.
+     */
+    std::string acc;
+
+    ParseQuoted(std::string_view s) : Parser(s) {};
+
+    virtual void operator()(std::shared_ptr<Parser> & state, Strings & r) override;
+};
+
+
+struct ParseUnquoted : public Parser {
+    /**
+     * @brief Accumulated string
+     *
+     * Parsed argument up to this point. Empty string is not representable in
+     * unquoted syntax, so we use it for the initial state.
+     */
+    std::string acc;
+
+    ParseUnquoted(std::string_view s) : Parser(s) {};
+
+    virtual void operator()(std::shared_ptr<Parser> & state, Strings & r) override {
+        if (remaining.empty()) {
+            if (!acc.empty())
+                r.push_back(acc);
+            state = nullptr; // done
+            return;
+        }
+        switch (remaining[0]) {
+            case ' ': case '\t': case '\n': case '\r':
+                if (!acc.empty())
+                    r.push_back(acc);
+                state = std::make_shared<ParseUnquoted>(ParseUnquoted(remaining.substr(1)));
+                return;
+            case '`':
+                if (remaining.size() > 1 && remaining[1] == '`') {
+                    state = std::make_shared<ParseQuoted>(ParseQuoted(remaining.substr(2)));
+                    return;
+                }
+                else
+                    throw Error("single backtick is not a supported syntax in the nix shebang.");
+
+            // reserved characters
+            // meaning to be determined, or may be reserved indefinitely so that
+            // #!nix syntax looks unambiguous
+            case '$':
+            case '*':
+            case '~':
+            case '<':
+            case '>':
+            case '|':
+            case ';':
+            case '(':
+            case ')':
+            case '[':
+            case ']':
+            case '{':
+            case '}':
+            case '\'':
+            case '"':
+            case '\\':
+                throw Error("unsupported unquoted character in nix shebang: " + std::string(1, remaining[0]) + ". Use double backticks to escape?");
+
+            case '#':
+                if (acc.empty()) {
+                    throw Error ("unquoted nix shebang argument cannot start with #. Use double backticks to escape?");
+                } else {
+                    acc += remaining[0];
+                    remaining = remaining.substr(1);
+                    return;
+                }
+
+            default:
+                acc += remaining[0];
+                remaining = remaining.substr(1);
+                return;
+        }
+        assert(false);
+    }
+};
+
+void ParseQuoted::operator()(std::shared_ptr<Parser> &state, Strings & r) {
+    if (remaining.empty()) {
+        throw Error("unterminated quoted string in nix shebang");
+    }
+    switch (remaining[0]) {
+        case '`':
+            if (remaining.size() > 1 && remaining[1] == '`') {
+                state = std::make_shared<ParseUnquoted>(ParseUnquoted(remaining.substr(2)));
+                r.push_back(acc);
+                return;
+            }
+            else {
+                acc += remaining[0];
+                remaining = remaining.substr(1);
+                return;
+            }
+        default:
+            acc += remaining[0];
+            remaining = remaining.substr(1);
+            return;
+    }
+    assert(false);
+}
+
+static Strings parseShebangContent(std::string_view s) {
+    Strings result;
+    std::shared_ptr<Parser> parserState(std::make_shared<ParseUnquoted>(ParseUnquoted(s)));
+
+    // trampoline == iterated strategy pattern
+    while (parserState) {
+        auto currentState = parserState;
+        (*currentState)(parserState, result);
+    }
+
+    return result;
+}
+
 void Args::parseCmdline(const Strings & _cmdline, bool allowShebang)
 {
     Strings pendingArgs;
@@ -121,13 +262,18 @@ void Args::parseCmdline(const Strings & _cmdline, bool allowShebang)
                 std::string line;
                 std::getline(stream,line);
                 static const std::string commentChars("#/\\%@*-");
+                std::string shebangContent;
                 while (std::getline(stream,line) && !line.empty() && commentChars.find(line[0]) != std::string::npos){
                     line = chomp(line);
 
                     std::smatch match;
-                    if (std::regex_match(line, match, std::regex("^#!\\s*nix\\s(.*)$")))
-                        for (const auto & word : shellwords(match[1].str()))
-                            cmdline.push_back(word);
+                    // We match one space after `nix` so that we preserve indentation.
+                    // No space is necessary for an empty line. An empty line has basically no effect.
+                    if (std::regex_match(line, match, std::regex("^#!\\s*nix(:? |$)(.*)$")))
+                        shebangContent += match[2].str() + "\n";
+                }
+                for (const auto & word : parseShebangContent(shebangContent)) {
+                    cmdline.push_back(word);
                 }
                 cmdline.push_back(script);
                 for (auto pos = savedArgs.begin(); pos != savedArgs.end();pos++)
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 6ca1dbd7a..5bb3f374b 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -138,49 +138,6 @@ std::string shellEscape(const std::string_view s)
     return r;
 }
 
-/* Recreate the effect of the perl shellwords function, breaking up a
- * string into arguments like a shell word, including escapes
- */
-std::vector<std::string> shellwords2(const std::string & s)
-{
-    std::regex whitespace("^(\\s+).*");
-    auto begin = s.cbegin();
-    std::vector<std::string> res;
-    std::string cur;
-    enum state {
-        sBegin,
-        sQuote
-    };
-    state st = sBegin;
-    auto it = begin;
-    for (; it != s.cend(); ++it) {
-        if (st == sBegin) {
-            std::smatch match;
-            if (regex_search(it, s.cend(), match, whitespace)) {
-                cur.append(begin, it);
-                res.push_back(cur);
-                cur.clear();
-                it = match[1].second;
-                begin = it;
-            }
-        }
-        switch (*it) {
-            case '"':
-                cur.append(begin, it);
-                begin = it + 1;
-                st = st == sBegin ? sQuote : sBegin;
-                break;
-            case '\\':
-                /* perl shellwords mostly just treats the next char as part of the string with no special processing */
-                cur.append(begin, it);
-                begin = ++it;
-                break;
-        }
-    }
-    cur.append(begin, it);
-    if (!cur.empty()) res.push_back(cur);
-    return res;
-}
 
 void ignoreException(Verbosity lvl)
 {
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index b7d3ac504..27faa4d6d 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -189,11 +189,6 @@ std::string toLower(const std::string & s);
 std::string shellEscape(const std::string_view s);
 
 
-/* Recreate the effect of the perl shellwords function, breaking up a
-   string into arguments like a shell word, including escapes. */
-std::vector<std::string> shellwords2(const std::string & s);
-
-
 /* Exception handling in destructors: print an error message, then
    ignore the exception. */
 void ignoreException(Verbosity lvl = lvlError);
diff --git a/src/nix/shell.md b/src/nix/shell.md
index 7e0e5f213..7c315fb3f 100644
--- a/src/nix/shell.md
+++ b/src/nix/shell.md
@@ -109,8 +109,10 @@ package like Terraform:
 
 ```bash
 #! /usr/bin/env nix
-#! nix shell --impure --expr
-#! nix "with (import (builtins.getFlake ''nixpkgs'') {}); terraform.withPlugins (plugins: [ plugins.openstack ])"
+#! nix shell --impure --expr ``
+#! nix with (import (builtins.getFlake ''nixpkgs'') {});
+#! nix terraform.withPlugins (plugins: [ plugins.openstack ])
+#! nix ``
 #! nix --command bash
 
 terraform "$@"
@@ -118,7 +120,7 @@ terraform "$@"
 
 > **Note**
 >
-> You must use double quotes (`"`) when passing a simple Nix expression
+> You must use double backticks (```` `` ````) when passing a simple Nix expression
 > in a nix shell shebang.
 
 Finally, using the merging of multiple nix shell shebangs the following
diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh
index 28b5e4e0f..a0a34ffa9 100644
--- a/tests/functional/flakes/flakes.sh
+++ b/tests/functional/flakes/flakes.sh
@@ -80,6 +80,7 @@ chmod +x "$nonFlakeDir/shebang.sh"
 git -C "$nonFlakeDir" add README.md shebang.sh
 git -C "$nonFlakeDir" commit -m 'Initial'
 
+# this also tests a fairly trivial double backtick quoted string, ``--command``
 cat > $nonFlakeDir/shebang-comments.sh <<EOF
 #! $(type -P env) nix
 # some comments
@@ -87,11 +88,23 @@ cat > $nonFlakeDir/shebang-comments.sh <<EOF
 # some comments
 #! nix --offline shell
 #! nix flake1#fooScript
-#! nix --no-write-lock-file --command bash
+#! nix --no-write-lock-file ``--command`` bash
 foo
 EOF
 chmod +x $nonFlakeDir/shebang-comments.sh
 
+cat > $nonFlakeDir/shebang-reject.sh <<EOF
+#! $(type -P env) nix
+# some comments
+# some comments
+# some comments
+#! nix --offline shell *
+#! nix flake1#fooScript
+#! nix --no-write-lock-file --command bash
+foo
+EOF
+chmod +x $nonFlakeDir/shebang-reject.sh
+
 # Construct a custom registry, additionally test the --registry flag
 nix registry add --registry "$registry" flake1 "git+file://$flake1Dir"
 nix registry add --registry "$registry" flake2 "git+file://$percentEncodedFlake2Dir"
@@ -539,3 +552,4 @@ expectStderr 1 nix flake metadata "$flake2Dir" --no-allow-dirty --reference-lock
 [[ $($nonFlakeDir/shebang.sh) = "foo" ]]
 [[ $($nonFlakeDir/shebang.sh "bar") = "foo"$'\n'"bar" ]]
 [[ $($nonFlakeDir/shebang-comments.sh ) = "foo" ]]
+expect 1 $nonFlakeDir/shebang-reject.sh 2>&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?'

From 198bc22e3b856bf2a86225c2ce5b3a7394e3ac0c Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 12 May 2023 19:54:54 +0200
Subject: [PATCH 091/294] nix: Add command baseDir to parse --expr relative to
 shebang script

---
 doc/manual/src/release-notes/rl-next.md |  1 +
 src/libcmd/installables.cc              |  3 ++-
 src/libutil/args.cc                     |  9 +++++++++
 src/libutil/args.hh                     | 20 ++++++++++++++++++++
 tests/functional/flakes/flakes.sh       | 19 +++++++++++++++++++
 5 files changed, 51 insertions(+), 1 deletion(-)

diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
index 4bff3c685..28b6d75f5 100644
--- a/doc/manual/src/release-notes/rl-next.md
+++ b/doc/manual/src/release-notes/rl-next.md
@@ -4,6 +4,7 @@
   contents of any `#! nix` lines and the script's location to a single call.
 
   Verbatim strings may be passed in double backtick (```` `` ````) quotes.
+  `--expr` resolves relative paths based on the shebang script location.
 
   Some examples:
   ```
diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index e7f58556f..528643dc5 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -445,7 +445,8 @@ Installables SourceExprCommand::parseInstallables(
         else if (file)
             state->evalFile(lookupFileArg(*state, *file), *vFile);
         else {
-            auto e = state->parseExprFromString(*expr, state->rootPath(CanonPath::fromCwd()));
+            CanonPath dir(CanonPath::fromCwd(getCommandBaseDir()));
+            auto e = state->parseExprFromString(*expr, state->rootPath(dir));
             state->eval(e, *vFile);
         }
 
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index ab6e0e266..0012b3f47 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -276,6 +276,7 @@ void Args::parseCmdline(const Strings & _cmdline, bool allowShebang)
                     cmdline.push_back(word);
                 }
                 cmdline.push_back(script);
+                commandBaseDir = dirOf(script);
                 for (auto pos = savedArgs.begin(); pos != savedArgs.end();pos++)
                     cmdline.push_back(*pos);
             }
@@ -336,6 +337,14 @@ void Args::parseCmdline(const Strings & _cmdline, bool allowShebang)
         d.completer(*completions, d.n, d.prefix);
 }
 
+Path Args::getCommandBaseDir() const
+{
+    if (parent)
+        return parent->getCommandBaseDir();
+    else
+        return commandBaseDir;
+}
+
 bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
 {
     assert(pos != end);
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index e753dcaf6..9c942606e 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -24,6 +24,16 @@ class AddCompletions;
 
 class Args
 {
+    /**
+     * @brief The command's "working directory", but only set when top level.
+     *
+     * Use getCommandBaseDir() to get the directory regardless of whether this
+     * is a top-level command or subcommand.
+     *
+     * @see getCommandBaseDir()
+     */
+    Path commandBaseDir = ".";
+
 public:
 
     /**
@@ -44,6 +54,16 @@ public:
      */
     virtual std::string doc() { return ""; }
 
+    /**
+     * @brief Get the base directory for the command.
+     *
+     * @return Generally the working directory, but in case of a shebang
+     *         interpreter, returns the directory of the script.
+     *
+     * This only returns the correct value after parseCmdline() has run.
+     */
+    Path getCommandBaseDir() const;
+
 protected:
 
     /**
diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh
index a0a34ffa9..76f3495dd 100644
--- a/tests/functional/flakes/flakes.sh
+++ b/tests/functional/flakes/flakes.sh
@@ -105,6 +105,24 @@ foo
 EOF
 chmod +x $nonFlakeDir/shebang-reject.sh
 
+cat > $nonFlakeDir/shebang-inline-expr.sh <<EOF
+#! $(type -P env) nix
+EOF
+cat >> $nonFlakeDir/shebang-inline-expr.sh <<"EOF"
+#! nix --offline shell
+#! nix --impure --expr ``
+#! nix let flake = (builtins.getFlake (toString ../flake1)).packages;
+#! nix     fooScript = flake.${builtins.currentSystem}.fooScript;
+#! nix     /* just a comment !@#$%^&*()__+ # */
+#! nix  in fooScript
+#! nix ``
+#! nix --no-write-lock-file --command bash
+set -ex
+foo
+echo "$@"
+EOF
+chmod +x $nonFlakeDir/shebang-inline-expr.sh
+
 # Construct a custom registry, additionally test the --registry flag
 nix registry add --registry "$registry" flake1 "git+file://$flake1Dir"
 nix registry add --registry "$registry" flake2 "git+file://$percentEncodedFlake2Dir"
@@ -552,4 +570,5 @@ expectStderr 1 nix flake metadata "$flake2Dir" --no-allow-dirty --reference-lock
 [[ $($nonFlakeDir/shebang.sh) = "foo" ]]
 [[ $($nonFlakeDir/shebang.sh "bar") = "foo"$'\n'"bar" ]]
 [[ $($nonFlakeDir/shebang-comments.sh ) = "foo" ]]
+[[ $($nonFlakeDir/shebang-inline-expr.sh baz) = "foo"$'\n'"baz" ]]
 expect 1 $nonFlakeDir/shebang-reject.sh 2>&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?'

From 466271568be7d3bcf0151dc7e09899775ac31f13 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 12 May 2023 19:56:04 +0200
Subject: [PATCH 092/294] nix: Parse --file relative to shebang script

---
 doc/manual/src/release-notes/rl-next.md |  2 +-
 src/libcmd/installables.cc              |  5 +++--
 tests/functional/flakes/flakes.sh       | 20 ++++++++++++++++++++
 3 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
index 28b6d75f5..7eae5d96e 100644
--- a/doc/manual/src/release-notes/rl-next.md
+++ b/doc/manual/src/release-notes/rl-next.md
@@ -4,7 +4,7 @@
   contents of any `#! nix` lines and the script's location to a single call.
 
   Verbatim strings may be passed in double backtick (```` `` ````) quotes.
-  `--expr` resolves relative paths based on the shebang script location.
+  `--file` and `--expr` resolve relative paths based on the script location.
 
   Some examples:
   ```
diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index 528643dc5..d897a01c4 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -442,8 +442,9 @@ Installables SourceExprCommand::parseInstallables(
             auto e = state->parseStdin();
             state->eval(e, *vFile);
         }
-        else if (file)
-            state->evalFile(lookupFileArg(*state, *file), *vFile);
+        else if (file) {
+            state->evalFile(lookupFileArg(*state, *file, CanonPath::fromCwd(getCommandBaseDir())), *vFile);
+        }
         else {
             CanonPath dir(CanonPath::fromCwd(getCommandBaseDir()));
             auto e = state->parseExprFromString(*expr, state->rootPath(dir));
diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh
index 76f3495dd..ccf1699f9 100644
--- a/tests/functional/flakes/flakes.sh
+++ b/tests/functional/flakes/flakes.sh
@@ -123,6 +123,25 @@ echo "$@"
 EOF
 chmod +x $nonFlakeDir/shebang-inline-expr.sh
 
+cat > $nonFlakeDir/fooScript.nix <<"EOF"
+let flake = (builtins.getFlake (toString ../flake1)).packages;
+    fooScript = flake.${builtins.currentSystem}.fooScript;
+ in fooScript
+EOF
+
+cat > $nonFlakeDir/shebang-file.sh <<EOF
+#! $(type -P env) nix
+EOF
+cat >> $nonFlakeDir/shebang-file.sh <<"EOF"
+#! nix --offline shell
+#! nix --impure --file ./fooScript.nix
+#! nix --no-write-lock-file --command bash
+set -ex
+foo
+echo "$@"
+EOF
+chmod +x $nonFlakeDir/shebang-file.sh
+
 # Construct a custom registry, additionally test the --registry flag
 nix registry add --registry "$registry" flake1 "git+file://$flake1Dir"
 nix registry add --registry "$registry" flake2 "git+file://$percentEncodedFlake2Dir"
@@ -571,4 +590,5 @@ expectStderr 1 nix flake metadata "$flake2Dir" --no-allow-dirty --reference-lock
 [[ $($nonFlakeDir/shebang.sh "bar") = "foo"$'\n'"bar" ]]
 [[ $($nonFlakeDir/shebang-comments.sh ) = "foo" ]]
 [[ $($nonFlakeDir/shebang-inline-expr.sh baz) = "foo"$'\n'"baz" ]]
+[[ $($nonFlakeDir/shebang-file.sh baz) = "foo"$'\n'"baz" ]]
 expect 1 $nonFlakeDir/shebang-reject.sh 2>&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?'

From 51bb69535b76060582f91e5c044d5752d8e3998b Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 12 May 2023 19:57:36 +0200
Subject: [PATCH 093/294] nix/installables.cc: Use getCommandBaseDir() where
 possible

These usages of the working directory are perhaps unlikely to
interact with shebangs, but the code is more consistent this way,
and we're less likely to miss usages that do interact.
---
 src/libcmd/installables.cc | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index d897a01c4..f840865d2 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -88,7 +88,7 @@ MixFlakeOptions::MixFlakeOptions()
             lockFlags.writeLockFile = false;
             lockFlags.inputOverrides.insert_or_assign(
                 flake::parseInputPath(inputPath),
-                parseFlakeRef(flakeRef, absPath("."), true));
+                parseFlakeRef(flakeRef, absPath(getCommandBaseDir()), true));
         }},
         .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) {
             if (n == 0) {
@@ -130,7 +130,7 @@ MixFlakeOptions::MixFlakeOptions()
             auto evalState = getEvalState();
             auto flake = flake::lockFlake(
                 *evalState,
-                parseFlakeRef(flakeRef, absPath(".")),
+                parseFlakeRef(flakeRef, absPath(getCommandBaseDir())),
                 { .writeLockFile = false });
             for (auto & [inputName, input] : flake.lockFile.root->inputs) {
                 auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes
@@ -294,7 +294,7 @@ void completeFlakeRefWithFragment(
                 prefixRoot = ".";
             }
             auto flakeRefS = std::string(prefix.substr(0, hash));
-            auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath("."));
+            auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(getCommandBaseDir()));
 
             auto evalCache = openEvalCache(*evalState,
                 std::make_shared<flake::LockedFlake>(lockFlake(*evalState, flakeRef, lockFlags)));
@@ -482,7 +482,7 @@ Installables SourceExprCommand::parseInstallables(
             }
 
             try {
-                auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath("."));
+                auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(getCommandBaseDir()));
                 result.push_back(make_ref<InstallableFlake>(
                         this,
                         getEvalState(),
@@ -756,7 +756,7 @@ std::vector<FlakeRef> RawInstallablesCommand::getFlakeRefsForCompletion()
     for (auto i : rawInstallables)
         res.push_back(parseFlakeRefWithFragment(
             expandTilde(i),
-            absPath(".")).first);
+            absPath(getCommandBaseDir())).first);
     return res;
 }
 
@@ -778,7 +778,7 @@ std::vector<FlakeRef> InstallableCommand::getFlakeRefsForCompletion()
     return {
         parseFlakeRefWithFragment(
             expandTilde(_installable),
-            absPath(".")).first
+            absPath(getCommandBaseDir())).first
     };
 }
 

From e91fd837ee997cc1879cc9035158260f3dc7cf67 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 23 Oct 2023 16:16:51 +0200
Subject: [PATCH 094/294] Move shebang docs from rl-next to nix.md

---
 doc/manual/src/release-notes/rl-next.md | 48 -------------------
 src/nix/nix.md                          | 61 +++++++++++++++++++++++++
 2 files changed, 61 insertions(+), 48 deletions(-)

diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
index 7eae5d96e..608699270 100644
--- a/doc/manual/src/release-notes/rl-next.md
+++ b/doc/manual/src/release-notes/rl-next.md
@@ -2,54 +2,6 @@
 
 - The experimental nix command is now a `#!-interpreter` by appending the
   contents of any `#! nix` lines and the script's location to a single call.
-
-  Verbatim strings may be passed in double backtick (```` `` ````) quotes.
-  `--file` and `--expr` resolve relative paths based on the script location.
-
-  Some examples:
-  ```
-  #!/usr/bin/env nix
-  #! nix shell --file ``<nixpkgs>`` hello --command bash
-
-  hello | cowsay
-  ```
-  or with flakes:
-  ```
-  #!/usr/bin/env nix
-  #! nix shell nixpkgs#bash nixpkgs#hello nixpkgs#cowsay --command bash
-
-  hello | cowsay
-  ```
-  or
-  ```bash
-  #! /usr/bin/env nix
-  #! nix shell --impure --expr ``
-  #! nix with (import (builtins.getFlake "nixpkgs") {});
-  #! nix terraform.withPlugins (plugins: [ plugins.openstack ])
-  #! nix ``
-  #! nix --command bash
-
-  terraform "$@"
-  ```
-  or
-  ```
-  #!/usr/bin/env nix
-  //! ```cargo
-  //! [dependencies]
-  //! time = "0.1.25"
-  //! ```
-  /*
-  #!nix shell nixpkgs#rustc nixpkgs#rust-script nixpkgs#cargo --command rust-script
-  */
-  fn main() {
-      for argument in std::env::args().skip(1) {
-          println!("{}", argument);
-      };
-      println!("{}", std::env::var("HOME").expect(""));
-      println!("{}", time::now().rfc822z());
-  }
-  // vim: ft=rust
-  ```
 - [URL flake references](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) now support [percent-encoded](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1) characters.
 
 - [Path-like flake references](@docroot@/command-ref/new-cli/nix3-flake.md#path-like-syntax) now accept arbitrary unicode characters (except `#` and `?`).
diff --git a/src/nix/nix.md b/src/nix/nix.md
index 6e7e8a649..5bf82a8bf 100644
--- a/src/nix/nix.md
+++ b/src/nix/nix.md
@@ -238,4 +238,65 @@ operate are determined as follows:
 Most `nix` subcommands operate on a *Nix store*. These are documented
 in [`nix help-stores`](./nix3-help-stores.md).
 
+# Shebang interpreter 
+
+The `nix` command can be used as a `#!` interpreter.
+Arguments to Nix can be passed on subsequent lines in the script.
+
+Verbatim strings may be passed in double backtick (```` `` ````) quotes.
+
+`--file` and `--expr` resolve relative paths based on the script location.
+
+Examples:
+
+```
+#!/usr/bin/env nix
+#! nix shell --file ``<nixpkgs>`` hello cowsay --command bash
+
+hello | cowsay
+```
+
+or with **flakes**:
+
+```
+#!/usr/bin/env nix
+#! nix shell nixpkgs#bash nixpkgs#hello nixpkgs#cowsay --command bash
+
+hello | cowsay
+```
+
+or with an **expression**:
+
+```bash
+#! /usr/bin/env nix
+#! nix shell --impure --expr ``
+#! nix with (import (builtins.getFlake "nixpkgs") {});
+#! nix terraform.withPlugins (plugins: [ plugins.openstack ])
+#! nix ``
+#! nix --command bash
+
+terraform "$@"
+```
+
+or with cascading interpreters. Note that the `#! nix` lines don't need to follow after the first line, to accomodate other interpreters.
+
+```
+#!/usr/bin/env nix
+//! ```cargo
+//! [dependencies]
+//! time = "0.1.25"
+//! ```
+/*
+#!nix shell nixpkgs#rustc nixpkgs#rust-script nixpkgs#cargo --command rust-script
+*/
+fn main() {
+    for argument in std::env::args().skip(1) {
+        println!("{}", argument);
+    };
+    println!("{}", std::env::var("HOME").expect(""));
+    println!("{}", time::now().rfc822z());
+}
+// vim: ft=rust
+```
+
 )""

From ffd414eb756dcb3c64348551d5dbaf674c0d4900 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 23 Oct 2023 18:38:54 +0200
Subject: [PATCH 095/294] Fix nix shebang interaction with #8131 overhaul
 completions

---
 src/libcmd/installables.cc |  4 +++-
 src/libutil/args.cc        | 19 ++++++++-----------
 src/libutil/args.hh        | 17 +----------------
 src/libutil/args/root.hh   | 14 +++++++++++++-
 4 files changed, 25 insertions(+), 29 deletions(-)

diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index f840865d2..1c6103020 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -294,7 +294,9 @@ void completeFlakeRefWithFragment(
                 prefixRoot = ".";
             }
             auto flakeRefS = std::string(prefix.substr(0, hash));
-            auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(getCommandBaseDir()));
+
+            // TODO: ideally this would use the command base directory instead of assuming ".".
+            auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath("."));
 
             auto evalCache = openEvalCache(*evalState,
                 std::make_shared<flake::LockedFlake>(lockFlake(*evalState, flakeRef, lockFlags)));
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 0012b3f47..5ba1e5c55 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -80,12 +80,6 @@ std::optional<std::string> RootArgs::needsCompletion(std::string_view s)
     return {};
 }
 
-void RootArgs::parseCmdline(const Strings & _cmdline)
-{
-    // Default via 5.1.2.2.1 in C standard
-    Args::parseCmdline(_cmdline, false);
-}
-
 /**
  * Basically this is `typedef std::optional<Parser> Parser(std::string_view s, Strings & r);`
  *
@@ -227,7 +221,7 @@ static Strings parseShebangContent(std::string_view s) {
     return result;
 }
 
-void Args::parseCmdline(const Strings & _cmdline, bool allowShebang)
+void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang)
 {
     Strings pendingArgs;
     bool dashDash = false;
@@ -339,10 +333,13 @@ void Args::parseCmdline(const Strings & _cmdline, bool allowShebang)
 
 Path Args::getCommandBaseDir() const
 {
-    if (parent)
-        return parent->getCommandBaseDir();
-    else
-        return commandBaseDir;
+    assert(parent);
+    return parent->getCommandBaseDir();
+}
+
+Path RootArgs::getCommandBaseDir() const
+{
+    return commandBaseDir;
 }
 
 bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 9c942606e..30a44cd10 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -24,24 +24,9 @@ class AddCompletions;
 
 class Args
 {
-    /**
-     * @brief The command's "working directory", but only set when top level.
-     *
-     * Use getCommandBaseDir() to get the directory regardless of whether this
-     * is a top-level command or subcommand.
-     *
-     * @see getCommandBaseDir()
-     */
-    Path commandBaseDir = ".";
 
 public:
 
-    /**
-     * Parse the command line with argv0, throwing a UsageError if something
-       goes wrong.
-     */
-    void parseCmdline(const Strings & _cmdline, bool allowShebang);
-
     /**
      * Return a short one-line description of the command.
     */
@@ -62,7 +47,7 @@ public:
      *
      * This only returns the correct value after parseCmdline() has run.
      */
-    Path getCommandBaseDir() const;
+    virtual Path getCommandBaseDir() const;
 
 protected:
 
diff --git a/src/libutil/args/root.hh b/src/libutil/args/root.hh
index bb98732a1..5c55c37a5 100644
--- a/src/libutil/args/root.hh
+++ b/src/libutil/args/root.hh
@@ -29,14 +29,26 @@ struct Completions final : AddCompletions
  */
 class RootArgs : virtual public Args
 {
+    /**
+     * @brief The command's "working directory", but only set when top level.
+     *
+     * Use getCommandBaseDir() to get the directory regardless of whether this
+     * is a top-level command or subcommand.
+     *
+     * @see getCommandBaseDir()
+     */
+    Path commandBaseDir = ".";
+
 public:
     /** Parse the command line, throwing a UsageError if something goes
      * wrong.
      */
-    void parseCmdline(const Strings & cmdline);
+    void parseCmdline(const Strings & cmdline, bool allowShebang = false);
 
     std::shared_ptr<Completions> completions;
 
+    Path getCommandBaseDir() const override;
+
 protected:
 
     friend class Args;

From 589d3387769b18de9c8d42035eea7ac1e21c6fde Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 6 Nov 2023 18:19:14 +0100
Subject: [PATCH 096/294] parseShebangs: Make strings with backtick sequences
 representable

---
 src/libutil/args.cc       | 32 ++++++++++++-
 src/libutil/args.hh       |  2 +
 src/libutil/tests/args.cc | 94 +++++++++++++++++++++++++++++++++++++++
 src/nix/nix.md            |  4 +-
 4 files changed, 129 insertions(+), 3 deletions(-)
 create mode 100644 src/libutil/tests/args.cc

diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 5ba1e5c55..4359c5e8e 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -189,12 +189,40 @@ void ParseQuoted::operator()(std::shared_ptr<Parser> &state, Strings & r) {
         throw Error("unterminated quoted string in nix shebang");
     }
     switch (remaining[0]) {
+        case ' ':
+            if ((remaining.size() == 3 && remaining[1] == '`' && remaining[2] == '`')
+                || (remaining.size() > 3 && remaining[1] == '`' && remaining[2] == '`' && remaining[3] != '`')) {
+                // exactly two backticks mark the end of a quoted string, but a preceding space is ignored if present.
+                state = std::make_shared<ParseUnquoted>(ParseUnquoted(remaining.substr(3)));
+                r.push_back(acc);
+                return;
+            }
+            else {
+                // just a normal space
+                acc += remaining[0];
+                remaining = remaining.substr(1);
+                return;
+            }
         case '`':
-            if (remaining.size() > 1 && remaining[1] == '`') {
+            // exactly two backticks mark the end of a quoted string
+            if ((remaining.size() == 2 && remaining[1] == '`')
+                || (remaining.size() > 2 && remaining[1] == '`' && remaining[2] != '`')) {
                 state = std::make_shared<ParseUnquoted>(ParseUnquoted(remaining.substr(2)));
                 r.push_back(acc);
                 return;
             }
+
+            // a sequence of at least 3 backticks is one escape-backtick which is ignored, followed by any number of backticks, which are verbatim
+            else if (remaining.size() >= 3 && remaining[1] == '`' && remaining[2] == '`') {
+                // ignore "escape" backtick
+                remaining = remaining.substr(1);
+                // add the rest
+                while (remaining.size() > 0 && remaining[0] == '`') {
+                    acc += '`';
+                    remaining = remaining.substr(1);
+                }
+                return;
+            }
             else {
                 acc += remaining[0];
                 remaining = remaining.substr(1);
@@ -208,7 +236,7 @@ void ParseQuoted::operator()(std::shared_ptr<Parser> &state, Strings & r) {
     assert(false);
 }
 
-static Strings parseShebangContent(std::string_view s) {
+Strings parseShebangContent(std::string_view s) {
     Strings result;
     std::shared_ptr<Parser> parserState(std::make_shared<ParseUnquoted>(ParseUnquoted(s)));
 
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 30a44cd10..7af82b178 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -409,4 +409,6 @@ public:
     virtual void add(std::string completion, std::string description = "") = 0;
 };
 
+Strings parseShebangContent(std::string_view s);
+
 }
diff --git a/src/libutil/tests/args.cc b/src/libutil/tests/args.cc
new file mode 100644
index 000000000..e7a16b0be
--- /dev/null
+++ b/src/libutil/tests/args.cc
@@ -0,0 +1,94 @@
+#include "../args.hh"
+#include <list>
+
+#include <gtest/gtest.h>
+
+namespace nix {
+
+    TEST(parseShebangContent, basic) {
+        std::list<std::string> r = parseShebangContent("hi there");
+        ASSERT_EQ(r.size(), 2);
+        auto i = r.begin();
+        ASSERT_EQ(*i++, "hi");
+        ASSERT_EQ(*i++, "there");
+    }
+
+    TEST(parseShebangContent, empty) {
+        std::list<std::string> r = parseShebangContent("");
+        ASSERT_EQ(r.size(), 0);
+    }
+
+    TEST(parseShebangContent, doubleBacktick) {
+        std::list<std::string> r = parseShebangContent("``\"ain't that nice\"``");
+        ASSERT_EQ(r.size(), 1);
+        auto i = r.begin();
+        ASSERT_EQ(*i++, "\"ain't that nice\"");
+    }
+
+    TEST(parseShebangContent, doubleBacktickEmpty) {
+        std::list<std::string> r = parseShebangContent("````");
+        ASSERT_EQ(r.size(), 1);
+        auto i = r.begin();
+        ASSERT_EQ(*i++, "");
+    }
+
+    TEST(parseShebangContent, doubleBacktickMarkdownInlineCode) {
+        std::list<std::string> r = parseShebangContent("``# I'm markdown section about `coolFunction` ``");
+        ASSERT_EQ(r.size(), 1);
+        auto i = r.begin();
+        ASSERT_EQ(*i++, "# I'm markdown section about `coolFunction`");
+    }
+
+    TEST(parseShebangContent, doubleBacktickMarkdownCodeBlockNaive) {
+        std::list<std::string> r = parseShebangContent("``Example 1\n```nix\na: a\n``` ``");
+        auto i = r.begin();
+        ASSERT_EQ(r.size(), 1);
+        ASSERT_EQ(*i++, "Example 1\n``nix\na: a\n``");
+    }
+
+    TEST(parseShebangContent, doubleBacktickMarkdownCodeBlockCorrect) {
+        std::list<std::string> r = parseShebangContent("``Example 1\n````nix\na: a\n```` ``");
+        auto i = r.begin();
+        ASSERT_EQ(r.size(), 1);
+        ASSERT_EQ(*i++, "Example 1\n```nix\na: a\n```");
+    }
+
+    TEST(parseShebangContent, doubleBacktickMarkdownCodeBlock2) {
+        std::list<std::string> r = parseShebangContent("``Example 1\n````nix\na: a\n````\nExample 2\n````nix\na: a\n```` ``");
+        auto i = r.begin();
+        ASSERT_EQ(r.size(), 1);
+        ASSERT_EQ(*i++, "Example 1\n```nix\na: a\n```\nExample 2\n```nix\na: a\n```");
+    }
+
+    TEST(parseShebangContent, singleBacktickInDoubleBacktickQuotes) {
+        std::list<std::string> r = parseShebangContent("``` ``");
+        auto i = r.begin();
+        ASSERT_EQ(r.size(), 1);
+        ASSERT_EQ(*i++, "`");
+    }
+
+    TEST(parseShebangContent, singleBacktickAndSpaceInDoubleBacktickQuotes) {
+        std::list<std::string> r = parseShebangContent("```  ``");
+        auto i = r.begin();
+        ASSERT_EQ(r.size(), 1);
+        ASSERT_EQ(*i++, "` ");
+    }
+
+    TEST(parseShebangContent, doubleBacktickInDoubleBacktickQuotes) {
+        std::list<std::string> r = parseShebangContent("````` ``");
+        auto i = r.begin();
+        ASSERT_EQ(r.size(), 1);
+        ASSERT_EQ(*i++, "``");
+    }
+
+    TEST(parseShebangContent, increasingQuotes) {
+        std::list<std::string> r = parseShebangContent("```` ``` `` ````` `` `````` ``");
+        auto i = r.begin();
+        ASSERT_EQ(r.size(), 4);
+        ASSERT_EQ(*i++, "");
+        ASSERT_EQ(*i++, "`");
+        ASSERT_EQ(*i++, "``");
+        ASSERT_EQ(*i++, "```");
+    }
+
+}
\ No newline at end of file
diff --git a/src/nix/nix.md b/src/nix/nix.md
index 5bf82a8bf..eb150f03b 100644
--- a/src/nix/nix.md
+++ b/src/nix/nix.md
@@ -243,7 +243,9 @@ in [`nix help-stores`](./nix3-help-stores.md).
 The `nix` command can be used as a `#!` interpreter.
 Arguments to Nix can be passed on subsequent lines in the script.
 
-Verbatim strings may be passed in double backtick (```` `` ````) quotes.
+Verbatim strings may be passed in double backtick (```` `` ````) quotes. <!-- that's markdown for two backticks in inline code. -->
+Sequences of _n_ backticks of 3 or longer are parsed as _n-1_ literal backticks.
+A single space before the closing ```` `` ```` is ignored if present.
 
 `--file` and `--expr` resolve relative paths based on the script location.
 

From ab69dc4da3ce5dc270e11b460c5b99f549bcf5d3 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 6 Nov 2023 19:15:36 +0100
Subject: [PATCH 097/294] Test parseShebangContent round trip

---
 src/libutil/tests/args.cc | 74 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 74 insertions(+)

diff --git a/src/libutil/tests/args.cc b/src/libutil/tests/args.cc
index e7a16b0be..bea74a8c8 100644
--- a/src/libutil/tests/args.cc
+++ b/src/libutil/tests/args.cc
@@ -1,7 +1,9 @@
 #include "../args.hh"
+#include "libutil/fs-sink.hh"
 #include <list>
 
 #include <gtest/gtest.h>
+#include <rapidcheck/gtest.h>
 
 namespace nix {
 
@@ -91,4 +93,76 @@ namespace nix {
         ASSERT_EQ(*i++, "```");
     }
 
+
+#ifndef COVERAGE
+
+// quick and dirty
+static inline std::string escape(std::string_view s_) {
+
+    std::string_view s = s_;
+    std::string r = "``";
+
+    // make a guess to allocate ahead of time
+    r.reserve(
+        // plain chars
+        s.size()
+        // quotes
+        + 5
+        // some "escape" backticks
+        + s.size() / 8);
+
+    while (!s.empty()) {
+        if (s[0] == '`' && s.size() >= 2 && s[1] == '`') {
+            // escape it
+            r += "`";
+            while (!s.empty() && s[0] == '`') {
+                r += "`";
+                s = s.substr(1);
+            }
+        } else {
+            r += s[0];
+            s = s.substr(1);
+        }
+    }
+
+    if (!r.empty()
+        && (
+            r[r.size() - 1] == '`'
+            || r[r.size() - 1] == ' '
+        )) {
+        r += " ";
+    }
+
+    r += "``";
+
+    return r;
+};
+
+RC_GTEST_PROP(
+    parseShebangContent,
+    prop_round_trip_single,
+    (const std::string & orig))
+{
+    auto escaped = escape(orig);
+    // RC_LOG() << "escaped: <[[" << escaped << "]]>" << std::endl;
+    auto ss = parseShebangContent(escaped);
+    RC_ASSERT(ss.size() == 1);
+    RC_ASSERT(*ss.begin() == orig);
+}
+
+RC_GTEST_PROP(
+    parseShebangContent,
+    prop_round_trip_two,
+    (const std::string & one, const std::string & two))
+{
+    auto ss = parseShebangContent(escape(one) + " " + escape(two));
+    RC_ASSERT(ss.size() == 2);
+    auto i = ss.begin();
+    RC_ASSERT(*i++ == one);
+    RC_ASSERT(*i++ == two);
+}
+
+
+#endif
+
 }
\ No newline at end of file

From c0c7c4b6cd1aefaa65fc11fcdc8df7e608960825 Mon Sep 17 00:00:00 2001
From: Felix Uhl <felix.uhl@netlight.com>
Date: Tue, 7 Nov 2023 21:16:18 +0100
Subject: [PATCH 098/294] Link to shebang interpreter docs from release notes

---
 doc/manual/src/release-notes/rl-next.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
index 608699270..da81ed83b 100644
--- a/doc/manual/src/release-notes/rl-next.md
+++ b/doc/manual/src/release-notes/rl-next.md
@@ -1,7 +1,8 @@
 # Release X.Y (202?-??-??)
 
-- The experimental nix command is now a `#!-interpreter` by appending the
-  contents of any `#! nix` lines and the script's location to a single call.
+- The experimental nix command can now act as a [shebang interpreter](@docroot@/command-ref/new-cli/nix.md#shebang-interpreter)
+  by appending the contents of any `#! nix` lines and the script's location to a single call.
+
 - [URL flake references](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) now support [percent-encoded](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1) characters.
 
 - [Path-like flake references](@docroot@/command-ref/new-cli/nix3-flake.md#path-like-syntax) now accept arbitrary unicode characters (except `#` and `?`).

From 6a47629530469b84d33444119e43c61effa88aa4 Mon Sep 17 00:00:00 2001
From: Jacek Galowicz <jacek@galowicz.de>
Date: Tue, 7 Nov 2023 13:38:52 +0100
Subject: [PATCH 099/294] Fix initialization of struct members (wrong order)

---
 src/libstore/nar-accessor.cc | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc
index 02993680f..58740b685 100644
--- a/src/libstore/nar-accessor.cc
+++ b/src/libstore/nar-accessor.cc
@@ -60,12 +60,22 @@ struct NarAccessor : public SourceAccessor
 
         void createDirectory(const Path & path) override
         {
-            createMember(path, {Type::tDirectory, false, 0, 0});
+            createMember(path, NarMember{ .stat = {
+                .type = Type::tDirectory,
+                .fileSize = 0,
+                .isExecutable = false,
+                .narOffset = 0
+            } });
         }
 
         void createRegularFile(const Path & path) override
         {
-            createMember(path, {Type::tRegular, false, 0, 0});
+            createMember(path, NarMember{ .stat = {
+                .type = Type::tRegular,
+                .fileSize = 0,
+                .isExecutable = false,
+                .narOffset = 0
+            } });
         }
 
         void closeRegularFile() override

From 77dceb2844276217bff321d80f601297f3581530 Mon Sep 17 00:00:00 2001
From: Jacek Galowicz <jacek@galowicz.de>
Date: Tue, 7 Nov 2023 13:39:10 +0100
Subject: [PATCH 100/294] Drop obsolete assert and cast

---
 src/libstore/nar-accessor.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc
index 58740b685..cfbbbd80b 100644
--- a/src/libstore/nar-accessor.cc
+++ b/src/libstore/nar-accessor.cc
@@ -88,9 +88,8 @@ struct NarAccessor : public SourceAccessor
 
         void preallocateContents(uint64_t size) override
         {
-            assert(size <= std::numeric_limits<uint64_t>::max());
             auto & st = parents.top()->stat;
-            st.fileSize = (uint64_t) size;
+            st.fileSize = size;
             st.narOffset = pos;
         }
 

From c581143e0c6721fba455e6616e7c6f07e47000b1 Mon Sep 17 00:00:00 2001
From: Jacek Galowicz <jacek@galowicz.de>
Date: Tue, 7 Nov 2023 13:39:30 +0100
Subject: [PATCH 101/294] Use structured binding for json iteration

---
 src/libstore/nar-accessor.cc | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc
index cfbbbd80b..1a4936736 100644
--- a/src/libstore/nar-accessor.cc
+++ b/src/libstore/nar-accessor.cc
@@ -137,9 +137,8 @@ struct NarAccessor : public SourceAccessor
 
             if (type == "directory") {
                 member.stat = {.type = Type::tDirectory};
-                for (auto i = v["entries"].begin(); i != v["entries"].end(); ++i) {
-                    std::string name = i.key();
-                    recurse(member.children[name], i.value());
+                for (const auto &[name, function] : v["entries"].items()) {
+                    recurse(member.children[name], function);
                 }
             } else if (type == "regular") {
                 member.stat = {

From df8bfe84cca62c89417d676af2c6fbe3bcf23412 Mon Sep 17 00:00:00 2001
From: Jacek Galowicz <jacek@galowicz.de>
Date: Tue, 7 Nov 2023 13:40:21 +0100
Subject: [PATCH 102/294] Fix consts and casts

---
 src/libstore/nar-accessor.cc    | 6 +++---
 src/libstore/nar-accessor.hh    | 2 +-
 src/libstore/nar-info.cc        | 4 ++--
 src/libstore/nar-info.hh        | 4 ++--
 src/libstore/worker-protocol.cc | 8 ++++----
 src/libstore/worker-protocol.hh | 4 ++--
 6 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc
index 1a4936736..15b05fe25 100644
--- a/src/libstore/nar-accessor.cc
+++ b/src/libstore/nar-accessor.cc
@@ -161,7 +161,7 @@ struct NarAccessor : public SourceAccessor
     {
         NarMember * current = &root;
 
-        for (auto & i : path) {
+        for (const auto & i : path) {
             if (current->stat.type != Type::tDirectory) return nullptr;
             auto child = current->children.find(std::string(i));
             if (child == current->children.end()) return nullptr;
@@ -194,7 +194,7 @@ struct NarAccessor : public SourceAccessor
             throw Error("path '%1%' inside NAR file is not a directory", path);
 
         DirEntries res;
-        for (auto & child : i.children)
+        for (const auto & child : i.children)
             res.insert_or_assign(child.first, std::nullopt);
 
         return res;
@@ -259,7 +259,7 @@ json listNar(ref<SourceAccessor> accessor, const CanonPath & path, bool recurse)
         {
             obj["entries"] = json::object();
             json &res2 = obj["entries"];
-            for (auto & [name, type] : accessor->readDirectory(path)) {
+            for (const auto & [name, type] : accessor->readDirectory(path)) {
                 if (recurse) {
                     res2[name] = listNar(accessor, path + name, true);
                 } else
diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh
index 433774524..0043897c6 100644
--- a/src/libstore/nar-accessor.hh
+++ b/src/libstore/nar-accessor.hh
@@ -25,7 +25,7 @@ ref<SourceAccessor> makeNarAccessor(Source & source);
  * readFile() method of the accessor to get the contents of files
  * inside the NAR.
  */
-typedef std::function<std::string(uint64_t, uint64_t)> GetNarBytes;
+using GetNarBytes = std::function<std::string(uint64_t, uint64_t)>;
 
 ref<SourceAccessor> makeLazyNarAccessor(
     const std::string & listing,
diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc
index ae2223fb0..1060a6c8b 100644
--- a/src/libstore/nar-info.cc
+++ b/src/libstore/nar-info.cc
@@ -38,12 +38,12 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
     while (pos < s.size()) {
 
         size_t colon = s.find(':', pos);
-        if (colon == std::string::npos) throw corrupt("expecting ':'");
+        if (colon == s.npos) throw corrupt("expecting ':'");
 
         std::string name(s, pos, colon - pos);
 
         size_t eol = s.find('\n', colon + 2);
-        if (eol == std::string::npos) throw corrupt("expecting '\\n'");
+        if (eol == s.npos) throw corrupt("expecting '\\n'");
 
         std::string value(s, colon + 2, eol - colon - 2);
 
diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh
index cec65ff70..fd538a7cd 100644
--- a/src/libstore/nar-info.hh
+++ b/src/libstore/nar-info.hh
@@ -17,10 +17,10 @@ struct NarInfo : ValidPathInfo
     uint64_t fileSize = 0;
 
     NarInfo() = delete;
-    NarInfo(const Store & store, std::string && name, ContentAddressWithReferences && ca, Hash narHash)
+    NarInfo(const Store & store, std::string name, ContentAddressWithReferences ca, Hash narHash)
         : ValidPathInfo(store, std::move(name), std::move(ca), narHash)
     { }
-    NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { }
+    NarInfo(StorePath path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { }
     NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { }
     NarInfo(const Store & store, const std::string & s, const std::string & whence);
 
diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc
index 1d202f8d1..7118558b1 100644
--- a/src/libstore/worker-protocol.cc
+++ b/src/libstore/worker-protocol.cc
@@ -31,14 +31,14 @@ std::optional<TrustedFlag> WorkerProto::Serialise<std::optional<TrustedFlag>>::r
 void WorkerProto::Serialise<std::optional<TrustedFlag>>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional<TrustedFlag> & optTrusted)
 {
     if (!optTrusted)
-        conn.to << (uint8_t)0;
+        conn.to << uint8_t{0};
     else {
         switch (*optTrusted) {
         case Trusted:
-            conn.to << (uint8_t)1;
+            conn.to << uint8_t{1};
             break;
         case NotTrusted:
-            conn.to << (uint8_t)2;
+            conn.to << uint8_t{2};
             break;
         default:
             assert(false);
@@ -101,7 +101,7 @@ void WorkerProto::Serialise<KeyedBuildResult>::write(const Store & store, Worker
 BuildResult WorkerProto::Serialise<BuildResult>::read(const Store & store, WorkerProto::ReadConn conn)
 {
     BuildResult res;
-    res.status = (BuildResult::Status) readInt(conn.from);
+    res.status = static_cast<BuildResult::Status>(readInt(conn.from));
     conn.from >> res.errorMsg;
     if (GET_PROTOCOL_MINOR(conn.version) >= 29) {
         conn.from
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index dcd54ad16..25d544ba7 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -171,7 +171,7 @@ enum struct WorkerProto::Op : uint64_t
  */
 inline Sink & operator << (Sink & sink, WorkerProto::Op op)
 {
-    return sink << (uint64_t) op;
+    return sink << static_cast<uint64_t>(op);
 }
 
 /**
@@ -181,7 +181,7 @@ inline Sink & operator << (Sink & sink, WorkerProto::Op op)
  */
 inline std::ostream & operator << (std::ostream & s, WorkerProto::Op op)
 {
-    return s << (uint64_t) op;
+    return s << static_cast<uint64_t>(op);
 }
 
 /**

From d854e8696b549de15ac9960736a39302d7846ece Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sat, 4 Nov 2023 15:57:43 -0400
Subject: [PATCH 103/294] Specify the size of the experimental feature array in
 a more robust way

See doc comment for details.
---
 src/libutil/experimental-features.cc | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc
index 47edca3a5..6b9427423 100644
--- a/src/libutil/experimental-features.cc
+++ b/src/libutil/experimental-features.cc
@@ -12,7 +12,19 @@ struct ExperimentalFeatureDetails
     std::string_view description;
 };
 
-constexpr std::array<ExperimentalFeatureDetails, 16> xpFeatureDetails = {{
+/**
+ * If two different PRs both add an experimental feature, and we just
+ * used a number for this, we *woudln't* get merge conflict and the
+ * counter will be incremented once instead of twice, causing a build
+ * failure.
+ *
+ * By instead defining this instead as 1 + the bottom experimental
+ * feature, we either have no issue at all if few features are not added
+ * at the end of the list, or a proper merge conflict if they are.
+ */
+constexpr size_t numXpFeatures = 1 + static_cast<size_t>(Xp::VerifiedFetches);
+
+constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails = {{
     {
         .tag = Xp::CaDerivations,
         .name = "ca-derivations",

From f0adb72c238aa6f21c2f07fe2e434a3adcea975d Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 8 Nov 2023 23:08:05 -0500
Subject: [PATCH 104/294] Mark `fetchTree` as unstable again

As discussed in our last meeting, we need a bit more time, but we are
"time boxing" the work left to do to ensure there is not unbounded
delay.

Rather than putting it back underneath `flakes`, though, put it
underneath its own `fetch-tree` experimental feature (which `flakes`
includes/implies). This signals our commitment to the plan to stabilize
it first without waiting to go through the rest of Flakes, and also will
give users a "release candidate" when we get closer to stabilization.

This reverts commit 4112dd1fc93c9ff03a5a4e8be773c45ebefbbd1f.
---
 doc/manual/src/release-notes/rl-next.md |  3 ++-
 src/libexpr/primops/fetchTree.cc        |  1 +
 src/libutil/config.cc                   |  8 ++++++--
 src/libutil/experimental-features.cc    | 15 +++++++++++++++
 src/libutil/experimental-features.hh    |  1 +
 tests/functional/config.sh              |  3 ++-
 6 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
index da81ed83b..addb7de71 100644
--- a/doc/manual/src/release-notes/rl-next.md
+++ b/doc/manual/src/release-notes/rl-next.md
@@ -17,7 +17,8 @@
 
 - `nix-shell` shebang lines now support single-quoted arguments.
 
-- `builtins.fetchTree` is now marked as stable.
+- `builtins.fetchTree` is now unstable under its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree).
+  As described in the document for that feature, this is because we anticipate polishing it and then stabilizing it before the rest of Flakes.
 
 - The interface for creating and updating lock files has been overhauled:
 
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 3717b9022..8031bf809 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -228,6 +228,7 @@ static RegisterPrimOp primop_fetchTree({
           ```
     )",
     .fun = prim_fetchTree,
+    .experimentalFeature = Xp::FetchTree,
 });
 
 static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v,
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 8e7901133..eddc4a588 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -330,9 +330,13 @@ template<> std::set<ExperimentalFeature> BaseSetting<std::set<ExperimentalFeatur
 {
     std::set<ExperimentalFeature> res;
     for (auto & s : tokenizeString<StringSet>(str)) {
-        if (auto thisXpFeature = parseExperimentalFeature(s); thisXpFeature)
+        if (auto thisXpFeature = parseExperimentalFeature(s); thisXpFeature) {
             res.insert(thisXpFeature.value());
-        else
+            // FIXME: Replace this hack with a proper notion of
+            // experimental feature implications/dependencies.
+            if (thisXpFeature.value() == Xp::Flakes)
+                res.insert(Xp::FetchTree);
+        } else
             warn("unknown experimental feature '%s'", s);
     }
     return res;
diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc
index 6b9427423..b0edbe185 100644
--- a/src/libutil/experimental-features.cc
+++ b/src/libutil/experimental-features.cc
@@ -74,6 +74,21 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
             flake`](@docroot@/command-ref/new-cli/nix3-flake.md) for details.
         )",
     },
+    {
+        .tag = Xp::FetchTree,
+        .name = "fetch-tree",
+        .description = R"(
+            Enable the use of the [`fetchTree`](@docroot@/language/builtins.md#builtins-fetchTree) built-in function in the Nix language.
+
+            `fetchTree` exposes a larger suite of fetching functionality in a more systematic way.
+            The same fetching functionality is always used for for
+            [`flakes`](#xp-feature-flakes).
+
+            This built-in was previously guarded by the `flakes` experimental feature because of that overlap,
+            but since the plan is to work on stabilizing this first (due 2024 Q1), we are putting it underneath a separate feature.
+            Once we've made the changes we want to make, enabling just this feature will serve as a "release candidate" --- allowing users to try out the functionality we want to stabilize and not any other functionality we don't yet want to, in isolation.
+        )",
+    },
     {
         .tag = Xp::NixCommand,
         .name = "nix-command",
diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh
index f005cc9ee..f580fd030 100644
--- a/src/libutil/experimental-features.hh
+++ b/src/libutil/experimental-features.hh
@@ -20,6 +20,7 @@ enum struct ExperimentalFeature
     CaDerivations,
     ImpureDerivations,
     Flakes,
+    FetchTree,
     NixCommand,
     RecursiveNix,
     NoUrlLiterals,
diff --git a/tests/functional/config.sh b/tests/functional/config.sh
index 723f575ed..0780c55d0 100644
--- a/tests/functional/config.sh
+++ b/tests/functional/config.sh
@@ -50,7 +50,8 @@ exp_cores=$(nix show-config | grep '^cores' | cut -d '=' -f 2 | xargs)
 exp_features=$(nix show-config | grep '^experimental-features' | cut -d '=' -f 2 | xargs)
 [[ $prev != $exp_cores ]]
 [[ $exp_cores == "4242" ]]
-[[ $exp_features == "flakes nix-command" ]]
+# flakes implies fetch-tree
+[[ $exp_features == "fetch-tree flakes nix-command" ]]
 
 # Test that it's possible to retrieve a single setting's value
 val=$(nix show-config | grep '^warn-dirty' | cut -d '=' -f  2 | xargs)

From 12953b942c7752568070e0b703b448dd8f16f21b Mon Sep 17 00:00:00 2001
From: Robert Hensing <roberth@users.noreply.github.com>
Date: Thu, 9 Nov 2023 07:08:56 +0100
Subject: [PATCH 105/294] Fixup docs

---
 doc/manual/src/release-notes/rl-next.md | 2 +-
 src/libutil/config.cc                   | 2 --
 src/libutil/experimental-features.cc    | 5 ++---
 3 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
index addb7de71..1e6ad6922 100644
--- a/doc/manual/src/release-notes/rl-next.md
+++ b/doc/manual/src/release-notes/rl-next.md
@@ -17,7 +17,7 @@
 
 - `nix-shell` shebang lines now support single-quoted arguments.
 
-- `builtins.fetchTree` is now unstable under its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree).
+- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree).
   As described in the document for that feature, this is because we anticipate polishing it and then stabilizing it before the rest of Flakes.
 
 - The interface for creating and updating lock files has been overhauled:
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index eddc4a588..96a0a4df8 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -332,8 +332,6 @@ template<> std::set<ExperimentalFeature> BaseSetting<std::set<ExperimentalFeatur
     for (auto & s : tokenizeString<StringSet>(str)) {
         if (auto thisXpFeature = parseExperimentalFeature(s); thisXpFeature) {
             res.insert(thisXpFeature.value());
-            // FIXME: Replace this hack with a proper notion of
-            // experimental feature implications/dependencies.
             if (thisXpFeature.value() == Xp::Flakes)
                 res.insert(Xp::FetchTree);
         } else
diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc
index b0edbe185..88fb55713 100644
--- a/src/libutil/experimental-features.cc
+++ b/src/libutil/experimental-features.cc
@@ -80,9 +80,8 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
         .description = R"(
             Enable the use of the [`fetchTree`](@docroot@/language/builtins.md#builtins-fetchTree) built-in function in the Nix language.
 
-            `fetchTree` exposes a larger suite of fetching functionality in a more systematic way.
-            The same fetching functionality is always used for for
-            [`flakes`](#xp-feature-flakes).
+            `fetchTree` exposes a large suite of fetching functionality in a more systematic way.
+            The [`flakes`](#xp-feature-flakes) feature flag always enables `fetch-tree`.
 
             This built-in was previously guarded by the `flakes` experimental feature because of that overlap,
             but since the plan is to work on stabilizing this first (due 2024 Q1), we are putting it underneath a separate feature.

From a903f85f84b78a28490f3aa9615ba87d070d01d1 Mon Sep 17 00:00:00 2001
From: Artturin <Artturin@artturin.com>
Date: Tue, 31 Oct 2023 01:36:13 +0200
Subject: [PATCH 106/294] `nix-env --query`: fix `--json` ignoring `--drv-path`

```json
{
  "AMB-plugins": {
    "drvPath": "/nix/store/l99cb7h2hy8dg005arsjbd9kx0w05d3h-AMB-plugins-0.8.1.drv",
    "name": "AMB-plugins-0.8.1",
    "outputName": "out",
    "outputs": {
      "out": null
    },
    "pname": "AMB-plugins",
    "system": "x86_64-linux",
    "version": "0.8.1"
  },
  "ArchiSteamFarm": {
    "drvPath": "/nix/store/nhplgyjj34fz6hjmnyih25gxscfh8s7b-ArchiSteamFarm-5.4.12.5.drv",
    "name": "ArchiSteamFarm-5.4.12.5",
    "outputName": "out",
    "outputs": {
      "out": null
    },
    "pname": "ArchiSteamFarm",
    "system": "x86_64-linux",
    "version": "5.4.12.5"
  },
...
```
---
 doc/manual/src/release-notes/rl-next.md | 2 ++
 src/nix-env/nix-env.cc                  | 9 +++++++--
 tests/functional/user-envs.sh           | 1 +
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
index 276252c37..2163a5392 100644
--- a/doc/manual/src/release-notes/rl-next.md
+++ b/doc/manual/src/release-notes/rl-next.md
@@ -12,6 +12,8 @@
 
 - Introduce a new built-in function [`builtins.convertHash`](@docroot@/language/builtins.md#builtins-convertHash).
 
+- Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set.
+
 - `nix-shell` shebang lines now support single-quoted arguments.
 
 - `builtins.fetchTree` is now marked as stable.
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 01742daa8..558d0d6cd 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -922,7 +922,7 @@ static VersionDiff compareVersionAgainstSet(
 }
 
 
-static void queryJSON(Globals & globals, std::vector<DrvInfo> & elems, bool printOutPath, bool printMeta)
+static void queryJSON(Globals & globals, std::vector<DrvInfo> & elems, bool printOutPath, bool printDrvPath, bool printMeta)
 {
     using nlohmann::json;
     json topObj = json::object();
@@ -953,6 +953,11 @@ static void queryJSON(Globals & globals, std::vector<DrvInfo> & elems, bool prin
                 }
             }
 
+            if (printDrvPath) {
+                auto drvPath = i.queryDrvPath();
+                if (drvPath) pkgObj["drvPath"] = globals.state->store->printStorePath(*drvPath);
+            }
+
             if (printMeta) {
                 json &metaObj = pkgObj["meta"];
                 metaObj = json::object();
@@ -1079,7 +1084,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
 
     /* Print the desired columns, or XML output. */
     if (jsonOutput) {
-        queryJSON(globals, elems, printOutPath, printMeta);
+        queryJSON(globals, elems, printOutPath, printDrvPath, printMeta);
         cout << '\n';
         return;
     }
diff --git a/tests/functional/user-envs.sh b/tests/functional/user-envs.sh
index d1260ba04..dcd6b1b97 100644
--- a/tests/functional/user-envs.sh
+++ b/tests/functional/user-envs.sh
@@ -26,6 +26,7 @@ nix-env -f ./user-envs.nix -qa --json --out-path | jq -e '.[] | select(.name ==
     .outputName == "out",
     (.outputs.out | test("'$NIX_STORE_DIR'.*-0\\.1"))
 ] | all'
+nix-env -f ./user-envs.nix -qa --json --drv-path | jq -e '.[] | select(.name == "bar-0.1") | (.drvPath | test("'$NIX_STORE_DIR'.*-0\\.1\\.drv"))'
 
 # Query descriptions.
 nix-env -f ./user-envs.nix -qa '*' --description | grepQuiet silly

From 1d5a48240cd3c5b81939b0562141772323550d99 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 9 Nov 2023 23:10:42 -0500
Subject: [PATCH 107/294] `.editorconfig`: Also affect Perl FFI `xs` file

This way `perl/lib/Nix/Store.xs` is affected.
---
 .editorconfig | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.editorconfig b/.editorconfig
index 887ecadba..86360e658 100644
--- a/.editorconfig
+++ b/.editorconfig
@@ -17,7 +17,7 @@ indent_style = space
 indent_size = 2
 
 # Match c++/shell/perl, set indent to spaces with width of four
-[*.{hpp,cc,hh,sh,pl}]
+[*.{hpp,cc,hh,sh,pl,xs}]
 indent_style = space
 indent_size = 4
 

From cf59ea83ec98522113bf2fd81678537a871d0339 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 10 Nov 2023 13:58:59 +0100
Subject: [PATCH 108/294] configure: Check for libgit2

---
 Makefile.config.in       | 7 ++++---
 configure.ac             | 6 ++++++
 src/libfetchers/local.mk | 2 +-
 3 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/Makefile.config.in b/Makefile.config.in
index 19992fa20..aadece0e1 100644
--- a/Makefile.config.in
+++ b/Makefile.config.in
@@ -8,7 +8,9 @@ CXX = @CXX@
 CXXFLAGS = @CXXFLAGS@
 CXXLTO = @CXXLTO@
 EDITLINE_LIBS = @EDITLINE_LIBS@
+ENABLE_BUILD = @ENABLE_BUILD@
 ENABLE_S3 = @ENABLE_S3@
+ENABLE_TESTS = @ENABLE_TESTS@
 GTEST_LIBS = @GTEST_LIBS@
 HAVE_LIBCPUID = @HAVE_LIBCPUID@
 HAVE_SECCOMP = @HAVE_SECCOMP@
@@ -17,6 +19,7 @@ LDFLAGS = @LDFLAGS@
 LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@
 LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
 LIBCURL_LIBS = @LIBCURL_LIBS@
+LIBGIT2_LIBS = @LIBGIT2_LIBS@
 LIBSECCOMP_LIBS = @LIBSECCOMP_LIBS@
 LOWDOWN_LIBS = @LOWDOWN_LIBS@
 OPENSSL_LIBS = @OPENSSL_LIBS@
@@ -35,6 +38,7 @@ docdir = @docdir@
 embedded_sandbox_shell = @embedded_sandbox_shell@
 exec_prefix = @exec_prefix@
 includedir = @includedir@
+internal_api_docs = @internal_api_docs@
 libdir = @libdir@
 libexecdir = @libexecdir@
 localstatedir = @localstatedir@
@@ -46,6 +50,3 @@ sandbox_shell = @sandbox_shell@
 storedir = @storedir@
 sysconfdir = @sysconfdir@
 system = @system@
-ENABLE_BUILD = @ENABLE_BUILD@
-ENABLE_TESTS = @ENABLE_TESTS@
-internal_api_docs = @internal_api_docs@
diff --git a/configure.ac b/configure.ac
index 75ce7d01d..1cda0852a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -335,9 +335,15 @@ AC_ARG_ENABLE(doc-gen, AS_HELP_STRING([--disable-doc-gen],[disable documentation
   doc_generate=$enableval, doc_generate=yes)
 AC_SUBST(doc_generate)
 
+
 # Look for lowdown library.
 PKG_CHECK_MODULES([LOWDOWN], [lowdown >= 0.9.0], [CXXFLAGS="$LOWDOWN_CFLAGS $CXXFLAGS"])
 
+
+# Look for libgit2.
+PKG_CHECK_MODULES([LIBGIT2], [libgit2])
+
+
 # Setuid installations.
 AC_CHECK_FUNCS([setresuid setreuid lchown])
 
diff --git a/src/libfetchers/local.mk b/src/libfetchers/local.mk
index f21651d77..266e7a211 100644
--- a/src/libfetchers/local.mk
+++ b/src/libfetchers/local.mk
@@ -8,6 +8,6 @@ libfetchers_SOURCES := $(wildcard $(d)/*.cc)
 
 libfetchers_CXXFLAGS += -I src/libutil -I src/libstore
 
-libfetchers_LDFLAGS += -pthread -lgit2 -larchive
+libfetchers_LDFLAGS += -pthread $(LIBGIT2_LIBS) -larchive
 
 libfetchers_LIBS = libutil libstore

From 3d9d5dc18977d21a04299f4a37b366f9a1d32051 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 3 Nov 2023 00:57:19 -0400
Subject: [PATCH 109/294] Create `MemorySink`

This is for writing to a `MemorySourceAccessor`.
---
 src/libutil/memory-source-accessor.cc | 56 +++++++++++++++++++++++++++
 src/libutil/memory-source-accessor.hh | 25 ++++++++++++
 2 files changed, 81 insertions(+)

diff --git a/src/libutil/memory-source-accessor.cc b/src/libutil/memory-source-accessor.cc
index f34f6c091..78a4dd298 100644
--- a/src/libutil/memory-source-accessor.cc
+++ b/src/libutil/memory-source-accessor.cc
@@ -121,4 +121,60 @@ CanonPath MemorySourceAccessor::addFile(CanonPath path, std::string && contents)
     return path;
 }
 
+
+using File = MemorySourceAccessor::File;
+
+void MemorySink::createDirectory(const Path & path)
+{
+    auto * f = dst.open(CanonPath{path}, File { File::Directory { } });
+    if (!f)
+        throw Error("file '%s' cannot be made because some parent file is not a directory", path);
+
+    if (!std::holds_alternative<File::Directory>(f->raw))
+        throw Error("file '%s' is not a directory", path);
+};
+
+void MemorySink::createRegularFile(const Path & path)
+{
+    auto * f = dst.open(CanonPath{path}, File { File::Regular {} });
+    if (!f)
+        throw Error("file '%s' cannot be made because some parent file is not a directory", path);
+    if (!(r = std::get_if<File::Regular>(&f->raw)))
+        throw Error("file '%s' is not a regular file", path);
+}
+
+void MemorySink::closeRegularFile()
+{
+    r = nullptr;
+}
+
+void MemorySink::isExecutable()
+{
+    assert(r);
+    r->executable = true;
+}
+
+void MemorySink::preallocateContents(uint64_t len)
+{
+    assert(r);
+    r->contents.reserve(len);
+}
+
+void MemorySink::receiveContents(std::string_view data)
+{
+    assert(r);
+    r->contents += data;
+}
+
+void MemorySink::createSymlink(const Path & path, const std::string & target)
+{
+    auto * f = dst.open(CanonPath{path}, File { File::Symlink { } });
+    if (!f)
+        throw Error("file '%s' cannot be made because some parent file is not a directory", path);
+    if (auto * s = std::get_if<File::Symlink>(&f->raw))
+        s->target = target;
+    else
+        throw Error("file '%s' is not a symbolic link", path);
+}
+
 }
diff --git a/src/libutil/memory-source-accessor.hh b/src/libutil/memory-source-accessor.hh
index 014fa8098..b908f3713 100644
--- a/src/libutil/memory-source-accessor.hh
+++ b/src/libutil/memory-source-accessor.hh
@@ -1,4 +1,5 @@
 #include "source-accessor.hh"
+#include "fs-sink.hh"
 #include "variant-wrapper.hh"
 
 namespace nix {
@@ -71,4 +72,28 @@ struct MemorySourceAccessor : virtual SourceAccessor
     CanonPath addFile(CanonPath path, std::string && contents);
 };
 
+/**
+ * Write to a `MemorySourceAccessor` at the given path
+ */
+struct MemorySink : ParseSink
+{
+    MemorySourceAccessor & dst;
+
+    MemorySink(MemorySourceAccessor & dst) : dst(dst) { }
+
+    void createDirectory(const Path & path) override;
+
+    void createRegularFile(const Path & path) override;
+    void receiveContents(std::string_view data) override;
+    void isExecutable() override;
+    void closeRegularFile() override;
+
+    void createSymlink(const Path & path, const std::string & target) override;
+
+    void preallocateContents(uint64_t size) override;
+
+private:
+    MemorySourceAccessor::File::Regular * r;
+};
+
 }

From 9afa697ab61ea6bbbb0d88e629b62606681cc744 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 8 Nov 2023 00:30:55 -0500
Subject: [PATCH 110/294] Refactor bash test build system a bit

The basic idea here is to separate a few intertwined notions:

1. Not all "run bash tests" are "install tests"

2. Not all "run bash tests" use `tests/functional/init.sh`, or any
   pre-test initialization at all.

This will used in the next commit when we have a test that check unit
test golden master data.

Also, move our custom `PS4` from the test to the test runner, as it is
part of how we want to display the tests, not the test themselves.

Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
---
 doc/manual/src/contributing/testing.md        | 10 ++++----
 mk/common-test.sh                             | 24 ++++++++++++++-----
 mk/debug-test.sh                              |  5 +++-
 mk/lib.mk                                     |  7 +++---
 mk/run-test.sh                                |  5 +++-
 mk/tests.mk                                   | 21 +++++++++-------
 .../common/vars-and-functions.sh.in           |  2 +-
 7 files changed, 48 insertions(+), 26 deletions(-)

diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md
index 3d75ebe7b..0b45b88a3 100644
--- a/doc/manual/src/contributing/testing.md
+++ b/doc/manual/src/contributing/testing.md
@@ -133,17 +133,17 @@ ran test tests/functional/${testName}.sh... [PASS]
 or without `make`:
 
 ```shell-session
-$ ./mk/run-test.sh tests/functional/${testName}.sh
+$ ./mk/run-test.sh tests/functional/${testName}.sh tests/functional/init.sh
 ran test tests/functional/${testName}.sh... [PASS]
 ```
 
 To see the complete output, one can also run:
 
 ```shell-session
-$ ./mk/debug-test.sh tests/functional/${testName}.sh
-+ foo
+$ ./mk/debug-test.sh tests/functional/${testName}.sh tests/functional/init.sh
++(${testName}.sh:1) foo
 output from foo
-+ bar
++(${testName}.sh:2) bar
 output from bar
 ...
 ```
@@ -175,7 +175,7 @@ edit it like so:
 Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point:
 
 ```shell-session
-$ ./mk/debug-test.sh tests/functional/${testName}.sh
+$ ./mk/debug-test.sh tests/functional/${testName}.sh tests/functional/init.sh
 ...
 + gdb blash blub
 GNU gdb (GDB) 12.1
diff --git a/mk/common-test.sh b/mk/common-test.sh
index 7ab25febf..00ccd1584 100644
--- a/mk/common-test.sh
+++ b/mk/common-test.sh
@@ -1,15 +1,27 @@
-test_dir=tests/functional
+# Remove overall test dir (at most one of the two should match) and
+# remove file extension.
+test_name=$(echo -n "$test" | sed \
+    -e "s|^unit-test-data/||" \
+    -e "s|^tests/functional/||" \
+    -e "s|\.sh$||" \
+    )
 
-test=$(echo -n "$test" | sed -e "s|^$test_dir/||")
-
-TESTS_ENVIRONMENT=("TEST_NAME=${test%.*}" 'NIX_REMOTE=')
+TESTS_ENVIRONMENT=(
+    "TEST_NAME=$test_name"
+    'NIX_REMOTE='
+    'PS4=+(${BASH_SOURCE[0]-$0}:$LINENO) '
+)
 
 : ${BASH:=/usr/bin/env bash}
 
+run () {
+   cd "$(dirname $1)" && env "${TESTS_ENVIRONMENT[@]}" $BASH -x -e -u -o pipefail $(basename $1)
+}
+
 init_test () {
-   cd "$test_dir" && env "${TESTS_ENVIRONMENT[@]}" $BASH -e init.sh 2>/dev/null > /dev/null
+   run "$init" 2>/dev/null > /dev/null
 }
 
 run_test_proper () {
-   cd "$test_dir/$(dirname $test)" && env "${TESTS_ENVIRONMENT[@]}" $BASH -e $(basename $test)
+   run "$test"
 }
diff --git a/mk/debug-test.sh b/mk/debug-test.sh
index b5b628ecd..52482c01e 100755
--- a/mk/debug-test.sh
+++ b/mk/debug-test.sh
@@ -3,9 +3,12 @@
 set -eu -o pipefail
 
 test=$1
+init=${2-}
 
 dir="$(dirname "${BASH_SOURCE[0]}")"
 source "$dir/common-test.sh"
 
-(init_test)
+if [ -n "$init" ]; then
+    (init_test)
+fi
 run_test_proper
diff --git a/mk/lib.mk b/mk/lib.mk
index e86a7f1a4..49abe9862 100644
--- a/mk/lib.mk
+++ b/mk/lib.mk
@@ -122,14 +122,15 @@ $(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(b
 $(foreach script, $(bin-scripts), $(eval programs-list += $(script)))
 $(foreach script, $(noinst-scripts), $(eval programs-list += $(script)))
 $(foreach template, $(template-files), $(eval $(call instantiate-template,$(template))))
+install_test_init=tests/functional/init.sh
 $(foreach test, $(install-tests), \
-  $(eval $(call run-install-test,$(test))) \
+  $(eval $(call run-test,$(test),$(install_test_init))) \
   $(eval installcheck: $(test).test))
 $(foreach test-group, $(install-tests-groups), \
-  $(eval $(call run-install-test-group,$(test-group))) \
+  $(eval $(call run-test-group,$(test-group),$(install_test_init))) \
   $(eval installcheck: $(test-group).test-group) \
   $(foreach test, $($(test-group)-tests), \
-    $(eval $(call run-install-test,$(test))) \
+    $(eval $(call run-test,$(test),$(install_test_init))) \
     $(eval $(test-group).test-group: $(test).test)))
 
 $(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/man$(patsubst .%,%,$(suffix $(file))))))
diff --git a/mk/run-test.sh b/mk/run-test.sh
index 1a1d65930..da9c5a473 100755
--- a/mk/run-test.sh
+++ b/mk/run-test.sh
@@ -8,6 +8,7 @@ yellow=""
 normal=""
 
 test=$1
+init=${2-}
 
 dir="$(dirname "${BASH_SOURCE[0]}")"
 source "$dir/common-test.sh"
@@ -21,7 +22,9 @@ if [ -t 1 ]; then
 fi
 
 run_test () {
-    (init_test 2>/dev/null > /dev/null)
+    if [ -n "$init" ]; then
+        (init_test 2>/dev/null > /dev/null)
+    fi
     log="$(run_test_proper 2>&1)" && status=0 || status=$?
 }
 
diff --git a/mk/tests.mk b/mk/tests.mk
index ec8128bdf..bac9b704a 100644
--- a/mk/tests.mk
+++ b/mk/tests.mk
@@ -2,19 +2,22 @@
 
 test-deps =
 
-define run-install-test
+define run-bash
 
-  .PHONY: $1.test
-  $1.test: $1 $(test-deps)
-	@env BASH=$(bash) $(bash) mk/run-test.sh $1 < /dev/null
-
-  .PHONY: $1.test-debug
-  $1.test-debug: $1 $(test-deps)
-	@env BASH=$(bash) $(bash) mk/debug-test.sh $1 < /dev/null
+  .PHONY: $1
+  $1: $2
+	@env BASH=$(bash) $(bash) $3 < /dev/null
 
 endef
 
-define run-install-test-group
+define run-test
+
+  $(eval $(call run-bash,$1.test,$1 $(test-deps),mk/run-test.sh $1 $2))
+  $(eval $(call run-bash,$1.test-debug,$1 $(test-deps),mk/debug-test.sh $1 $2))
+
+endef
+
+define run-test-group
 
   .PHONY: $1.test-group
 
diff --git a/tests/functional/common/vars-and-functions.sh.in b/tests/functional/common/vars-and-functions.sh.in
index 967d6be54..848988af9 100644
--- a/tests/functional/common/vars-and-functions.sh.in
+++ b/tests/functional/common/vars-and-functions.sh.in
@@ -4,7 +4,7 @@ if [[ -z "${COMMON_VARS_AND_FUNCTIONS_SH_SOURCED-}" ]]; then
 
 COMMON_VARS_AND_FUNCTIONS_SH_SOURCED=1
 
-export PS4='+(${BASH_SOURCE[0]-$0}:$LINENO) '
+set +x
 
 export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default/tests\/functional//}
 export NIX_STORE_DIR

From 20b95d622336cf982082d7daf3075339f6edce70 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sat, 4 Nov 2023 15:35:38 -0400
Subject: [PATCH 111/294] Git object hashing in libutil

This is the core functionality but just unit-tested and not yet made
part of the store layer. This is because there is some tech debt around
(a) repeated boilerplate hashing objects (b) better integration of the
new `SourceAccessor` type that needs to be cleaned up first.

Part of RFC 133

Co-Authored-By: Matthew Bauer <mjbauer95@gmail.com>
Co-Authored-By: Carlo Nucera <carlo.nucera@protonmail.com>
Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
Co-authored-by: Florian Klink <flokli@flokli.de>
---
 src/libutil/experimental-features.cc          |   8 +
 src/libutil/experimental-features.hh          |   1 +
 src/libutil/git.cc                            | 263 +++++++++++++++++-
 src/libutil/git.hh                            | 141 +++++++++-
 src/libutil/serialise.cc                      |   4 +
 src/libutil/serialise.hh                      |   1 +
 src/libutil/tests/git.cc                      | 249 +++++++++++++++--
 src/libutil/tests/local.mk                    |   4 +
 unit-test-data/libutil/git/check-data.sh      |  31 +++
 .../libutil/git/hello-world-blob.bin          | Bin 0 -> 24 bytes
 unit-test-data/libutil/git/hello-world.bin    | Bin 0 -> 16 bytes
 unit-test-data/libutil/git/tree.bin           | Bin 0 -> 100 bytes
 unit-test-data/libutil/git/tree.txt           |   3 +
 13 files changed, 667 insertions(+), 38 deletions(-)
 create mode 100644 unit-test-data/libutil/git/check-data.sh
 create mode 100644 unit-test-data/libutil/git/hello-world-blob.bin
 create mode 100644 unit-test-data/libutil/git/hello-world.bin
 create mode 100644 unit-test-data/libutil/git/tree.bin
 create mode 100644 unit-test-data/libutil/git/tree.txt

diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc
index 88fb55713..ac4d189e1 100644
--- a/src/libutil/experimental-features.cc
+++ b/src/libutil/experimental-features.cc
@@ -96,6 +96,14 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
             [`nix`](@docroot@/command-ref/new-cli/nix.md) for details.
         )",
     },
+    {
+        .tag = Xp::GitHashing,
+        .name = "git-hashing",
+        .description = R"(
+            Allow creating (content-addressed) store objects which are hashed via Git's hashing algorithm.
+            These store objects will not be understandable by older versions of Nix.
+        )",
+    },
     {
         .tag = Xp::RecursiveNix,
         .name = "recursive-nix",
diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh
index f580fd030..c355b8081 100644
--- a/src/libutil/experimental-features.hh
+++ b/src/libutil/experimental-features.hh
@@ -22,6 +22,7 @@ enum struct ExperimentalFeature
     Flakes,
     FetchTree,
     NixCommand,
+    GitHashing,
     RecursiveNix,
     NoUrlLiterals,
     FetchClosure,
diff --git a/src/libutil/git.cc b/src/libutil/git.cc
index f35c2fdb7..a4bd60096 100644
--- a/src/libutil/git.cc
+++ b/src/libutil/git.cc
@@ -1,9 +1,263 @@
-#include "git.hh"
-
+#include <cerrno>
+#include <algorithm>
+#include <vector>
+#include <map>
 #include <regex>
+#include <strings.h> // for strcasecmp
+
+#include "signals.hh"
+#include "config.hh"
+#include "hash.hh"
+#include "posix-source-accessor.hh"
+
+#include "git.hh"
+#include "serialise.hh"
+
+namespace nix::git {
+
+using namespace nix;
+using namespace std::string_literals;
+
+std::optional<Mode> decodeMode(RawMode m) {
+    switch (m) {
+        case (RawMode) Mode::Directory:
+        case (RawMode) Mode::Executable:
+        case (RawMode) Mode::Regular:
+        case (RawMode) Mode::Symlink:
+            return (Mode) m;
+        default:
+            return std::nullopt;
+    }
+}
+
+
+static std::string getStringUntil(Source & source, char byte)
+{
+    std::string s;
+    char n[1];
+    source(std::string_view { n, 1 });
+    while (*n != byte) {
+        s += *n;
+        source(std::string_view { n, 1 });
+    }
+    return s;
+}
+
+
+static std::string getString(Source & source, int n)
+{
+    std::string v;
+    v.resize(n);
+    source(v);
+    return v;
+}
+
+
+void parse(
+    ParseSink & sink,
+    const Path & sinkPath,
+    Source & source,
+    std::function<SinkHook> hook,
+    const ExperimentalFeatureSettings & xpSettings)
+{
+    xpSettings.require(Xp::GitHashing);
+
+    auto type = getString(source, 5);
+
+    if (type == "blob ") {
+        sink.createRegularFile(sinkPath);
+
+        unsigned long long size = std::stoi(getStringUntil(source, 0));
+
+        sink.preallocateContents(size);
+
+        unsigned long long left = size;
+        std::string buf;
+        buf.reserve(65536);
+
+        while (left) {
+            checkInterrupt();
+            buf.resize(std::min((unsigned long long)buf.capacity(), left));
+            source(buf);
+            sink.receiveContents(buf);
+            left -= buf.size();
+        }
+    } else if (type == "tree ") {
+        unsigned long long size = std::stoi(getStringUntil(source, 0));
+        unsigned long long left = size;
+
+        sink.createDirectory(sinkPath);
+
+        while (left) {
+            std::string perms = getStringUntil(source, ' ');
+            left -= perms.size();
+            left -= 1;
+
+            RawMode rawMode = std::stoi(perms, 0, 8);
+            auto modeOpt = decodeMode(rawMode);
+            if (!modeOpt)
+                throw Error("Unknown Git permission: %o", perms);
+            auto mode = std::move(*modeOpt);
+
+            std::string name = getStringUntil(source, '\0');
+            left -= name.size();
+            left -= 1;
+
+            std::string hashs = getString(source, 20);
+            left -= 20;
+
+            Hash hash(htSHA1);
+            std::copy(hashs.begin(), hashs.end(), hash.hash);
+
+            hook(name, TreeEntry {
+                .mode = mode,
+                .hash = hash,
+            });
+
+            if (mode == Mode::Executable)
+                sink.isExecutable();
+        }
+    } else throw Error("input doesn't look like a Git object");
+}
+
+
+std::optional<Mode> convertMode(SourceAccessor::Type type)
+{
+    switch (type) {
+    case SourceAccessor::tSymlink:   return Mode::Symlink;
+    case SourceAccessor::tRegular:   return Mode::Regular;
+    case SourceAccessor::tDirectory: return Mode::Directory;
+    case SourceAccessor::tMisc:      return std::nullopt;
+    default: abort();
+    }
+}
+
+
+void restore(ParseSink & sink, Source & source, std::function<RestoreHook> hook)
+{
+    parse(sink, "", source, [&](Path name, TreeEntry entry) {
+        auto [accessor, from] = hook(entry.hash);
+        auto stat = accessor->lstat(from);
+        auto gotOpt = convertMode(stat.type);
+        if (!gotOpt)
+            throw Error("file '%s' (git hash %s) has an unsupported type",
+                from,
+                entry.hash.to_string(HashFormat::Base16, false));
+        auto & got = *gotOpt;
+        if (got != entry.mode)
+            throw Error("git mode of file '%s' (git hash %s) is %o but expected %o",
+                from,
+                entry.hash.to_string(HashFormat::Base16, false),
+                (RawMode) got,
+                (RawMode) entry.mode);
+        copyRecursive(
+            *accessor, from,
+            sink, name);
+    });
+}
+
+
+void dumpBlobPrefix(
+    uint64_t size, Sink & sink,
+    const ExperimentalFeatureSettings & xpSettings)
+{
+    xpSettings.require(Xp::GitHashing);
+    auto s = fmt("blob %d\0"s, std::to_string(size));
+    sink(s);
+}
+
+
+void dumpTree(const Tree & entries, Sink & sink,
+    const ExperimentalFeatureSettings & xpSettings)
+{
+    xpSettings.require(Xp::GitHashing);
+
+    std::string v1;
+
+    for (auto & [name, entry] : entries) {
+        auto name2 = name;
+        if (entry.mode == Mode::Directory) {
+            assert(name2.back() == '/');
+            name2.pop_back();
+        }
+        v1 += fmt("%o %s\0"s, static_cast<RawMode>(entry.mode), name2);
+        std::copy(entry.hash.hash, entry.hash.hash + entry.hash.hashSize, std::back_inserter(v1));
+    }
+
+    {
+        auto s = fmt("tree %d\0"s, v1.size());
+        sink(s);
+    }
+
+    sink(v1);
+}
+
+
+Mode dump(
+    SourceAccessor & accessor, const CanonPath & path,
+    Sink & sink,
+    std::function<DumpHook> hook,
+    PathFilter & filter,
+    const ExperimentalFeatureSettings & xpSettings)
+{
+    auto st = accessor.lstat(path);
+
+    switch (st.type) {
+    case SourceAccessor::tRegular:
+    {
+        accessor.readFile(path, sink, [&](uint64_t size) {
+            dumpBlobPrefix(size, sink, xpSettings);
+        });
+        return st.isExecutable
+            ? Mode::Executable
+            : Mode::Regular;
+    }
+
+    case SourceAccessor::tDirectory:
+    {
+        Tree entries;
+        for (auto & [name, _] : accessor.readDirectory(path)) {
+            auto child = path + name;
+            if (!filter(child.abs())) continue;
+
+            auto entry = hook(child);
+
+            auto name2 = name;
+            if (entry.mode == Mode::Directory)
+                name2 += "/";
+
+            entries.insert_or_assign(std::move(name2), std::move(entry));
+        }
+        dumpTree(entries, sink, xpSettings);
+        return Mode::Directory;
+    }
+
+    case SourceAccessor::tSymlink:
+    case SourceAccessor::tMisc:
+    default:
+        throw Error("file '%1%' has an unsupported type", path);
+    }
+}
+
+
+TreeEntry dumpHash(
+    HashType ht,
+    SourceAccessor & accessor, const CanonPath & path, PathFilter & filter)
+{
+    std::function<DumpHook> hook;
+    hook = [&](const CanonPath & path) -> TreeEntry {
+        auto hashSink = HashSink(ht);
+        auto mode = dump(accessor, path, hashSink, hook, filter);
+        auto hash = hashSink.finish().first;
+        return {
+            .mode = mode,
+            .hash = hash,
+        };
+    };
+
+    return hook(path);
+}
 
-namespace nix {
-namespace git {
 
 std::optional<LsRemoteRefLine> parseLsRemoteLine(std::string_view line)
 {
@@ -22,4 +276,3 @@ std::optional<LsRemoteRefLine> parseLsRemoteLine(std::string_view line)
 }
 
 }
-}
diff --git a/src/libutil/git.hh b/src/libutil/git.hh
index bf2b9a286..303460072 100644
--- a/src/libutil/git.hh
+++ b/src/libutil/git.hh
@@ -5,9 +5,127 @@
 #include <string_view>
 #include <optional>
 
-namespace nix {
+#include "types.hh"
+#include "serialise.hh"
+#include "hash.hh"
+#include "source-accessor.hh"
+#include "fs-sink.hh"
 
-namespace git {
+namespace nix::git {
+
+using RawMode = uint32_t;
+
+enum struct Mode : RawMode {
+    Directory = 0040000,
+    Executable = 0100755,
+    Regular = 0100644,
+    Symlink = 0120000,
+};
+
+std::optional<Mode> decodeMode(RawMode m);
+
+/**
+ * An anonymous Git tree object entry (no name part).
+ */
+struct TreeEntry
+{
+    Mode mode;
+    Hash hash;
+
+    GENERATE_CMP(TreeEntry, me->mode, me->hash);
+};
+
+/**
+ * A Git tree object, fully decoded and stored in memory.
+ *
+ * Directory names must end in a `/` for sake of sorting. See
+ * https://github.com/mirage/irmin/issues/352
+ */
+using Tree = std::map<std::string, TreeEntry>;
+
+/**
+ * Callback for processing a child hash with `parse`
+ *
+ * The function should
+ *
+ * 1. Obtain the file system objects denoted by `gitHash`
+ *
+ * 2. Ensure they match `mode`
+ *
+ * 3. Feed them into the same sink `parse` was called with
+ *
+ * Implementations may seek to memoize resources (bandwidth, storage,
+ * etc.) for the same Git hash.
+ */
+using SinkHook = void(const Path & name, TreeEntry entry);
+
+void parse(
+    ParseSink & sink, const Path & sinkPath,
+    Source & source,
+    std::function<SinkHook> hook,
+    const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
+
+/**
+ * Assists with writing a `SinkHook` step (2).
+ */
+std::optional<Mode> convertMode(SourceAccessor::Type type);
+
+/**
+ * Simplified version of `SinkHook` for `restore`.
+ *
+ * Given a `Hash`, return a `SourceAccessor` and `CanonPath` pointing to
+ * the file system object with that path.
+ */
+using RestoreHook = std::pair<SourceAccessor *, CanonPath>(Hash);
+
+/**
+ * Wrapper around `parse` and `RestoreSink`
+ */
+void restore(ParseSink & sink, Source & source, std::function<RestoreHook> hook);
+
+/**
+ * Dumps a single file to a sink
+ *
+ * @param xpSettings for testing purposes
+ */
+void dumpBlobPrefix(
+    uint64_t size, Sink & sink,
+    const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
+
+/**
+ * Dumps a representation of a git tree to a sink
+ */
+void dumpTree(
+    const Tree & entries, Sink & sink,
+    const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
+
+/**
+ * Callback for processing a child with `dump`
+ *
+ * The function should return the Git hash and mode of the file at the
+ * given path in the accessor passed to `dump`.
+ *
+ * Note that if the child is a directory, its child in must also be so
+ * processed in order to compute this information.
+ */
+using DumpHook = TreeEntry(const CanonPath & path);
+
+Mode dump(
+    SourceAccessor & accessor, const CanonPath & path,
+    Sink & sink,
+    std::function<DumpHook> hook,
+    PathFilter & filter = defaultPathFilter,
+    const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
+
+/**
+ * Recursively dumps path, hashing as we go.
+ *
+ * A smaller wrapper around `dump`.
+ */
+TreeEntry dumpHash(
+    HashType ht,
+    SourceAccessor & accessor, const CanonPath & path,
+    PathFilter & filter = defaultPathFilter);
 
 /**
  * A line from the output of `git ls-remote --symref`.
@@ -16,15 +134,17 @@ namespace git {
  *
  * - Symbolic references of the form
  *
- *      ref: {target}	{reference}
- *
- *    where {target} is itself a reference and {reference} is optional
+ *   ```
+ *   ref: {target} {reference}
+ *   ```
+ *   where {target} is itself a reference and {reference} is optional
  *
  * - Object references of the form
  *
- *      {target}	{reference}
- *
- *    where {target} is a commit id and {reference} is mandatory
+ *   ```
+ *   {target}  {reference}
+ *   ```
+ *   where {target} is a commit id and {reference} is mandatory
  */
 struct LsRemoteRefLine {
     enum struct Kind {
@@ -36,8 +156,9 @@ struct LsRemoteRefLine {
     std::optional<std::string> reference;
 };
 
+/**
+ * Parse an `LsRemoteRefLine`
+ */
 std::optional<LsRemoteRefLine> parseLsRemoteLine(std::string_view line);
 
 }
-
-}
diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index 725ddbb8d..d7950b11b 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -74,6 +74,10 @@ void Source::operator () (char * data, size_t len)
     }
 }
 
+void Source::operator () (std::string_view data)
+{
+    (*this)((char *)data.data(), data.size());
+}
 
 void Source::drainInto(Sink & sink)
 {
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 9e07226bf..3f57ce88b 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -73,6 +73,7 @@ struct Source
      * an error if it is not going to be available.
      */
     void operator () (char * data, size_t len);
+    void operator () (std::string_view data);
 
     /**
      * Store up to ‘len’ in the buffer pointed to by ‘data’, and
diff --git a/src/libutil/tests/git.cc b/src/libutil/tests/git.cc
index 5b5715fc2..2842ea4d0 100644
--- a/src/libutil/tests/git.cc
+++ b/src/libutil/tests/git.cc
@@ -1,33 +1,236 @@
-#include "git.hh"
 #include <gtest/gtest.h>
 
+#include "git.hh"
+#include "memory-source-accessor.hh"
+
+#include "tests/characterization.hh"
+
 namespace nix {
 
-    TEST(GitLsRemote, parseSymrefLineWithReference) {
-        auto line = "ref: refs/head/main	HEAD";
-        auto res = git::parseLsRemoteLine(line);
-        ASSERT_TRUE(res.has_value());
-        ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Symbolic);
-        ASSERT_EQ(res->target, "refs/head/main");
-        ASSERT_EQ(res->reference, "HEAD");
+using namespace git;
+
+class GitTest : public CharacterizationTest
+{
+    Path unitTestData = getUnitTestData() + "/libutil/git";
+
+public:
+
+    Path goldenMaster(std::string_view testStem) const override {
+        return unitTestData + "/" + testStem;
     }
 
-    TEST(GitLsRemote, parseSymrefLineWithNoReference) {
-        auto line = "ref: refs/head/main";
-        auto res = git::parseLsRemoteLine(line);
-        ASSERT_TRUE(res.has_value());
-        ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Symbolic);
-        ASSERT_EQ(res->target, "refs/head/main");
-        ASSERT_EQ(res->reference, std::nullopt);
-    }
+    /**
+     * We set these in tests rather than the regular globals so we don't have
+     * to worry about race conditions if the tests run concurrently.
+     */
+    ExperimentalFeatureSettings mockXpSettings;
 
-    TEST(GitLsRemote, parseObjectRefLine) {
-        auto line = "abc123	refs/head/main";
-        auto res = git::parseLsRemoteLine(line);
-        ASSERT_TRUE(res.has_value());
-        ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Object);
-        ASSERT_EQ(res->target, "abc123");
-        ASSERT_EQ(res->reference, "refs/head/main");
+private:
+
+    void SetUp() override
+    {
+        mockXpSettings.set("experimental-features", "git-hashing");
     }
+};
+
+TEST(GitMode, gitMode_directory) {
+    Mode m = Mode::Directory;
+    RawMode r = 0040000;
+    ASSERT_EQ(static_cast<RawMode>(m), r);
+    ASSERT_EQ(decodeMode(r), std::optional { m });
+};
+
+TEST(GitMode, gitMode_executable) {
+    Mode m = Mode::Executable;
+    RawMode r = 0100755;
+    ASSERT_EQ(static_cast<RawMode>(m), r);
+    ASSERT_EQ(decodeMode(r), std::optional { m });
+};
+
+TEST(GitMode, gitMode_regular) {
+    Mode m = Mode::Regular;
+    RawMode r = 0100644;
+    ASSERT_EQ(static_cast<RawMode>(m), r);
+    ASSERT_EQ(decodeMode(r), std::optional { m });
+};
+
+TEST(GitMode, gitMode_symlink) {
+    Mode m = Mode::Symlink;
+    RawMode r = 0120000;
+    ASSERT_EQ(static_cast<RawMode>(m), r);
+    ASSERT_EQ(decodeMode(r), std::optional { m });
+};
+
+TEST_F(GitTest, blob_read) {
+    readTest("hello-world-blob.bin", [&](const auto & encoded) {
+        StringSource in { encoded };
+        StringSink out;
+        RegularFileSink out2 { out };
+        parse(out2, "", in, [](auto &, auto) {}, mockXpSettings);
+
+        auto expected = readFile(goldenMaster("hello-world.bin"));
+
+        ASSERT_EQ(out.s, expected);
+    });
 }
 
+TEST_F(GitTest, blob_write) {
+    writeTest("hello-world-blob.bin", [&]() {
+        auto decoded = readFile(goldenMaster("hello-world.bin"));
+        StringSink s;
+        dumpBlobPrefix(decoded.size(), s, mockXpSettings);
+        s(decoded);
+        return s.s;
+    });
+}
+
+/**
+ * This data is for "shallow" tree tests. However, we use "real" hashes
+ * so that we can check our test data in the corresponding functional
+ * test (`git-hashing/unit-test-data`).
+ */
+const static Tree tree = {
+    {
+        "Foo",
+        {
+            .mode = Mode::Regular,
+            // hello world with special chars from above
+            .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", htSHA1),
+        },
+    },
+    {
+        "bAr",
+        {
+            .mode = Mode::Executable,
+            // ditto
+            .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", htSHA1),
+        },
+    },
+    {
+        "baZ/",
+        {
+            .mode = Mode::Directory,
+            // Empty directory hash
+            .hash = Hash::parseAny("4b825dc642cb6eb9a060e54bf8d69288fbee4904", htSHA1),
+        },
+    },
+};
+
+TEST_F(GitTest, tree_read) {
+    readTest("tree.bin", [&](const auto & encoded) {
+        StringSource in { encoded };
+        NullParseSink out;
+        Tree got;
+        parse(out, "", in, [&](auto & name, auto entry) {
+            auto name2 = name;
+            if (entry.mode == Mode::Directory)
+                name2 += '/';
+            got.insert_or_assign(name2, std::move(entry));
+        }, mockXpSettings);
+
+        ASSERT_EQ(got, tree);
+    });
+}
+
+TEST_F(GitTest, tree_write) {
+    writeTest("tree.bin", [&]() {
+        StringSink s;
+        dumpTree(tree, s, mockXpSettings);
+        return s.s;
+    });
+}
+
+TEST_F(GitTest, both_roundrip) {
+    using File = MemorySourceAccessor::File;
+
+    MemorySourceAccessor files;
+    files.root = File::Directory {
+        .contents {
+            {
+                "foo",
+                File::Regular {
+                    .contents = "hello\n\0\n\tworld!",
+                },
+            },
+            {
+                "bar",
+                File::Directory {
+                    .contents = {
+                        {
+                            "baz",
+                            File::Regular {
+                                .executable = true,
+                                .contents = "good day,\n\0\n\tworld!",
+                            },
+                        },
+                    },
+                },
+            },
+        },
+    };
+
+    std::map<Hash, std::string> cas;
+
+    std::function<DumpHook> dumpHook;
+    dumpHook = [&](const CanonPath & path) {
+        StringSink s;
+        HashSink hashSink { htSHA1 };
+        TeeSink s2 { s, hashSink };
+        auto mode = dump(
+            files, path, s2, dumpHook,
+            defaultPathFilter, mockXpSettings);
+        auto hash = hashSink.finish().first;
+        cas.insert_or_assign(hash, std::move(s.s));
+        return TreeEntry {
+            .mode = mode,
+            .hash = hash,
+        };
+    };
+
+    auto root = dumpHook(CanonPath::root);
+
+    MemorySourceAccessor files2;
+
+    MemorySink sinkFiles2 { files2 };
+
+    std::function<void(const Path, const Hash &)> mkSinkHook;
+    mkSinkHook = [&](const Path prefix, const Hash & hash) {
+        StringSource in { cas[hash] };
+        parse(sinkFiles2, prefix, in, [&](const Path & name, const auto & entry) {
+            mkSinkHook(prefix + "/" + name, entry.hash);
+        }, mockXpSettings);
+    };
+
+    mkSinkHook("", root.hash);
+
+    ASSERT_EQ(files, files2);
+}
+
+TEST(GitLsRemote, parseSymrefLineWithReference) {
+    auto line = "ref: refs/head/main	HEAD";
+    auto res = parseLsRemoteLine(line);
+    ASSERT_TRUE(res.has_value());
+    ASSERT_EQ(res->kind, LsRemoteRefLine::Kind::Symbolic);
+    ASSERT_EQ(res->target, "refs/head/main");
+    ASSERT_EQ(res->reference, "HEAD");
+}
+
+TEST(GitLsRemote, parseSymrefLineWithNoReference) {
+    auto line = "ref: refs/head/main";
+    auto res = parseLsRemoteLine(line);
+    ASSERT_TRUE(res.has_value());
+    ASSERT_EQ(res->kind, LsRemoteRefLine::Kind::Symbolic);
+    ASSERT_EQ(res->target, "refs/head/main");
+    ASSERT_EQ(res->reference, std::nullopt);
+}
+
+TEST(GitLsRemote, parseObjectRefLine) {
+    auto line = "abc123	refs/head/main";
+    auto res = parseLsRemoteLine(line);
+    ASSERT_TRUE(res.has_value());
+    ASSERT_EQ(res->kind, LsRemoteRefLine::Kind::Object);
+    ASSERT_EQ(res->target, "abc123");
+    ASSERT_EQ(res->reference, "refs/head/main");
+}
+
+}
diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk
index 167915439..5a970c0f2 100644
--- a/src/libutil/tests/local.mk
+++ b/src/libutil/tests/local.mk
@@ -27,3 +27,7 @@ libutil-tests_CXXFLAGS += -I src/libutil
 libutil-tests_LIBS = libutil
 
 libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS)
+
+check: unit-test-data/libutil/git/check-data.sh.test
+
+$(eval $(call run-test,unit-test-data/libutil/git/check-data.sh))
diff --git a/unit-test-data/libutil/git/check-data.sh b/unit-test-data/libutil/git/check-data.sh
new file mode 100644
index 000000000..68b705c95
--- /dev/null
+++ b/unit-test-data/libutil/git/check-data.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/git-hashing/unit-test-data
+mkdir -p $TEST_ROOT
+
+repo="$TEST_ROOT/scratch"
+git init "$repo"
+
+git -C "$repo" config user.email "you@example.com"
+git -C "$repo" config user.name "Your Name"
+
+# `-w` to write for tree test
+freshlyAddedHash=$(git -C "$repo" hash-object -w -t blob --stdin < "./hello-world.bin")
+encodingHash=$(sha1sum -b < "./hello-world-blob.bin" | head -c 40)
+
+# If the hashes match, then `hello-world-blob.bin` must be the encoding
+# of `hello-world.bin`.
+[[ "$encodingHash" == "$freshlyAddedHash" ]]
+
+# Create empty directory object for tree test
+echo -n | git -C "$repo" hash-object -w -t tree --stdin
+
+# Relies on both child hashes already existing in the git store
+freshlyAddedHash=$(git -C "$repo" mktree < "./tree.txt")
+encodingHash=$(sha1sum -b < "./tree.bin" | head -c 40)
+
+# If the hashes match, then `tree.bin` must be the encoding of the
+# directory denoted by `tree.txt` interpreted as git directory listing.
+[[ "$encodingHash" == "$freshlyAddedHash" ]]
diff --git a/unit-test-data/libutil/git/hello-world-blob.bin b/unit-test-data/libutil/git/hello-world-blob.bin
new file mode 100644
index 0000000000000000000000000000000000000000..255f5df55ccedb2dae5f541d516896ffffcdb526
GIT binary patch
literal 24
fcmYew$xl)+G-L2c&B@7E;9}t7R0z*6%1HqLQkDjQ

literal 0
HcmV?d00001

diff --git a/unit-test-data/libutil/git/hello-world.bin b/unit-test-data/libutil/git/hello-world.bin
new file mode 100644
index 0000000000000000000000000000000000000000..63ddb340119baf8492d2da53af47e8c7cfcd5eb2
GIT binary patch
literal 16
XcmeZB&B@7E;9}t7R0z*6%1HqLBqsz~

literal 0
HcmV?d00001

diff --git a/unit-test-data/libutil/git/tree.bin b/unit-test-data/libutil/git/tree.bin
new file mode 100644
index 0000000000000000000000000000000000000000..5256ec140702fef5f88bd5750caf7cd57c03e5ac
GIT binary patch
literal 100
zcmXRZN=;R;G-5C`FfcPQQE<!8XGp%g*+Fpj`j$zTZUwJ*e{uZ$*|<$WX>(Ikg(Sx!
ktkNb1K%kJ67{%b-6no6+bl%Pd2~WL$T$|MK`<*8X01f;sp#T5?

literal 0
HcmV?d00001

diff --git a/unit-test-data/libutil/git/tree.txt b/unit-test-data/libutil/git/tree.txt
new file mode 100644
index 000000000..be3d02920
--- /dev/null
+++ b/unit-test-data/libutil/git/tree.txt
@@ -0,0 +1,3 @@
+100644 blob 63ddb340119baf8492d2da53af47e8c7cfcd5eb2	Foo
+100755 blob 63ddb340119baf8492d2da53af47e8c7cfcd5eb2	bAr
+040000 tree 4b825dc642cb6eb9a060e54bf8d69288fbee4904	baZ

From fd5a4a846752873331b6549f0778181dc4ecc2f3 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Fri, 10 Nov 2023 12:12:13 -0500
Subject: [PATCH 112/294] nix upgrade-nix: make the source URL an option

This new option enables organizations to more easily manage their Nix
fleet's deployment, and ensure a consistent and planned rollout of Nix
upgrades.
---
 src/libstore/globals.hh | 10 ++++++++++
 src/nix/upgrade-nix.cc  |  7 +++----
 src/nix/upgrade-nix.md  |  6 ++++--
 3 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 8e034f5a9..27caf42c4 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -1084,6 +1084,16 @@ public:
         true, // document default
         Xp::ConfigurableImpureEnv
     };
+
+    Setting<std::string> upgradeNixStorePathUrl{
+        this,
+        "https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix",
+        "upgrade-nix-store-path-url",
+        R"(
+          Used by `nix upgrade-nix`, the URL of the file that contains the
+          store paths of the latest Nix release.
+        )"
+    };
 };
 
 
diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc
index c529c2363..4c7a74e16 100644
--- a/src/nix/upgrade-nix.cc
+++ b/src/nix/upgrade-nix.cc
@@ -14,7 +14,6 @@ using namespace nix;
 struct CmdUpgradeNix : MixDryRun, StoreCommand
 {
     Path profileDir;
-    std::string storePathsUrl = "https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix";
 
     CmdUpgradeNix()
     {
@@ -30,7 +29,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
             .longName = "nix-store-paths-url",
             .description = "The URL of the file that contains the store paths of the latest Nix release.",
             .labels = {"url"},
-            .handler = {&storePathsUrl}
+            .handler = {&(std::string&) settings.upgradeNixStorePathUrl}
         });
     }
 
@@ -44,7 +43,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
 
     std::string description() override
     {
-        return "upgrade Nix to the stable version declared in Nixpkgs";
+        return "upgrade Nix to the latest stable version";
     }
 
     std::string doc() override
@@ -145,7 +144,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
         Activity act(*logger, lvlInfo, actUnknown, "querying latest Nix version");
 
         // FIXME: use nixos.org?
-        auto req = FileTransferRequest(storePathsUrl);
+        auto req = FileTransferRequest((std::string&) settings.upgradeNixStorePathUrl);
         auto res = getFileTransfer()->download(req);
 
         auto state = std::make_unique<EvalState>(SearchPath{}, store);
diff --git a/src/nix/upgrade-nix.md b/src/nix/upgrade-nix.md
index cce88c397..3a3bf61b9 100644
--- a/src/nix/upgrade-nix.md
+++ b/src/nix/upgrade-nix.md
@@ -16,8 +16,10 @@ R""(
 
 # Description
 
-This command upgrades Nix to the stable version declared in Nixpkgs.
-This stable version is defined in [nix-fallback-paths.nix](https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix)
+This command upgrades Nix to the stable version.
+
+By default, the latest stable version is defined by Nixpkgs, in
+[nix-fallback-paths.nix](https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix)
 and updated manually. It may not always be the latest tagged release.
 
 By default, it locates the directory containing the `nix` binary in the `$PATH`

From 0be84c83b242b6e6a22400727752072b298e7cab Mon Sep 17 00:00:00 2001
From: Tom Bereknyei <tomberek@gmail.com>
Date: Mon, 6 Nov 2023 10:29:37 -0500
Subject: [PATCH 113/294] key and cat: no need for progressBar

otherwise the output will be invisible in common terminal configurations
---
 src/nix/cat.cc  | 2 ++
 src/nix/sigs.cc | 3 +++
 2 files changed, 5 insertions(+)

diff --git a/src/nix/cat.cc b/src/nix/cat.cc
index 6e5a736f2..4df086d4f 100644
--- a/src/nix/cat.cc
+++ b/src/nix/cat.cc
@@ -1,6 +1,7 @@
 #include "command.hh"
 #include "store-api.hh"
 #include "nar-accessor.hh"
+#include "progress-bar.hh"
 
 using namespace nix;
 
@@ -13,6 +14,7 @@ struct MixCat : virtual Args
         auto st = accessor->lstat(CanonPath(path));
         if (st.type != SourceAccessor::Type::tRegular)
             throw Error("path '%1%' is not a regular file", path);
+        stopProgressBar();
         writeFull(STDOUT_FILENO, accessor->readFile(CanonPath(path)));
     }
 };
diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc
index a68616355..39555c9ea 100644
--- a/src/nix/sigs.cc
+++ b/src/nix/sigs.cc
@@ -3,6 +3,7 @@
 #include "shared.hh"
 #include "store-api.hh"
 #include "thread-pool.hh"
+#include "progress-bar.hh"
 
 #include <atomic>
 
@@ -174,6 +175,7 @@ struct CmdKeyGenerateSecret : Command
         if (!keyName)
             throw UsageError("required argument '--key-name' is missing");
 
+        stopProgressBar();
         writeFull(STDOUT_FILENO, SecretKey::generate(*keyName).to_string());
     }
 };
@@ -195,6 +197,7 @@ struct CmdKeyConvertSecretToPublic : Command
     void run() override
     {
         SecretKey secretKey(drainFD(STDIN_FILENO));
+        stopProgressBar();
         writeFull(STDOUT_FILENO, secretKey.toPublicKey().to_string());
     }
 };

From e4cbdd26e0e6a2a5907dff8e60c3645f7d94423a Mon Sep 17 00:00:00 2001
From: Bouke van der Bijl <i@bou.ke>
Date: Mon, 13 Nov 2023 17:13:52 +0100
Subject: [PATCH 114/294] Add TODO comment for include try/catch

---
 src/libutil/config.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 17380b6d8..ab873b4a8 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -125,7 +125,9 @@ static void applyConfigInner(const std::string & contents, const std::string & p
                 try {
                     std::string includedContents = readFile(path);
                     applyConfigInner(includedContents, p, parsedContents);
-                } catch (SysError &) { }
+                } catch (SysError &) {
+                    // TODO: Do we actually want to ignore this? Or is it better to fail?
+                }
             } else if (!ignoreMissing) {
                 throw Error("file '%1%' included from '%2%' not found", p, path);
             }

From d6898cd58b1a685404ba6878c317e60be9473a9a Mon Sep 17 00:00:00 2001
From: Bouke van der Bijl <i@bou.ke>
Date: Mon, 13 Nov 2023 17:14:05 +0100
Subject: [PATCH 115/294] Move applyConfigFile to lambda inside libstore

---
 src/libstore/globals.cc | 11 +++++++++--
 src/libutil/config.cc   |  8 --------
 src/libutil/config.hh   |  6 ------
 3 files changed, 9 insertions(+), 16 deletions(-)

diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 9c25d9868..0aecd2b6a 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -111,7 +111,14 @@ Settings::Settings()
 
 void loadConfFile()
 {
-    globalConfig.applyConfigFile(settings.nixConfDir + "/nix.conf");
+    auto applyConfigFile = [&](const Path & path) {
+        try {
+            std::string contents = readFile(path);
+            globalConfig.applyConfig(contents, path);
+        } catch (SysError &) { }
+    };
+
+    applyConfigFile(settings.nixConfDir + "/nix.conf");
 
     /* We only want to send overrides to the daemon, i.e. stuff from
        ~/.nix/nix.conf or the command line. */
@@ -119,7 +126,7 @@ void loadConfFile()
 
     auto files = settings.nixUserConfFiles;
     for (auto file = files.rbegin(); file != files.rend(); file++) {
-        globalConfig.applyConfigFile(*file);
+        applyConfigFile(*file);
     }
 
     auto nixConfEnv = getEnv("NIX_CONFIG");
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index ab873b4a8..ad16c86bd 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -165,14 +165,6 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string
             set(name, value);
 }
 
-void AbstractConfig::applyConfigFile(const Path & path)
-{
-    try {
-        std::string contents = readFile(path);
-        applyConfig(contents, path);
-    } catch (SysError &) { }
-}
-
 void Config::resetOverridden()
 {
     for (auto & s : _settings)
diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index 38c3ce0c4..d49eb602d 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -82,12 +82,6 @@ public:
      */
     void applyConfig(const std::string & contents, const std::string & path = "<unknown>");
 
-    /**
-     * Applies a nix configuration file
-     * - path: the location of the config file to apply
-     */
-    void applyConfigFile(const Path & path);
-
     /**
      * Resets the `overridden` flag of all Settings
      */

From 742a63b98f2008161fd00bdbbd39b8f1b14f6443 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 13 Nov 2023 22:01:50 +0000
Subject: [PATCH 116/294] build(deps): bump zeebe-io/backport-action from 2.1.0
 to 2.1.1

Bumps [zeebe-io/backport-action](https://github.com/zeebe-io/backport-action) from 2.1.0 to 2.1.1.
- [Release notes](https://github.com/zeebe-io/backport-action/releases)
- [Commits](https://github.com/zeebe-io/backport-action/compare/v2.1.0...v2.1.1)

---
updated-dependencies:
- dependency-name: zeebe-io/backport-action
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/backport.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml
index 893f4a56f..975c90b91 100644
--- a/.github/workflows/backport.yml
+++ b/.github/workflows/backport.yml
@@ -21,7 +21,7 @@ jobs:
           fetch-depth: 0
       - name: Create backport PRs
         # should be kept in sync with `version`
-        uses: zeebe-io/backport-action@v2.1.0
+        uses: zeebe-io/backport-action@v2.1.1
         with:
           # Config README: https://github.com/zeebe-io/backport-action#backport-action
           github_token: ${{ secrets.GITHUB_TOKEN }}

From ad99c8950b86b8f354f5c72efe690d3cba045d99 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andreas=20St=C3=BChrk?= <andy@hammerhartes.de>
Date: Mon, 13 Nov 2023 23:19:27 +0100
Subject: [PATCH 117/294] Update comment to reflect bind mounts are now used
 for store in chroot

---
 src/libstore/build/local-derivation-goal.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index adb011e30..a9f930773 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -652,8 +652,8 @@ void LocalDerivationGoal::startBuilder()
 #if __linux__
         /* Create a temporary directory in which we set up the chroot
            environment using bind-mounts.  We put it in the Nix store
-           to ensure that we can create hard-links to non-directory
-           inputs in the fake Nix store in the chroot (see below). */
+           so that the build outputs can be moved efficiently from the
+           chroot to their final location. */
         chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot";
         deletePath(chrootRootDir);
 

From 21bb180547118e29a66bf091bd6b1dd911b3114d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2023 13:30:51 +0100
Subject: [PATCH 118/294] Use libgit2 with ssh-exec support

See https://github.com/libgit2/libgit2/pull/6617. This ensures that we
get support for ~/.ssh/config, known_hosts etc.
---
 flake.lock                   | 17 +++++++++++++++++
 flake.nix                    |  9 +++++++--
 src/libfetchers/git-utils.cc | 17 -----------------
 3 files changed, 24 insertions(+), 19 deletions(-)

diff --git a/flake.lock b/flake.lock
index 991cef1ee..2b1d96e4e 100644
--- a/flake.lock
+++ b/flake.lock
@@ -16,6 +16,22 @@
         "type": "github"
       }
     },
+    "libgit2": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1697646580,
+        "narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=",
+        "owner": "libgit2",
+        "repo": "libgit2",
+        "rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5",
+        "type": "github"
+      },
+      "original": {
+        "owner": "libgit2",
+        "repo": "libgit2",
+        "type": "github"
+      }
+    },
     "lowdown-src": {
       "flake": false,
       "locked": {
@@ -67,6 +83,7 @@
     "root": {
       "inputs": {
         "flake-compat": "flake-compat",
+        "libgit2": "libgit2",
         "lowdown-src": "lowdown-src",
         "nixpkgs": "nixpkgs",
         "nixpkgs-regression": "nixpkgs-regression"
diff --git a/flake.nix b/flake.nix
index e71aa5374..d6a173081 100644
--- a/flake.nix
+++ b/flake.nix
@@ -7,8 +7,9 @@
   inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
   inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
   inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; };
+  inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; };
 
-  outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src, flake-compat }:
+  outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src, flake-compat, libgit2 }:
 
     let
       inherit (nixpkgs) lib;
@@ -194,7 +195,11 @@
             bzip2 xz brotli editline
             openssl sqlite
             libarchive
-            libgit2
+            (pkgs.libgit2.overrideAttrs (attrs: {
+              src = libgit2;
+              version = libgit2.lastModifiedDate;
+              cmakeFlags = (attrs.cmakeFlags or []) ++ ["-DUSE_SSH=exec"];
+            }))
             boost
             lowdown-nix
             libsodium
diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index 3a0e2d02f..1ec50099b 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -336,9 +336,6 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         const std::string & url,
         const std::string & refspec) override
     {
-        /* FIXME: use libgit2. Unfortunately, it doesn't support
-           ssh_config at the moment. */
-        #if 0
         Remote remote;
 
         if (git_remote_create_anonymous(Setter(remote), *this, url.c_str()))
@@ -352,20 +349,6 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
 
         if (git_remote_fetch(remote.get(), &refspecs2, nullptr, nullptr))
             throw Error("fetching '%s' from '%s': %s", refspec, url, git_error_last()->message);
-        #endif
-
-        // FIXME: git stderr messes up our progress indicator, so
-        // we're using --quiet for now. Should process its stderr.
-        runProgram("git", true,
-            { "-C", path.abs(),
-              "--bare",
-              "fetch",
-              "--quiet",
-              "--force",
-              "--",
-              url,
-              refspec
-            }, {}, true);
     }
 
     void verifyCommit(

From d74d2fdaa721cd7cddceca2e0b4063a1d891bb9f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2023 13:35:26 +0100
Subject: [PATCH 119/294] Move statusCallbackTrampoline

---
 src/libfetchers/git-utils.cc | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index 1ec50099b..ffcc92fc7 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -130,11 +130,6 @@ T peelObject(git_repository * repo, git_object * obj, git_object_t type)
     return obj2;
 }
 
-int statusCallbackTrampoline(const char * path, unsigned int statusFlags, void * payload)
-{
-    return (*((std::function<int(const char * path, unsigned int statusFlags)> *) payload))(path, statusFlags);
-}
-
 struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
 {
     CanonPath path;
@@ -255,6 +250,12 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         return result;
     }
 
+    // Helper for statusCallback below.
+    static int statusCallbackTrampoline(const char * path, unsigned int statusFlags, void * payload)
+    {
+        return (*((std::function<int(const char * path, unsigned int statusFlags)> *) payload))(path, statusFlags);
+    }
+
     WorkdirInfo getWorkdirInfo() override
     {
         WorkdirInfo info;

From 38b07d63479ebdd4f43145264a026a22a72d940b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2023 13:38:03 +0100
Subject: [PATCH 120/294] src/libfetchers/git.cc: Apply suggestion

Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
---
 src/libfetchers/git.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 9c2a7df16..12233ed0a 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -332,7 +332,7 @@ struct GitInputScheme : InputScheme
            whether the working directory is dirty compared to HEAD. */
         GitRepo::WorkdirInfo workdirInfo;
 
-        /* URL of the repo, or its path if isLocal. */
+        /* URL of the repo, or its path if isLocal. Never a `file` URL. */
         std::string url;
 
         void warnDirty() const

From 25cf8f107125eda79e7faece90e7e05093a39e65 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2023 13:57:24 +0100
Subject: [PATCH 121/294] src/libfetchers/union-input-accessor.cc: Apply
 suggestion

Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
---
 src/libfetchers/union-input-accessor.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/libfetchers/union-input-accessor.cc b/src/libfetchers/union-input-accessor.cc
index ae942cb41..f9472efa7 100644
--- a/src/libfetchers/union-input-accessor.cc
+++ b/src/libfetchers/union-input-accessor.cc
@@ -12,8 +12,7 @@ struct UnionInputAccessor : InputAccessor
         // Currently we require a root filesystem. This could be relaxed.
         assert(mounts.contains(CanonPath::root));
 
-        // FIXME: should check that every mount point exists. Or we
-        // could return dummy parent directories automatically.
+        // FIXME: return dummy parent directories automatically?
     }
 
     std::string readFile(const CanonPath & path) override

From 4329bdf6a30fadad66384f0b8c835d7dba9f87b3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2023 13:58:27 +0100
Subject: [PATCH 122/294] Move comment

---
 src/libfetchers/cache.hh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh
index b517d496e..c8d3248bc 100644
--- a/src/libfetchers/cache.hh
+++ b/src/libfetchers/cache.hh
@@ -6,13 +6,14 @@
 
 namespace nix::fetchers {
 
+/*
+ * A cache for arbitrary `Attrs` -> `Attrs` mappings with a timestamp
+ * for expiration.
+ */
 struct Cache
 {
     virtual ~Cache() { }
 
-    /* A cache for arbitrary Attrs -> Attrs mappings with a timestamp
-       for expiration. */
-
     /*
      * Add a value to the cache. The cache is an arbitrary mapping of
      * Attrs to Attrs.

From 21140c987b7a301c01498864efbc3d92be04aced Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2023 13:59:00 +0100
Subject: [PATCH 123/294] Fix doxygen comments

---
 src/libfetchers/cache.hh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh
index c8d3248bc..f70589267 100644
--- a/src/libfetchers/cache.hh
+++ b/src/libfetchers/cache.hh
@@ -6,7 +6,7 @@
 
 namespace nix::fetchers {
 
-/*
+/**
  * A cache for arbitrary `Attrs` -> `Attrs` mappings with a timestamp
  * for expiration.
  */
@@ -14,7 +14,7 @@ struct Cache
 {
     virtual ~Cache() { }
 
-    /*
+    /**
      * Add a value to the cache. The cache is an arbitrary mapping of
      * Attrs to Attrs.
      */
@@ -22,13 +22,13 @@ struct Cache
         const Attrs & inAttrs,
         const Attrs & infoAttrs) = 0;
 
-    /*
+    /**
      * Look up a key with infinite TTL.
      */
     virtual std::optional<Attrs> lookup(
         const Attrs & inAttrs) = 0;
 
-    /*
+    /**
      * Look up a key. Return nothing if its TTL has exceeded
      * `settings.tarballTTL`.
      */
@@ -41,7 +41,7 @@ struct Cache
         Attrs infoAttrs;
     };
 
-    /*
+    /**
      * Look up a key. Return a bool denoting whether its TTL has
      * exceeded `settings.tarballTTL`.
      */

From 7f576f5dfe11c3f6b0e69179de95c921caddda18 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2023 14:01:38 +0100
Subject: [PATCH 124/294] Rename UnionInputAccessor to MountedInputAccessor

---
 src/libfetchers/git.cc                                 | 10 +++++-----
 ...ion-input-accessor.cc => mounted-input-accessor.cc} | 10 +++++-----
 src/libfetchers/mounted-input-accessor.hh              |  9 +++++++++
 src/libfetchers/union-input-accessor.hh                |  9 ---------
 4 files changed, 19 insertions(+), 19 deletions(-)
 rename src/libfetchers/{union-input-accessor.cc => mounted-input-accessor.cc} (86%)
 create mode 100644 src/libfetchers/mounted-input-accessor.hh
 delete mode 100644 src/libfetchers/union-input-accessor.hh

diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 12233ed0a..90c6ad531 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -9,7 +9,7 @@
 #include "processes.hh"
 #include "git.hh"
 #include "fs-input-accessor.hh"
-#include "union-input-accessor.hh"
+#include "mounted-input-accessor.hh"
 #include "git-utils.hh"
 
 #include "fetch-settings.hh"
@@ -587,7 +587,7 @@ struct GitInputScheme : InputScheme
 
         auto accessor = repo->getAccessor(rev);
 
-        /* If the repo has submodules, fetch them and return a union
+        /* If the repo has submodules, fetch them and return a mounted
            input accessor consisting of the accessor for the top-level
            repo and the accessors for the submodules. */
         if (repoInfo.submodules) {
@@ -611,7 +611,7 @@ struct GitInputScheme : InputScheme
 
             if (!mounts.empty()) {
                 mounts.insert_or_assign(CanonPath::root, accessor);
-                accessor = makeUnionInputAccessor(std::move(mounts));
+                accessor = makeMountedInputAccessor(std::move(mounts));
             }
         }
 
@@ -636,7 +636,7 @@ struct GitInputScheme : InputScheme
         ref<InputAccessor> accessor =
             makeFSInputAccessor(CanonPath(repoInfo.url), repoInfo.workdirInfo.files, makeNotAllowedError(repoInfo.url));
 
-        /* If the repo has submodules, return a union input accessor
+        /* If the repo has submodules, return a mounted input accessor
            consisting of the accessor for the top-level repo and the
            accessors for the submodule workdirs. */
         if (repoInfo.submodules && !repoInfo.workdirInfo.submodules.empty()) {
@@ -660,7 +660,7 @@ struct GitInputScheme : InputScheme
             }
 
             mounts.insert_or_assign(CanonPath::root, accessor);
-            accessor = makeUnionInputAccessor(std::move(mounts));
+            accessor = makeMountedInputAccessor(std::move(mounts));
         }
 
         if (!repoInfo.workdirInfo.isDirty) {
diff --git a/src/libfetchers/union-input-accessor.cc b/src/libfetchers/mounted-input-accessor.cc
similarity index 86%
rename from src/libfetchers/union-input-accessor.cc
rename to src/libfetchers/mounted-input-accessor.cc
index f9472efa7..49917f6e5 100644
--- a/src/libfetchers/union-input-accessor.cc
+++ b/src/libfetchers/mounted-input-accessor.cc
@@ -1,12 +1,12 @@
-#include "union-input-accessor.hh"
+#include "mounted-input-accessor.hh"
 
 namespace nix {
 
-struct UnionInputAccessor : InputAccessor
+struct MountedInputAccessor : InputAccessor
 {
     std::map<CanonPath, ref<InputAccessor>> mounts;
 
-    UnionInputAccessor(std::map<CanonPath, ref<InputAccessor>> _mounts)
+    MountedInputAccessor(std::map<CanonPath, ref<InputAccessor>> _mounts)
         : mounts(std::move(_mounts))
     {
         // Currently we require a root filesystem. This could be relaxed.
@@ -71,9 +71,9 @@ struct UnionInputAccessor : InputAccessor
     }
 };
 
-ref<InputAccessor> makeUnionInputAccessor(std::map<CanonPath, ref<InputAccessor>> mounts)
+ref<InputAccessor> makeMountedInputAccessor(std::map<CanonPath, ref<InputAccessor>> mounts)
 {
-    return make_ref<UnionInputAccessor>(std::move(mounts));
+    return make_ref<MountedInputAccessor>(std::move(mounts));
 }
 
 }
diff --git a/src/libfetchers/mounted-input-accessor.hh b/src/libfetchers/mounted-input-accessor.hh
new file mode 100644
index 000000000..b557c5dad
--- /dev/null
+++ b/src/libfetchers/mounted-input-accessor.hh
@@ -0,0 +1,9 @@
+#pragma once
+
+#include "input-accessor.hh"
+
+namespace nix {
+
+ref<InputAccessor> makeMountedInputAccessor(std::map<CanonPath, ref<InputAccessor>> mounts);
+
+}
diff --git a/src/libfetchers/union-input-accessor.hh b/src/libfetchers/union-input-accessor.hh
deleted file mode 100644
index 6a1649c1d..000000000
--- a/src/libfetchers/union-input-accessor.hh
+++ /dev/null
@@ -1,9 +0,0 @@
-#pragma once
-
-#include "input-accessor.hh"
-
-namespace nix {
-
-ref<InputAccessor> makeUnionInputAccessor(std::map<CanonPath, ref<InputAccessor>> mounts);
-
-}

From c257c824475c92cdfda5daa027db334b6a0137f8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2023 14:47:17 +0100
Subject: [PATCH 125/294] Cleanup

---
 src/libfetchers/mounted-input-accessor.cc | 10 ++++------
 src/libutil/canon-path.cc                 |  7 +++++++
 src/libutil/canon-path.hh                 |  6 ++++++
 3 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/src/libfetchers/mounted-input-accessor.cc b/src/libfetchers/mounted-input-accessor.cc
index 49917f6e5..6f397eb17 100644
--- a/src/libfetchers/mounted-input-accessor.cc
+++ b/src/libfetchers/mounted-input-accessor.cc
@@ -54,18 +54,16 @@ struct MountedInputAccessor : InputAccessor
     std::pair<ref<InputAccessor>, CanonPath> resolve(CanonPath path)
     {
         // Find the nearest parent of `path` that is a mount point.
-        std::vector<std::string> ss;
+        std::vector<std::string> subpath;
         while (true) {
             auto i = mounts.find(path);
             if (i != mounts.end()) {
-                auto subpath = CanonPath::root;
-                for (auto j = ss.rbegin(); j != ss.rend(); ++j)
-                    subpath.push(*j);
-                return {i->second, std::move(subpath)};
+                std::reverse(subpath.begin(), subpath.end());
+                return {i->second, CanonPath(subpath)};
             }
 
             assert(!path.isRoot());
-            ss.push_back(std::string(*path.baseName()));
+            subpath.push_back(std::string(*path.baseName()));
             path.pop();
         }
     }
diff --git a/src/libutil/canon-path.cc b/src/libutil/canon-path.cc
index f678fae94..1e465f1f6 100644
--- a/src/libutil/canon-path.cc
+++ b/src/libutil/canon-path.cc
@@ -13,6 +13,13 @@ CanonPath::CanonPath(std::string_view raw, const CanonPath & root)
     : path(absPath((Path) raw, root.abs()))
 { }
 
+CanonPath::CanonPath(const std::vector<std::string> & elems)
+    : path("/")
+{
+    for (auto & s : elems)
+        push(s);
+}
+
 CanonPath CanonPath::fromCwd(std::string_view path)
 {
     return CanonPath(unchecked_t(), absPath((Path) path));
diff --git a/src/libutil/canon-path.hh b/src/libutil/canon-path.hh
index eefe05ed5..6d0519f4f 100644
--- a/src/libutil/canon-path.hh
+++ b/src/libutil/canon-path.hh
@@ -6,6 +6,7 @@
 #include <cassert>
 #include <iostream>
 #include <set>
+#include <vector>
 
 namespace nix {
 
@@ -46,6 +47,11 @@ public:
         : path(std::move(path))
     { }
 
+    /**
+     * Construct a canon path from a vector of elements.
+     */
+    CanonPath(const std::vector<std::string> & elems);
+
     static CanonPath fromCwd(std::string_view path = ".");
 
     static CanonPath root;

From 4944cdb94d03742176cc7881f126e981c0e7e21c Mon Sep 17 00:00:00 2001
From: vicky1999 <ravivignesh2000@gmail.com>
Date: Tue, 14 Nov 2023 19:59:48 +0530
Subject: [PATCH 126/294] nar dump-path command renamed to nar pack

---
 src/nix/dump-path.cc     | 10 +++++++++-
 src/nix/nar-dump-path.md |  3 ++-
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc
index c4edc894b..0850d4c1c 100644
--- a/src/nix/dump-path.cc
+++ b/src/nix/dump-path.cc
@@ -61,4 +61,12 @@ struct CmdDumpPath2 : Command
     }
 };
 
-static auto rDumpPath2 = registerCommand2<CmdDumpPath2>({"nar", "dump-path"});
+struct CmdNarDumpPath : CmdDumpPath2 {
+    void run() override {
+        warn("'nix nar dump-path' is a deprecated alias for 'nix nar pack'");
+        CmdDumpPath2::run();
+    }
+};
+
+static auto rCmdNarPack = registerCommand2<CmdDumpPath2>({"nar", "pack"});
+static auto rCmdNarDumpPath = registerCommand2<CmdNarDumpPath>({"nar", "dump-path"});
diff --git a/src/nix/nar-dump-path.md b/src/nix/nar-dump-path.md
index 26191ad25..29eaacfdb 100644
--- a/src/nix/nar-dump-path.md
+++ b/src/nix/nar-dump-path.md
@@ -5,7 +5,7 @@ R""(
 * To serialise directory `foo` as a NAR:
 
   ```console
-  # nix nar dump-path ./foo > foo.nar
+  # nix nar pack ./foo > foo.nar
   ```
 
 # Description
@@ -15,3 +15,4 @@ This command generates a NAR file containing the serialisation of
 symbolic links. The NAR is written to standard output.
 
 )""
+

From e07e3c106a9ac0537210e62286c4e696573e9f6f Mon Sep 17 00:00:00 2001
From: vicky1999 <ravivignesh2000@gmail.com>
Date: Tue, 14 Nov 2023 20:02:33 +0530
Subject: [PATCH 127/294] code cleanup

---
 src/nix/nar-dump-path.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/nix/nar-dump-path.md b/src/nix/nar-dump-path.md
index 29eaacfdb..de82202de 100644
--- a/src/nix/nar-dump-path.md
+++ b/src/nix/nar-dump-path.md
@@ -15,4 +15,3 @@ This command generates a NAR file containing the serialisation of
 symbolic links. The NAR is written to standard output.
 
 )""
-

From 6ec6b8aa363f566a8da0d6959753efa452b152cc Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2023 15:52:18 +0100
Subject: [PATCH 128/294] Improve git submodule error reporting

---
 src/libfetchers/fetchers.cc | 10 ++++++++++
 src/libfetchers/fetchers.hh |  2 ++
 src/libfetchers/git.cc      |  4 ++--
 3 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index 19e089aa8..c2513e076 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -220,6 +220,16 @@ std::pair<StorePath, Input> Input::fetch(ref<Store> store) const
     return {std::move(storePath), input};
 }
 
+std::pair<ref<InputAccessor>, Input> Input::getAccessor(ref<Store> store) const
+{
+    try {
+        return scheme->getAccessor(store, *this);
+    } catch (Error & e) {
+        e.addTrace({}, "while fetching the input '%s'", to_string());
+        throw;
+    }
+}
+
 Input Input::applyOverrides(
     std::optional<std::string> ref,
     std::optional<Hash> rev) const
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index 6db1615f2..ce5aa4c69 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -83,6 +83,8 @@ public:
      */
     std::pair<StorePath, Input> fetch(ref<Store> store) const;
 
+    std::pair<ref<InputAccessor>, Input> getAccessor(ref<Store> store) const;
+
     Input applyOverrides(
         std::optional<std::string> ref,
         std::optional<Hash> rev) const;
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 90c6ad531..71ae74dde 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -605,7 +605,7 @@ struct GitInputScheme : InputScheme
                 attrs.insert_or_assign("rev", submoduleRev.gitRev());
                 auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs));
                 auto [submoduleAccessor, submoduleInput2] =
-                    submoduleInput.scheme->getAccessor(store, submoduleInput);
+                    submoduleInput.getAccessor(store);
                 mounts.insert_or_assign(submodule.path, submoduleAccessor);
             }
 
@@ -649,7 +649,7 @@ struct GitInputScheme : InputScheme
                 attrs.insert_or_assign("url", submodulePath.abs());
                 auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs));
                 auto [submoduleAccessor, submoduleInput2] =
-                    submoduleInput.scheme->getAccessor(store, submoduleInput);
+                    submoduleInput.getAccessor(store);
 
                 /* If the submodule is dirty, mark this repo dirty as
                    well. */

From 2964a9f562748cc698ee1f6ecf1e0da4e63211b9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2023 16:00:21 +0100
Subject: [PATCH 129/294] Fix relative submodule handling

Tested on

  nix flake prefetch 'git+https://github.com/blender/blender.git?rev=4ed8a360e956daf2591add4d3c9ec0719e2628fe&submodules=1'
---
 src/libfetchers/git-utils.cc | 12 ++++++++++--
 src/libfetchers/git-utils.hh |  4 +++-
 src/libfetchers/git.cc       |  2 +-
 src/libutil/url.cc           |  8 ++++++++
 src/libutil/url.hh           |  5 +++++
 5 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index ffcc92fc7..1edafbf33 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -308,13 +308,21 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
 
     std::vector<std::tuple<Submodule, Hash>> getSubmodules(const Hash & rev) override;
 
-    std::string resolveSubmoduleUrl(const std::string & url) override
+    std::string resolveSubmoduleUrl(
+        const std::string & url,
+        const std::string & base) override
     {
         git_buf buf = GIT_BUF_INIT;
         if (git_submodule_resolve_url(&buf, *this, url.c_str()))
             throw Error("resolving Git submodule URL '%s'", url);
         Finally cleanup = [&]() { git_buf_dispose(&buf); };
-        return buf.ptr;
+
+        std::string res(buf.ptr);
+
+        if (!hasPrefix(res, "/") && res.find("://") == res.npos)
+            res = parseURL(base + "/" + res).canonicalise().to_string();
+
+        return res;
     }
 
     bool hasObject(const Hash & oid_) override
diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh
index 7efbdedce..e0cb2c34f 100644
--- a/src/libfetchers/git-utils.hh
+++ b/src/libfetchers/git-utils.hh
@@ -59,7 +59,9 @@ struct GitRepo
      */
     virtual std::vector<std::tuple<Submodule, Hash>> getSubmodules(const Hash & rev) = 0;
 
-    virtual std::string resolveSubmoduleUrl(const std::string & url) = 0;
+    virtual std::string resolveSubmoduleUrl(
+        const std::string & url,
+        const std::string & base) = 0;
 
     struct TarballInfo
     {
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 71ae74dde..177c8b66e 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -594,7 +594,7 @@ struct GitInputScheme : InputScheme
             std::map<CanonPath, nix::ref<InputAccessor>> mounts;
 
             for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev)) {
-                auto resolved = repo->resolveSubmoduleUrl(submodule.url);
+                auto resolved = repo->resolveSubmoduleUrl(submodule.url, repoInfo.url);
                 debug("Git submodule %s: %s %s %s -> %s",
                     submodule.path, submodule.url, submodule.branch, submoduleRev.gitRev(), resolved);
                 fetchers::Attrs attrs;
diff --git a/src/libutil/url.cc b/src/libutil/url.cc
index 9b438e6cd..57b64d607 100644
--- a/src/libutil/url.cc
+++ b/src/libutil/url.cc
@@ -2,6 +2,7 @@
 #include "url-parts.hh"
 #include "util.hh"
 #include "split.hh"
+#include "canon-path.hh"
 
 namespace nix {
 
@@ -141,6 +142,13 @@ bool ParsedURL::operator ==(const ParsedURL & other) const
         && fragment == other.fragment;
 }
 
+ParsedURL ParsedURL::canonicalise()
+{
+    ParsedURL res(*this);
+    res.path = CanonPath(res.path).abs();
+    return res;
+}
+
 /**
  * Parse a URL scheme of the form '(applicationScheme\+)?transportScheme'
  * into a tuple '(applicationScheme, transportScheme)'
diff --git a/src/libutil/url.hh b/src/libutil/url.hh
index 26c2dcc28..833f54678 100644
--- a/src/libutil/url.hh
+++ b/src/libutil/url.hh
@@ -19,6 +19,11 @@ struct ParsedURL
     std::string to_string() const;
 
     bool operator ==(const ParsedURL & other) const;
+
+    /**
+     * Remove `.` and `..` path elements.
+     */
+    ParsedURL canonicalise();
 };
 
 MakeError(BadURL, Error);

From 9c7749e13508996eb9df83b1692664cc8cdbf952 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 14 Nov 2023 11:42:25 -0500
Subject: [PATCH 130/294] Fix makefile bug confusing `libnixutil-test` exe vs
 lib

The `-exe` variant is the program, the unsuffixed variant is the
library.

The corrected usage matches `libnixstore-test`.
---
 src/libutil/tests/local.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk
index 5a970c0f2..c8b8557cb 100644
--- a/src/libutil/tests/local.mk
+++ b/src/libutil/tests/local.mk
@@ -1,6 +1,6 @@
 check: libutil-tests_RUN
 
-programs += libutil-tests
+programs += libutil-tests-exe
 
 libutil-tests-exe_NAME = libnixutil-tests
 

From 70b396649c127760e4b123da41451aa7456bc68d Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <silvan.mosberger@tweag.io>
Date: Wed, 15 Nov 2023 00:03:44 +0100
Subject: [PATCH 131/294] doc: logical implication is right-associative

    nix-repl> bools = [ false true ]

    nix-repl> combinations = builtins.concatMap (a: builtins.concatMap (b: map (c: { inherit a b c; }) bools) bools) bools

    nix-repl> builtins.all ({ a, b, c }: (a -> b -> c) == (a -> (b -> c))) combinations
    true

    nix-repl> builtins.all ({ a, b, c }: (a -> b -> c) == ((a -> b) -> c)) combinations
    false
---
 doc/manual/src/language/operators.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md
index cc825b4cf..e9cbb5c92 100644
--- a/doc/manual/src/language/operators.md
+++ b/doc/manual/src/language/operators.md
@@ -25,7 +25,7 @@
 | Inequality                             | *expr* `!=` *expr*                         | none          | 11         |
 | Logical conjunction (`AND`)            | *bool* `&&` *bool*                         | left          | 12         |
 | Logical disjunction (`OR`)             | *bool* <code>\|\|</code> *bool*            | left          | 13         |
-| [Logical implication]                  | *bool* `->` *bool*                         | none          | 14         |
+| [Logical implication]                  | *bool* `->` *bool*                         | right         | 14         |
 
 [string]: ./values.md#type-string
 [path]: ./values.md#type-path

From 28909999116781e194e2eb1646f3ccec005e774f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 15 Nov 2023 13:57:20 +0100
Subject: [PATCH 132/294] Show Git fetch progress

---
 src/libfetchers/git-utils.cc | 30 +++++++++++++++++++++++++++++-
 src/libfetchers/git.cc       |  2 --
 src/libmain/progress-bar.cc  |  8 ++++++++
 src/libutil/logging.hh       |  2 ++
 4 files changed, 39 insertions(+), 3 deletions(-)

diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index 1edafbf33..b7ef05c10 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -3,6 +3,7 @@
 #include "cache.hh"
 #include "finally.hh"
 #include "processes.hh"
+#include "signals.hh"
 
 #include <boost/core/span.hpp>
 
@@ -341,10 +342,32 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
 
     ref<InputAccessor> getAccessor(const Hash & rev) override;
 
+    static int sidebandProgressCallback(const char * str, int len, void * payload)
+    {
+        auto act = (Activity *) payload;
+        act->result(resFetchStatus, trim(std::string_view(str, len)));
+        return _isInterrupted ? -1 : 0;
+    }
+
+    static int transferProgressCallback(const git_indexer_progress * stats, void * payload)
+    {
+        auto act = (Activity *) payload;
+        act->result(resFetchStatus,
+            fmt("%d/%d objects received, %d/%d deltas indexed, %.1f MiB",
+                stats->received_objects,
+                stats->total_objects,
+                stats->indexed_deltas,
+                stats->total_deltas,
+                stats->received_bytes / (1024.0 * 1024.0)));
+        return _isInterrupted ? -1 : 0;
+    }
+
     void fetch(
         const std::string & url,
         const std::string & refspec) override
     {
+        Activity act(*logger, lvlTalkative, actFetchTree, fmt("fetching Git repository '%s'", url));
+
         Remote remote;
 
         if (git_remote_create_anonymous(Setter(remote), *this, url.c_str()))
@@ -356,7 +379,12 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
             .count = 1
         };
 
-        if (git_remote_fetch(remote.get(), &refspecs2, nullptr, nullptr))
+        git_fetch_options opts = GIT_FETCH_OPTIONS_INIT;
+        opts.callbacks.payload = &act;
+        opts.callbacks.sideband_progress = sidebandProgressCallback;
+        opts.callbacks.transfer_progress = transferProgressCallback;
+
+        if (git_remote_fetch(remote.get(), &refspecs2, &opts, nullptr))
             throw Error("fetching '%s' from '%s': %s", refspec, url, git_error_last()->message);
     }
 
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 177c8b66e..3e7dcd8de 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -520,8 +520,6 @@ struct GitInputScheme : InputScheme
             }
 
             if (doFetch) {
-                Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", repoInfo.url));
-
                 try {
                     auto fetchRef = repoInfo.allRefs
                         ? "refs/*"
diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc
index a7aee47c3..3aa012ee1 100644
--- a/src/libmain/progress-bar.cc
+++ b/src/libmain/progress-bar.cc
@@ -340,6 +340,14 @@ public:
             state->activitiesByType[type].expected += j;
             update(*state);
         }
+
+        else if (type == resFetchStatus) {
+            auto i = state->its.find(act);
+            assert(i != state->its.end());
+            ActInfo & actInfo = *i->second;
+            actInfo.lastLine = getS(fields, 0);
+            update(*state);
+        }
     }
 
     void update(State & state)
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 5aa6bee95..183f2d8e1 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -23,6 +23,7 @@ typedef enum {
     actQueryPathInfo = 109,
     actPostBuildHook = 110,
     actBuildWaiting = 111,
+    actFetchTree = 112,
 } ActivityType;
 
 typedef enum {
@@ -34,6 +35,7 @@ typedef enum {
     resProgress = 105,
     resSetExpected = 106,
     resPostBuildLogLine = 107,
+    resFetchStatus = 108,
 } ResultType;
 
 typedef uint64_t ActivityId;

From 5dd4ae86877cedaf70ea70d80b89c66b850bdc5a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 15 Nov 2023 14:08:34 +0100
Subject: [PATCH 133/294] Remove unused cacheType field

---
 src/libfetchers/git.cc | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 3e7dcd8de..b066b384c 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -323,8 +323,6 @@ struct GitInputScheme : InputScheme
         bool submodules = false;
         bool allRefs = false;
 
-        std::string cacheType;
-
         /* Whether this is a local, non-bare repository. */
         bool isLocal = false;
 
@@ -371,11 +369,6 @@ struct GitInputScheme : InputScheme
             .allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false)
         };
 
-        repoInfo.cacheType = "git";
-        if (repoInfo.shallow) repoInfo.cacheType += "-shallow";
-        if (repoInfo.submodules) repoInfo.cacheType += "-submodules";
-        if (repoInfo.allRefs) repoInfo.cacheType += "-all-refs";
-
         // file:// URIs are normally not cloned (but otherwise treated the
         // same as remote URIs, i.e. we don't use the working tree or
         // HEAD). Exception: If _NIX_FORCE_HTTP is set, or the repo is a bare git

From 7ab91e72387b96d1926f1b9c95b919020d4ba962 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 15 Nov 2023 14:43:30 +0100
Subject: [PATCH 134/294] Implement shallow fetching

---
 src/libfetchers/git-utils.cc |  4 ++-
 src/libfetchers/git-utils.hh |  3 ++-
 src/libfetchers/git.cc       | 48 +++++++++++++++++++-----------------
 3 files changed, 31 insertions(+), 24 deletions(-)

diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index b7ef05c10..f554dcc5f 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -364,7 +364,8 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
 
     void fetch(
         const std::string & url,
-        const std::string & refspec) override
+        const std::string & refspec,
+        bool shallow) override
     {
         Activity act(*logger, lvlTalkative, actFetchTree, fmt("fetching Git repository '%s'", url));
 
@@ -380,6 +381,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         };
 
         git_fetch_options opts = GIT_FETCH_OPTIONS_INIT;
+        opts.depth = shallow ? 1 : GIT_FETCH_DEPTH_FULL;
         opts.callbacks.payload = &act;
         opts.callbacks.sideband_progress = sidebandProgressCallback;
         opts.callbacks.transfer_progress = transferProgressCallback;
diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh
index e0cb2c34f..1def82071 100644
--- a/src/libfetchers/git-utils.hh
+++ b/src/libfetchers/git-utils.hh
@@ -75,7 +75,8 @@ struct GitRepo
 
     virtual void fetch(
         const std::string & url,
-        const std::string & refspec) = 0;
+        const std::string & refspec,
+        bool shallow) = 0;
 
     /**
      * Verify that commit `rev` is signed by one of the keys in
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index b066b384c..7208a0b6d 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -219,9 +219,6 @@ struct GitInputScheme : InputScheme
                 || name == "publicKeys")
                 experimentalFeatureSettings.require(Xp::VerifiedFetches);
 
-        maybeGetBoolAttr(attrs, "shallow");
-        maybeGetBoolAttr(attrs, "submodules");
-        maybeGetBoolAttr(attrs, "allRefs");
         maybeGetBoolAttr(attrs, "verifyCommit");
 
         if (auto ref = maybeGetStrAttr(attrs, "ref")) {
@@ -234,6 +231,9 @@ struct GitInputScheme : InputScheme
         auto url = fixGitURL(getStrAttr(attrs, "url"));
         parseURL(url);
         input.attrs["url"] = url;
+        getShallowAttr(input);
+        getSubmodulesAttr(input);
+        getAllRefsAttr(input);
         return input;
     }
 
@@ -243,8 +243,10 @@ struct GitInputScheme : InputScheme
         if (url.scheme != "git") url.scheme = "git+" + url.scheme;
         if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev());
         if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref);
-        if (maybeGetBoolAttr(input.attrs, "shallow").value_or(false))
+        if (getShallowAttr(input))
             url.query.insert_or_assign("shallow", "1");
+        if (getSubmodulesAttr(input))
+            url.query.insert_or_assign("submodules", "1");
         if (maybeGetBoolAttr(input.attrs, "verifyCommit").value_or(false))
             url.query.insert_or_assign("verifyCommit", "1");
         auto publicKeys = getPublicKeys(input.attrs);
@@ -319,10 +321,6 @@ struct GitInputScheme : InputScheme
 
     struct RepoInfo
     {
-        bool shallow = false;
-        bool submodules = false;
-        bool allRefs = false;
-
         /* Whether this is a local, non-bare repository. */
         bool isLocal = false;
 
@@ -347,11 +345,21 @@ struct GitInputScheme : InputScheme
         std::string gitDir = ".git";
     };
 
+    bool getShallowAttr(const Input & input) const
+    {
+        return maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
+    }
+
     bool getSubmodulesAttr(const Input & input) const
     {
         return maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
     }
 
+    bool getAllRefsAttr(const Input & input) const
+    {
+        return maybeGetBoolAttr(input.attrs, "allRefs").value_or(false);
+    }
+
     RepoInfo getRepoInfo(const Input & input) const
     {
         auto checkHashType = [&](const std::optional<Hash> & hash)
@@ -363,11 +371,7 @@ struct GitInputScheme : InputScheme
         if (auto rev = input.getRev())
             checkHashType(rev);
 
-        RepoInfo repoInfo {
-            .shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false),
-            .submodules = getSubmodulesAttr(input),
-            .allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false)
-        };
+        RepoInfo repoInfo;
 
         // file:// URIs are normally not cloned (but otherwise treated the
         // same as remote URIs, i.e. we don't use the working tree or
@@ -501,7 +505,7 @@ struct GitInputScheme : InputScheme
             if (auto rev = input.getRev()) {
                 doFetch = !repo->hasObject(*rev);
             } else {
-                if (repoInfo.allRefs) {
+                if (getAllRefsAttr(input)) {
                     doFetch = true;
                 } else {
                     /* If the local ref is older than ‘tarball-ttl’ seconds, do a
@@ -514,7 +518,7 @@ struct GitInputScheme : InputScheme
 
             if (doFetch) {
                 try {
-                    auto fetchRef = repoInfo.allRefs
+                    auto fetchRef = getAllRefsAttr(input)
                         ? "refs/*"
                         : ref.compare(0, 5, "refs/") == 0
                         ? ref
@@ -522,7 +526,7 @@ struct GitInputScheme : InputScheme
                         ? ref
                         : "refs/heads/" + ref;
 
-                    repo->fetch(repoInfo.url, fmt("%s:%s", fetchRef, fetchRef));
+                    repo->fetch(repoInfo.url, fmt("%s:%s", fetchRef, fetchRef), getShallowAttr(input));
                 } catch (Error & e) {
                     if (!pathExists(localRefFile)) throw;
                     logError(e.info());
@@ -556,7 +560,7 @@ struct GitInputScheme : InputScheme
 
         auto isShallow = repo->isShallow();
 
-        if (isShallow && !repoInfo.shallow)
+        if (isShallow && !getShallowAttr(input))
             throw Error("'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified", repoInfo.url);
 
         // FIXME: check whether rev is an ancestor of ref?
@@ -568,7 +572,7 @@ struct GitInputScheme : InputScheme
             {"lastModified", getLastModified(repoInfo, repoDir, rev)},
         });
 
-        if (!repoInfo.shallow)
+        if (!getShallowAttr(input))
             infoAttrs.insert_or_assign("revCount",
                 getRevCount(repoInfo, repoDir, rev));
 
@@ -581,7 +585,7 @@ struct GitInputScheme : InputScheme
         /* If the repo has submodules, fetch them and return a mounted
            input accessor consisting of the accessor for the top-level
            repo and the accessors for the submodules. */
-        if (repoInfo.submodules) {
+        if (getSubmodulesAttr(input)) {
             std::map<CanonPath, nix::ref<InputAccessor>> mounts;
 
             for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev)) {
@@ -607,7 +611,7 @@ struct GitInputScheme : InputScheme
         }
 
         assert(!origRev || origRev == rev);
-        if (!repoInfo.shallow)
+        if (!getShallowAttr(input))
             input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
         input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified"));
 
@@ -619,7 +623,7 @@ struct GitInputScheme : InputScheme
         RepoInfo & repoInfo,
         Input && input) const
     {
-        if (repoInfo.submodules)
+        if (getSubmodulesAttr(input))
             /* Create mountpoints for the submodules. */
             for (auto & submodule : repoInfo.workdirInfo.submodules)
                 repoInfo.workdirInfo.files.insert(submodule.path);
@@ -630,7 +634,7 @@ struct GitInputScheme : InputScheme
         /* If the repo has submodules, return a mounted input accessor
            consisting of the accessor for the top-level repo and the
            accessors for the submodule workdirs. */
-        if (repoInfo.submodules && !repoInfo.workdirInfo.submodules.empty()) {
+        if (getSubmodulesAttr(input) && !repoInfo.workdirInfo.submodules.empty()) {
             std::map<CanonPath, nix::ref<InputAccessor>> mounts;
 
             for (auto & submodule : repoInfo.workdirInfo.submodules) {

From 84128461b68f6274f1cbf309fd019959132f3c2a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <theophane.hufschmitt@tweag.io>
Date: Wed, 15 Nov 2023 09:23:26 +0100
Subject: [PATCH 135/294] Add a new `nix store add` command

Deprecate `nix store add-file` and `nix store add-path`, and replace
them with a single `nix store add` command.
---
 doc/manual/src/release-notes/rl-next.md |  2 +
 src/nix/add-file.md                     | 28 ----------
 src/nix/add-to-store.cc                 | 70 +++++++++++++++++--------
 src/nix/{add-path.md => add.md}         |  2 +-
 tests/functional/add.sh                 | 17 ++++++
 5 files changed, 68 insertions(+), 51 deletions(-)
 delete mode 100644 src/nix/add-file.md
 rename src/nix/{add-path.md => add.md} (94%)

diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
index 1e6ad6922..422f1fce8 100644
--- a/doc/manual/src/release-notes/rl-next.md
+++ b/doc/manual/src/release-notes/rl-next.md
@@ -73,3 +73,5 @@
   [`XDG_DATA_DIRS`](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html#variables) is now populated with the path to the `/share` subdirectory of the current profile.
   This means that command completion scripts, `.desktop` files, and similar artifacts installed via [`nix-env`](@docroot@/command-ref/nix-env.md) or [`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md)
   (experimental) can be found by any program that follows the [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html).
+
+- A new command `nix store add` has been added. It replaces `nix store add-file` and `nix store add-path` which are now deprecated.
diff --git a/src/nix/add-file.md b/src/nix/add-file.md
deleted file mode 100644
index ed237a035..000000000
--- a/src/nix/add-file.md
+++ /dev/null
@@ -1,28 +0,0 @@
-R""(
-
-# Description
-
-Copy the regular file *path* to the Nix store, and print the resulting
-store path on standard output.
-
-> **Warning**
->
-> The resulting store path is not registered as a garbage
-> collector root, so it could be deleted before you have a
-> chance to register it.
-
-# Examples
-
-Add a regular file to the store:
-
-```console
-# echo foo > bar
-
-# nix store add-file ./bar
-/nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar
-
-# cat /nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar
-foo
-```
-
-)""
diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index 39e5cc99d..f9d487ada 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -5,11 +5,22 @@
 
 using namespace nix;
 
+static FileIngestionMethod parseIngestionMethod(std::string_view input)
+{
+    if (input == "flat") {
+        return FileIngestionMethod::Flat;
+    } else if (input == "nar") {
+        return FileIngestionMethod::Recursive;
+    } else {
+        throw UsageError("Unknown hash mode '%s', expect `flat` or `nar`");
+    }
+}
+
 struct CmdAddToStore : MixDryRun, StoreCommand
 {
     Path path;
     std::optional<std::string> namePart;
-    FileIngestionMethod ingestionMethod;
+    FileIngestionMethod ingestionMethod = FileIngestionMethod::Recursive;
 
     CmdAddToStore()
     {
@@ -23,6 +34,23 @@ struct CmdAddToStore : MixDryRun, StoreCommand
             .labels = {"name"},
             .handler = {&namePart},
         });
+
+        addFlag({
+            .longName  = "mode",
+            .shortName = 'n',
+            .description = R"(
+    How to compute the hash of the input.
+    One of:
+
+    - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function.
+
+    - `flat`: Assumes that the input is a single file and directly passes it to the hash function;
+            )",
+            .labels = {"hash-mode"},
+            .handler = {[this](std::string s) {
+                this->ingestionMethod = parseIngestionMethod(s);
+            }},
+        });
     }
 
     void run(ref<Store> store) override
@@ -62,6 +90,22 @@ struct CmdAddToStore : MixDryRun, StoreCommand
     }
 };
 
+struct CmdAdd : CmdAddToStore
+{
+
+    std::string description() override
+    {
+        return "Add a file or directory to the Nix store";
+    }
+
+    std::string doc() override
+    {
+        return
+          #include "add.md"
+          ;
+    }
+};
+
 struct CmdAddFile : CmdAddToStore
 {
     CmdAddFile()
@@ -71,36 +115,18 @@ struct CmdAddFile : CmdAddToStore
 
     std::string description() override
     {
-        return "add a regular file to the Nix store";
-    }
-
-    std::string doc() override
-    {
-        return
-          #include "add-file.md"
-          ;
+        return "Deprecated. Use [`nix store add --mode flat`](@docroot@/command-ref/new-cli/nix3-store-add.md) instead.";
     }
 };
 
 struct CmdAddPath : CmdAddToStore
 {
-    CmdAddPath()
-    {
-        ingestionMethod = FileIngestionMethod::Recursive;
-    }
-
     std::string description() override
     {
-        return "add a path to the Nix store";
-    }
-
-    std::string doc() override
-    {
-        return
-          #include "add-path.md"
-          ;
+        return "Deprecated alias to [`nix store add`](@docroot@/command-ref/new-cli/nix3-store-add.md).";
     }
 };
 
 static auto rCmdAddFile = registerCommand2<CmdAddFile>({"store", "add-file"});
 static auto rCmdAddPath = registerCommand2<CmdAddPath>({"store", "add-path"});
+static auto rCmdAdd = registerCommand2<CmdAdd>({"store", "add"});
diff --git a/src/nix/add-path.md b/src/nix/add.md
similarity index 94%
rename from src/nix/add-path.md
rename to src/nix/add.md
index 87473611d..d38cd21d8 100644
--- a/src/nix/add-path.md
+++ b/src/nix/add.md
@@ -19,7 +19,7 @@ Add a directory to the store:
 # mkdir dir
 # echo foo > dir/bar
 
-# nix store add-path ./dir
+# nix store add ./dir
 /nix/store/6pmjx56pm94n66n4qw1nff0y1crm8nqg-dir
 
 # cat /nix/store/6pmjx56pm94n66n4qw1nff0y1crm8nqg-dir/bar
diff --git a/tests/functional/add.sh b/tests/functional/add.sh
index 5c3eed793..d0fedcb25 100644
--- a/tests/functional/add.sh
+++ b/tests/functional/add.sh
@@ -26,3 +26,20 @@ hash2=$(nix-hash --type sha256 --base32 ./dummy)
 echo $hash2
 
 test "$hash1" = "sha256:$hash2"
+
+#### New style commands
+
+clearStore
+
+(
+    path1=$(nix store add ./dummy)
+    path2=$(nix store add --mode nar ./dummy)
+    path3=$(nix store add-path ./dummy)
+    [[ "$path1" == "$path2" ]]
+    [[ "$path1" == "$path3" ]]
+)
+(
+    path1=$(nix store add --mode flat ./dummy)
+    path2=$(nix store add-file ./dummy)
+    [[ "$path1" == "$path2" ]]
+)

From 5196613e8290a9ee81f1b9d88e7bc61cc3f64d2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <theophane.hufschmitt@tweag.io>
Date: Fri, 18 Nov 2022 11:13:32 +0100
Subject: [PATCH 136/294] Use boost small vectors instead of VLAs in the
 primops

VLAs are a dangerous feature, and their usage triggers an undefined
behavior since theire size can be zero in some cases.
So replace them with `boost::small_vector`s which fit the same goal but
are safer.

It's also incidentally consistently 1% faster on the benchmarks.
---
 src/libexpr/primops.cc | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 8d3a18526..e7587506a 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -29,7 +29,6 @@
 
 #include <cmath>
 
-
 namespace nix {
 
 
@@ -2729,8 +2728,8 @@ static void prim_catAttrs(EvalState & state, const PosIdx pos, Value * * args, V
     auto attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.catAttrs"));
     state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.catAttrs");
 
-    Value * res[args[1]->listSize()];
-    unsigned int found = 0;
+    boost::container::small_vector<Value *, 256> res(args[1]->listSize());
+    size_t found = 0;
 
     for (auto v2 : args[1]->listItems()) {
         state.forceAttrs(*v2, pos, "while evaluating an element in the list passed as second argument to builtins.catAttrs");
@@ -3064,9 +3063,8 @@ static void prim_filter(EvalState & state, const PosIdx pos, Value * * args, Val
 
     state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filter");
 
-    // FIXME: putting this on the stack is risky.
-    Value * vs[args[1]->listSize()];
-    unsigned int k = 0;
+    boost::container::small_vector<Value *, 256> vs(args[1]->listSize());
+    size_t k = 0;
 
     bool same = true;
     for (unsigned int n = 0; n < args[1]->listSize(); ++n) {
@@ -3450,7 +3448,7 @@ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args,
     state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.concatMap");
     auto nrLists = args[1]->listSize();
 
-    Value lists[nrLists];
+    boost::container::small_vector<Value, 64> lists(nrLists);
     size_t len = 0;
 
     for (unsigned int n = 0; n < nrLists; ++n) {

From ba3cb4a04949e043669299da5497bea27b944598 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <theophane.hufschmitt@tweag.io>
Date: Fri, 18 Nov 2022 13:02:06 +0100
Subject: [PATCH 137/294] Remove all the occurences of VLAs

There's generally no strict reason for using them, and they are somewhat
fishy, so let's avoid them.
---
 src/libexpr/eval.cc         | 13 ++++++++-----
 src/libstore/derivations.cc |  9 ++++-----
 src/libstore/gc.cc          |  9 ++-------
 3 files changed, 14 insertions(+), 17 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index dfe81cbf7..d853b104b 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -41,6 +41,7 @@
 #include <boost/coroutine2/coroutine.hpp>
 #include <boost/coroutine2/protected_fixedsize_stack.hpp>
 #include <boost/context/stack_context.hpp>
+#include <boost/container/small_vector.hpp>
 
 #endif
 
@@ -1691,7 +1692,8 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                 /* We have all the arguments, so call the primop with
                    the previous and new arguments. */
 
-                Value * vArgs[arity];
+                assert(arity < 64);
+                Value * vArgs[64];
                 auto n = argsDone;
                 for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->primOpApp.left)
                     vArgs[--n] = arg->primOpApp.right;
@@ -1748,11 +1750,12 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v)
     Value vFun;
     fun->eval(state, env, vFun);
 
-    Value * vArgs[args.size()];
+
+    boost::container::small_vector<Value *, 64> vArgs(args.size());
     for (size_t i = 0; i < args.size(); ++i)
         vArgs[i] = args[i]->maybeThunk(state, env);
 
-    state.callFunction(vFun, args.size(), vArgs, v, pos);
+    state.callFunction(vFun, args.size(), vArgs.data(), v, pos);
 }
 
 
@@ -1991,8 +1994,8 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
         return result;
     };
 
-    Value values[es->size()];
-    Value * vTmpP = values;
+    boost::container::small_vector<Value, 64> values(es->size());
+    Value * vTmpP = values.data();
 
     for (auto & [i_pos, i] : *es) {
         Value & vTmp = *vTmpP++;
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 1fecd1c97..6d9c8b9d6 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -151,11 +151,10 @@ StorePath writeDerivation(Store & store,
 /* Read string `s' from stream `str'. */
 static void expect(std::istream & str, std::string_view s)
 {
-    char s2[s.size()];
-    str.read(s2, s.size());
-    std::string_view s2View { s2, s.size() };
-    if (s2View != s)
-        throw FormatError("expected string '%s', got '%s'", s, s2View);
+    for (auto & c : s) {
+        if (str.get() != c)
+            throw FormatError("expected string '%1%'", s);
+    }
 }
 
 
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 8d05ae4bd..ddec43fdc 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -330,9 +330,7 @@ typedef std::unordered_map<Path, std::unordered_set<std::string>> UncheckedRoots
 
 static void readProcLink(const std::string & file, UncheckedRoots & roots)
 {
-    /* 64 is the starting buffer size gnu readlink uses... */
-    auto bufsiz = ssize_t{64};
-try_again:
+    constexpr auto bufsiz = PATH_MAX;
     char buf[bufsiz];
     auto res = readlink(file.c_str(), buf, bufsiz);
     if (res == -1) {
@@ -341,10 +339,7 @@ try_again:
         throw SysError("reading symlink");
     }
     if (res == bufsiz) {
-        if (SSIZE_MAX / 2 < bufsiz)
-            throw Error("stupidly long symlink");
-        bufsiz *= 2;
-        goto try_again;
+        throw Error("stupidly long symlink");
     }
     if (res > 0 && buf[0] == '/')
         roots[std::string(static_cast<char *>(buf), res)]

From 0daccb1121dfd5e98db3e41ba992b1b2c413dfc8 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Thu, 16 Nov 2023 11:10:25 +0100
Subject: [PATCH 138/294] libexpr: Check primop arity earlier

---
 src/libexpr/eval.cc              | 20 ++++++++++++++++++--
 src/libexpr/eval.hh              | 12 ++++++++++++
 src/libexpr/tests/value/print.cc |  3 ++-
 src/libexpr/value.hh             |  8 +-------
 4 files changed, 33 insertions(+), 10 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index d853b104b..1425eab97 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -723,6 +723,23 @@ void EvalState::addConstant(const std::string & name, Value * v, Constant info)
 }
 
 
+void PrimOp::check()
+{
+    if (arity > maxPrimOpArity) {
+        throw Error("primop arity must not exceed %1%", maxPrimOpArity);
+    }
+}
+
+
+void Value::mkPrimOp(PrimOp * p)
+{
+    p->check();
+    clearValue();
+    internalType = tPrimOp;
+    primOp = p;
+}
+
+
 Value * EvalState::addPrimOp(PrimOp && primOp)
 {
     /* Hack to make constants lazy: turn them into a application of
@@ -1692,8 +1709,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                 /* We have all the arguments, so call the primop with
                    the previous and new arguments. */
 
-                assert(arity < 64);
-                Value * vArgs[64];
+                Value * vArgs[maxPrimOpArity];
                 auto n = argsDone;
                 for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->primOpApp.left)
                     vArgs[--n] = arg->primOpApp.right;
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 048dff42b..5ee6359a8 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -18,6 +18,12 @@
 
 namespace nix {
 
+/**
+ * We put a limit on primop arity because it lets us use a fixed size array on
+ * the stack. 16 is already an impractical number of arguments. Use an attrset
+ * argument for such overly complicated functions.
+ */
+constexpr size_t maxPrimOpArity = 64;
 
 class Store;
 class EvalState;
@@ -71,6 +77,12 @@ struct PrimOp
      * Optional experimental for this to be gated on.
      */
     std::optional<ExperimentalFeature> experimentalFeature;
+
+    /**
+     * Validity check to be performed by functions that introduce primops,
+     * such as RegisterPrimOp() and Value::mkPrimOp().
+     */
+    void check();
 };
 
 /**
diff --git a/src/libexpr/tests/value/print.cc b/src/libexpr/tests/value/print.cc
index 5e96e12ec..a4f6fc014 100644
--- a/src/libexpr/tests/value/print.cc
+++ b/src/libexpr/tests/value/print.cc
@@ -114,7 +114,8 @@ TEST_F(ValuePrintingTests, vLambda)
 TEST_F(ValuePrintingTests, vPrimOp)
 {
     Value vPrimOp;
-    vPrimOp.mkPrimOp(nullptr);
+    PrimOp primOp{};
+    vPrimOp.mkPrimOp(&primOp);
 
     test(vPrimOp, "<PRIMOP>");
 }
diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 622e613ea..191cc30ba 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -354,13 +354,7 @@ public:
         // Value will be overridden anyways
     }
 
-    inline void mkPrimOp(PrimOp * p)
-    {
-        clearValue();
-        internalType = tPrimOp;
-        primOp = p;
-    }
-
+    void mkPrimOp(PrimOp * p);
 
     inline void mkPrimOpApp(Value * l, Value * r)
     {

From 12c91a823e80b5e0a14a0abb0f34a6633b14bbfe Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Thu, 16 Nov 2023 11:27:31 +0100
Subject: [PATCH 139/294] maxPrimOpArity: 64 -> 8

This makes stack usage significantly more compact, allowing larger
amounts of data to be processed on the same stack.

PrimOp functions with more than 8 positional (curried) arguments
should use an attrset instead.
---
 src/libexpr/eval.hh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 5ee6359a8..ce798ed96 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -20,10 +20,10 @@ namespace nix {
 
 /**
  * We put a limit on primop arity because it lets us use a fixed size array on
- * the stack. 16 is already an impractical number of arguments. Use an attrset
+ * the stack. 8 is already an impractical number of arguments. Use an attrset
  * argument for such overly complicated functions.
  */
-constexpr size_t maxPrimOpArity = 64;
+constexpr size_t maxPrimOpArity = 8;
 
 class Store;
 class EvalState;

From 9fa133dde5610dfb0605399ffea83081bda1c6fc Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Thu, 16 Nov 2023 11:33:10 +0100
Subject: [PATCH 140/294] readProcLink: Replace unnecessary value judgement by
 actual info

---
 src/libstore/gc.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index ddec43fdc..93fa60682 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -339,7 +339,7 @@ static void readProcLink(const std::string & file, UncheckedRoots & roots)
         throw SysError("reading symlink");
     }
     if (res == bufsiz) {
-        throw Error("stupidly long symlink");
+        throw Error("overly long symlink starting with '%1%'", std::string_view(buf, bufsiz));
     }
     if (res > 0 && buf[0] == '/')
         roots[std::string(static_cast<char *>(buf), res)]

From 206ece0f41142536a856c62c49bd202282f12db8 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Thu, 16 Nov 2023 12:18:37 +0100
Subject: [PATCH 141/294] builtins.{any,all}: Use constant errorCtx

Clang warned that the expanded code used to have a buffer overflow.
Very strange, but also very avoidable.
---
 src/libexpr/primops.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index e7587506a..d104b7180 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -3189,10 +3189,14 @@ static void anyOrAll(bool any, EvalState & state, const PosIdx pos, Value * * ar
     state.forceFunction(*args[0], pos, std::string("while evaluating the first argument passed to builtins.") + (any ? "any" : "all"));
     state.forceList(*args[1], pos, std::string("while evaluating the second argument passed to builtins.") + (any ? "any" : "all"));
 
+    std::string_view errorCtx = any
+        ? "while evaluating the return value of the function passed to builtins.any"
+        : "while evaluating the return value of the function passed to builtins.all";
+
     Value vTmp;
     for (auto elem : args[1]->listItems()) {
         state.callFunction(*args[0], *elem, vTmp, pos);
-        bool res = state.forceBool(vTmp, pos, std::string("while evaluating the return value of the function passed to builtins.") + (any ? "any" : "all"));
+        bool res = state.forceBool(vTmp, pos, errorCtx);
         if (res == any) {
             v.mkBool(any);
             return;

From 91114a6fa48e2eb9399c23938eb12fdbd4fcda42 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Thu, 16 Nov 2023 12:44:10 +0100
Subject: [PATCH 142/294] ExprCall::eval: Heap allocate at arity 5+

---
 src/libexpr/eval.cc | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 1425eab97..bfbda52ef 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -1766,8 +1766,13 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v)
     Value vFun;
     fun->eval(state, env, vFun);
 
-
-    boost::container::small_vector<Value *, 64> vArgs(args.size());
+    // Empirical arity of Nixpkgs lambdas by regex e.g. ([a-zA-Z]+:(\s|(/\*.*\/)|(#.*\n))*){5}
+    // 2: over 4000
+    // 3: about 300
+    // 4: about 60
+    // 5: under 10
+    // This excluded attrset lambdas (`{...}:`). Contributions of mixed lambdas appears insignificant at ~150 total.
+    boost::container::small_vector<Value *, 4> vArgs(args.size());
     for (size_t i = 0; i < args.size(); ++i)
         vArgs[i] = args[i]->maybeThunk(state, env);
 

From 898c47384f651f51b3e4b63c271da274db8fca2e Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Thu, 16 Nov 2023 12:48:37 +0100
Subject: [PATCH 143/294] primops: Err on the side of less stack usage

Try to stay away from stack overflows.

These small vectors use stack space. Most instances will not need
to allocate because in general most things are small, and large
things are worth heap allocating.

16 * 3 * word = 384 bytes is still quite a bit, but these functions
tend not to be part of deep recursions.
---
 src/libexpr/eval.cc    | 2 +-
 src/libexpr/primops.cc | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index bfbda52ef..2fcbf3311 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -2015,7 +2015,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
         return result;
     };
 
-    boost::container::small_vector<Value, 64> values(es->size());
+    boost::container::small_vector<Value, 16> values(es->size());
     Value * vTmpP = values.data();
 
     for (auto & [i_pos, i] : *es) {
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index d104b7180..7aa212281 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -2549,7 +2549,7 @@ static void prim_removeAttrs(EvalState & state, const PosIdx pos, Value * * args
     /* Get the attribute names to be removed.
        We keep them as Attrs instead of Symbols so std::set_difference
        can be used to remove them from attrs[0]. */
-    boost::container::small_vector<Attr, 64> names;
+    boost::container::small_vector<Attr, 16> names;
     names.reserve(args[1]->listSize());
     for (auto elem : args[1]->listItems()) {
         state.forceStringNoCtx(*elem, pos, "while evaluating the values of the second argument passed to builtins.removeAttrs");
@@ -3452,7 +3452,7 @@ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args,
     state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.concatMap");
     auto nrLists = args[1]->listSize();
 
-    boost::container::small_vector<Value, 64> lists(nrLists);
+    boost::container::small_vector<Value, 16> lists(nrLists);
     size_t len = 0;
 
     for (unsigned int n = 0; n < nrLists; ++n) {

From 1b9813e4e60836ddb1467efd50c572e7579ac945 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Thu, 16 Nov 2023 13:04:27 +0100
Subject: [PATCH 144/294] primops: Name stack reservation limits

---
 src/libexpr/eval.cc    |  3 ++-
 src/libexpr/primops.cc |  6 +++---
 src/libexpr/primops.hh | 16 ++++++++++++++++
 3 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 2fcbf3311..8b0ada517 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -1,6 +1,7 @@
 #include "eval.hh"
 #include "eval-settings.hh"
 #include "hash.hh"
+#include "primops.hh"
 #include "types.hh"
 #include "util.hh"
 #include "store-api.hh"
@@ -2015,7 +2016,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
         return result;
     };
 
-    boost::container::small_vector<Value, 16> values(es->size());
+    boost::container::small_vector<Value, conservativeStackReservation> values(es->size());
     Value * vTmpP = values.data();
 
     for (auto & [i_pos, i] : *es) {
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 7aa212281..adce95bed 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -2728,7 +2728,7 @@ static void prim_catAttrs(EvalState & state, const PosIdx pos, Value * * args, V
     auto attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.catAttrs"));
     state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.catAttrs");
 
-    boost::container::small_vector<Value *, 256> res(args[1]->listSize());
+    boost::container::small_vector<Value *, nonRecursiveStackReservation> res(args[1]->listSize());
     size_t found = 0;
 
     for (auto v2 : args[1]->listItems()) {
@@ -3063,7 +3063,7 @@ static void prim_filter(EvalState & state, const PosIdx pos, Value * * args, Val
 
     state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filter");
 
-    boost::container::small_vector<Value *, 256> vs(args[1]->listSize());
+    boost::container::small_vector<Value *, nonRecursiveStackReservation> vs(args[1]->listSize());
     size_t k = 0;
 
     bool same = true;
@@ -3452,7 +3452,7 @@ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args,
     state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.concatMap");
     auto nrLists = args[1]->listSize();
 
-    boost::container::small_vector<Value, 16> lists(nrLists);
+    boost::container::small_vector<Value, conservativeStackReservation> lists(nrLists);
     size_t len = 0;
 
     for (unsigned int n = 0; n < nrLists; ++n) {
diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh
index 930e7f32a..1d5d5710d 100644
--- a/src/libexpr/primops.hh
+++ b/src/libexpr/primops.hh
@@ -8,6 +8,22 @@
 
 namespace nix {
 
+/**
+ * For functions where we do not expect deep recursion, we can use a sizable
+ * part of the stack a free allocation space.
+ *
+ * Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes.
+ */
+constexpr size_t nonRecursiveStackReservation = 256;
+
+/**
+ * Functions that maybe applied to self-similar inputs, such as concatMap on a
+ * tree, should reserve a smaller part of the stack for allocation.
+ *
+ * Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes.
+ */
+constexpr size_t conservativeStackReservation = 16;
+
 struct RegisterPrimOp
 {
     typedef std::vector<PrimOp> PrimOps;

From a96be29db536177fdc284b51a3b2af44a70496e0 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Thu, 16 Nov 2023 13:13:01 +0100
Subject: [PATCH 145/294] removeAttrs: increase stack reservation to 64

---
 src/libexpr/primops.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index adce95bed..e274c3c0c 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -2549,7 +2549,8 @@ static void prim_removeAttrs(EvalState & state, const PosIdx pos, Value * * args
     /* Get the attribute names to be removed.
        We keep them as Attrs instead of Symbols so std::set_difference
        can be used to remove them from attrs[0]. */
-    boost::container::small_vector<Attr, 16> names;
+    // 64: large enough to fit the attributes of a derivation
+    boost::container::small_vector<Attr, 64> names;
     names.reserve(args[1]->listSize());
     for (auto elem : args[1]->listItems()) {
         state.forceStringNoCtx(*elem, pos, "while evaluating the values of the second argument passed to builtins.removeAttrs");

From 4e27f1947a444a36d6a85f41cbf1afdc70ac6c4c Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Thu, 16 Nov 2023 13:23:17 +0100
Subject: [PATCH 146/294] libexpr: Reduce nonRecursiveStackReservation

128 is still beyond the point where the allocation overhead is
insignificant, but we don't anticipate to overflow for these
use cases, so it's fine.
---
 src/libexpr/primops.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh
index 1d5d5710d..45486608f 100644
--- a/src/libexpr/primops.hh
+++ b/src/libexpr/primops.hh
@@ -14,7 +14,7 @@ namespace nix {
  *
  * Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes.
  */
-constexpr size_t nonRecursiveStackReservation = 256;
+constexpr size_t nonRecursiveStackReservation = 128;
 
 /**
  * Functions that maybe applied to self-similar inputs, such as concatMap on a

From 6c8f4ef3502aa214557541ec00538e41aeced6e3 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 14 Nov 2023 10:52:57 -0500
Subject: [PATCH 147/294] Allow installing unit tests

Closes #9343

See that issue for motivation.

Installing these is disabled by default, but we enable it (and the
additional output we want isntall these too so as not to clutter the
existing ones) to use in cross builds and dev shells.
---
 Makefile.config.in          |  3 +++
 configure.ac                | 12 ++++++++++++
 flake.nix                   | 10 ++++++++--
 src/libexpr/tests/local.mk  |  6 +++++-
 src/libstore/tests/local.mk | 12 ++++++++++--
 src/libutil/tests/local.mk  | 12 ++++++++++--
 6 files changed, 48 insertions(+), 7 deletions(-)

diff --git a/Makefile.config.in b/Makefile.config.in
index 19992fa20..1482db81f 100644
--- a/Makefile.config.in
+++ b/Makefile.config.in
@@ -28,6 +28,8 @@ SODIUM_LIBS = @SODIUM_LIBS@
 SQLITE3_LIBS = @SQLITE3_LIBS@
 bash = @bash@
 bindir = @bindir@
+checkbindir = @checkbindir@
+checklibdir = @checklibdir@
 datadir = @datadir@
 datarootdir = @datarootdir@
 doc_generate = @doc_generate@
@@ -48,4 +50,5 @@ sysconfdir = @sysconfdir@
 system = @system@
 ENABLE_BUILD = @ENABLE_BUILD@
 ENABLE_TESTS = @ENABLE_TESTS@
+INSTALL_UNIT_TESTS = @INSTALL_UNIT_TESTS@
 internal_api_docs = @internal_api_docs@
diff --git a/configure.ac b/configure.ac
index 75ce7d01d..281ba2c32 100644
--- a/configure.ac
+++ b/configure.ac
@@ -167,6 +167,18 @@ AC_ARG_ENABLE(tests, AS_HELP_STRING([--disable-tests],[Do not build the tests]),
   ENABLE_TESTS=$enableval, ENABLE_TESTS=yes)
 AC_SUBST(ENABLE_TESTS)
 
+AC_ARG_ENABLE(install-unit-tests, AS_HELP_STRING([--enable-install-unit-tests],[Install the unit tests for running later (default no)]),
+  INSTALL_UNIT_TESTS=$enableval, INSTALL_UNIT_TESTS=no)
+AC_SUBST(INSTALL_UNIT_TESTS)
+
+AC_ARG_WITH(check-bin-dir, AS_HELP_STRING([--with-check-bin-dir=PATH],[path to install unit tests for running later (defaults to $libexecdir/nix)]),
+  checkbindir=$withval, checkbindir=$libexecdir/nix)
+AC_SUBST(checkbindir)
+
+AC_ARG_WITH(check-lib-dir, AS_HELP_STRING([--with-check-lib-dir=PATH],[path to install unit tests for running later (defaults to $libdir)]),
+  checklibdir=$withval, checklibdir=$libdir)
+AC_SUBST(checklibdir)
+
 # Building without API docs is the default as Nix' C++ interfaces are internal and unstable.
 AC_ARG_ENABLE(internal_api_docs, AS_HELP_STRING([--enable-internal-api-docs],[Build API docs for Nix's internal unstable C++ interfaces]),
   internal_api_docs=$enableval, internal_api_docs=no)
diff --git a/flake.nix b/flake.nix
index 51d818423..05ab7b06d 100644
--- a/flake.nix
+++ b/flake.nix
@@ -164,6 +164,10 @@
 
         testConfigureFlags = [
           "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include"
+        ] ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
+          "--enable-install-unit-tests"
+          "--with-check-bin-dir=${builtins.placeholder "check"}/bin"
+          "--with-check-lib-dir=${builtins.placeholder "check"}/lib"
         ];
 
         internalApiDocsConfigureFlags = [
@@ -404,7 +408,8 @@
             src = nixSrc;
             VERSION_SUFFIX = versionSuffix;
 
-            outputs = [ "out" "dev" "doc" ];
+            outputs = [ "out" "dev" "doc" ]
+              ++ lib.optional (currentStdenv.hostPlatform != currentStdenv.buildPlatform) "check";
 
             nativeBuildInputs = nativeBuildDeps;
             buildInputs = buildDeps
@@ -710,7 +715,8 @@
           stdenv.mkDerivation {
             name = "nix";
 
-            outputs = [ "out" "dev" "doc" ];
+            outputs = [ "out" "dev" "doc" ]
+              ++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) "check";
 
             nativeBuildInputs = nativeBuildDeps
               ++ lib.optional stdenv.cc.isClang pkgs.buildPackages.bear
diff --git a/src/libexpr/tests/local.mk b/src/libexpr/tests/local.mk
index 331a5ead6..6d2a04aaf 100644
--- a/src/libexpr/tests/local.mk
+++ b/src/libexpr/tests/local.mk
@@ -6,7 +6,11 @@ libexpr-tests_NAME := libnixexpr-tests
 
 libexpr-tests_DIR := $(d)
 
-libexpr-tests_INSTALL_DIR :=
+ifeq ($(INSTALL_UNIT_TESTS), yes)
+  libexpr-tests_INSTALL_DIR := $(checkbindir)
+else
+  libexpr-tests_INSTALL_DIR :=
+endif
 
 libexpr-tests_SOURCES := \
     $(wildcard $(d)/*.cc) \
diff --git a/src/libstore/tests/local.mk b/src/libstore/tests/local.mk
index 03becc7d1..e9b8b4f99 100644
--- a/src/libstore/tests/local.mk
+++ b/src/libstore/tests/local.mk
@@ -6,7 +6,11 @@ libstore-tests-exe_NAME = libnixstore-tests
 
 libstore-tests-exe_DIR := $(d)
 
-libstore-tests-exe_INSTALL_DIR :=
+ifeq ($(INSTALL_UNIT_TESTS), yes)
+  libstore-tests-exe_INSTALL_DIR := $(checkbindir)
+else
+  libstore-tests-exe_INSTALL_DIR :=
+endif
 
 libstore-tests-exe_LIBS = libstore-tests
 
@@ -18,7 +22,11 @@ libstore-tests_NAME = libnixstore-tests
 
 libstore-tests_DIR := $(d)
 
-libstore-tests_INSTALL_DIR :=
+ifeq ($(INSTALL_UNIT_TESTS), yes)
+  libstore-tests_INSTALL_DIR := $(checklibdir)
+else
+  libstore-tests_INSTALL_DIR :=
+endif
 
 libstore-tests_SOURCES := $(wildcard $(d)/*.cc)
 
diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk
index c8b8557cb..e6fc4e364 100644
--- a/src/libutil/tests/local.mk
+++ b/src/libutil/tests/local.mk
@@ -6,7 +6,11 @@ libutil-tests-exe_NAME = libnixutil-tests
 
 libutil-tests-exe_DIR := $(d)
 
-libutil-tests-exe_INSTALL_DIR :=
+ifeq ($(INSTALL_UNIT_TESTS), yes)
+  libutil-tests-exe_INSTALL_DIR := $(checkbindir)
+else
+  libutil-tests-exe_INSTALL_DIR :=
+endif
 
 libutil-tests-exe_LIBS = libutil-tests
 
@@ -18,7 +22,11 @@ libutil-tests_NAME = libnixutil-tests
 
 libutil-tests_DIR := $(d)
 
-libutil-tests_INSTALL_DIR :=
+ifeq ($(INSTALL_UNIT_TESTS), yes)
+  libutil-tests_INSTALL_DIR := $(checklibdir)
+else
+  libutil-tests_INSTALL_DIR :=
+endif
 
 libutil-tests_SOURCES := $(wildcard $(d)/*.cc)
 

From 31ebc6028b3682969d86a7b39ae87131c41cc604 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 16 Nov 2023 16:45:14 +0100
Subject: [PATCH 148/294] Fix symlink handling

This restores the symlink handling behaviour prior to
94812cca98fbb157e5f64a15a85a2b852d289feb.

Fixes #9298.
---
 src/libexpr/eval.hh                            |  2 +-
 src/libexpr/parser.y                           | 18 +++++++++++++-----
 .../lang/eval-okay-symlink-resolution.exp      |  1 +
 .../lang/eval-okay-symlink-resolution.nix      |  1 +
 .../symlink-resolution/foo/lib/default.nix     |  1 +
 .../lang/symlink-resolution/foo/overlays       |  1 +
 .../symlink-resolution/overlays/overlay.nix    |  1 +
 7 files changed, 19 insertions(+), 6 deletions(-)
 create mode 100644 tests/functional/lang/eval-okay-symlink-resolution.exp
 create mode 100644 tests/functional/lang/eval-okay-symlink-resolution.nix
 create mode 100644 tests/functional/lang/symlink-resolution/foo/lib/default.nix
 create mode 120000 tests/functional/lang/symlink-resolution/foo/overlays
 create mode 100644 tests/functional/lang/symlink-resolution/overlays/overlay.nix

diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 048dff42b..9257a0e48 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -827,7 +827,7 @@ std::string showType(const Value & v);
 /**
  * If `path` refers to a directory, then append "/default.nix".
  */
-SourcePath resolveExprPath(const SourcePath & path);
+SourcePath resolveExprPath(SourcePath path);
 
 struct InvalidPathError : EvalError
 {
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index b86cef217..f6cf1f689 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -686,17 +686,25 @@ Expr * EvalState::parse(
 }
 
 
-SourcePath resolveExprPath(const SourcePath & path)
+SourcePath resolveExprPath(SourcePath path)
 {
+    unsigned int followCount = 0, maxFollow = 1024;
+
     /* If `path' is a symlink, follow it.  This is so that relative
        path references work. */
-    auto path2 = path.resolveSymlinks();
+    while (true) {
+        // Basic cycle/depth limit to avoid infinite loops.
+        if (++followCount >= maxFollow)
+            throw Error("too many symbolic links encountered while traversing the path '%s'", path);
+        if (path.lstat().type != InputAccessor::tSymlink) break;
+        path = {path.accessor, CanonPath(path.readLink(), path.path.parent().value_or(CanonPath::root))};
+    }
 
     /* If `path' refers to a directory, append `/default.nix'. */
-    if (path2.lstat().type == InputAccessor::tDirectory)
-        return path2 + "default.nix";
+    if (path.lstat().type == InputAccessor::tDirectory)
+        return path + "default.nix";
 
-    return path2;
+    return path;
 }
 
 
diff --git a/tests/functional/lang/eval-okay-symlink-resolution.exp b/tests/functional/lang/eval-okay-symlink-resolution.exp
new file mode 100644
index 000000000..8b8441b91
--- /dev/null
+++ b/tests/functional/lang/eval-okay-symlink-resolution.exp
@@ -0,0 +1 @@
+"test"
diff --git a/tests/functional/lang/eval-okay-symlink-resolution.nix b/tests/functional/lang/eval-okay-symlink-resolution.nix
new file mode 100644
index 000000000..ffb1818bd
--- /dev/null
+++ b/tests/functional/lang/eval-okay-symlink-resolution.nix
@@ -0,0 +1 @@
+import symlink-resolution/foo/overlays/overlay.nix
diff --git a/tests/functional/lang/symlink-resolution/foo/lib/default.nix b/tests/functional/lang/symlink-resolution/foo/lib/default.nix
new file mode 100644
index 000000000..8b8441b91
--- /dev/null
+++ b/tests/functional/lang/symlink-resolution/foo/lib/default.nix
@@ -0,0 +1 @@
+"test"
diff --git a/tests/functional/lang/symlink-resolution/foo/overlays b/tests/functional/lang/symlink-resolution/foo/overlays
new file mode 120000
index 000000000..0d44a21c5
--- /dev/null
+++ b/tests/functional/lang/symlink-resolution/foo/overlays
@@ -0,0 +1 @@
+../overlays
\ No newline at end of file
diff --git a/tests/functional/lang/symlink-resolution/overlays/overlay.nix b/tests/functional/lang/symlink-resolution/overlays/overlay.nix
new file mode 100644
index 000000000..b0368308e
--- /dev/null
+++ b/tests/functional/lang/symlink-resolution/overlays/overlay.nix
@@ -0,0 +1 @@
+import ../lib

From 96d67620d551c7143b6682cfff74a2ee2edbe863 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 16 Nov 2023 17:12:06 +0100
Subject: [PATCH 149/294] Fix a broken generated header file dependency

https://hydra.nixos.org/build/240882042
---
 src/libexpr/local.mk | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk
index d243b9cec..ed7bf9490 100644
--- a/src/libexpr/local.mk
+++ b/src/libexpr/local.mk
@@ -43,7 +43,9 @@ $(foreach i, $(wildcard src/libexpr/value/*.hh), \
 $(foreach i, $(wildcard src/libexpr/flake/*.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644)))
 
-$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh
+$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh
+
+$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh
 
 $(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh
 

From c81937576928ca494af0be6e7c61f2070be5d353 Mon Sep 17 00:00:00 2001
From: Dominic Shelton <frogamic@protonmail.com>
Date: Fri, 17 Nov 2023 17:50:17 +1100
Subject: [PATCH 150/294] doc: Add example of inherit in a let expression

---
 doc/manual/src/language/constructs.md | 54 ++++++++++++++++++++-------
 1 file changed, 40 insertions(+), 14 deletions(-)

diff --git a/doc/manual/src/language/constructs.md b/doc/manual/src/language/constructs.md
index a3590f55d..bd7bd92a5 100644
--- a/doc/manual/src/language/constructs.md
+++ b/doc/manual/src/language/constructs.md
@@ -132,6 +132,32 @@ a = src-set.a; b = src-set.b; c = src-set.c;
 when used while defining local variables in a let-expression or while
 defining a set.
 
+in a let expression, inherit can be used to selectively bring specific attributes of a set into scope. For example
+
+
+```nix
+let
+  x = { a = 1; b = 2; };
+  inherit (builtins) attrNames;
+in
+{
+  names = attrNames x;
+}
+```
+
+is equivalent to
+
+```nix
+let
+  x = { a = 1; b = 2; };
+in
+{
+  names = builtins.attrNames x;
+}
+```
+
+both resolve to `{ names = [ "a" "b" ]; }`.
+
 ## Functions
 
 Functions have the following form:
@@ -146,65 +172,65 @@ three kinds of patterns:
 
   - If a pattern is a single identifier, then the function matches any
     argument. Example:
-    
+
     ```nix
     let negate = x: !x;
         concat = x: y: x + y;
     in if negate true then concat "foo" "bar" else ""
     ```
-    
+
     Note that `concat` is a function that takes one argument and returns
     a function that takes another argument. This allows partial
     parameterisation (i.e., only filling some of the arguments of a
     function); e.g.,
-    
+
     ```nix
     map (concat "foo") [ "bar" "bla" "abc" ]
     ```
-    
+
     evaluates to `[ "foobar" "foobla" "fooabc" ]`.
 
   - A *set pattern* of the form `{ name1, name2, …, nameN }` matches a
     set containing the listed attributes, and binds the values of those
     attributes to variables in the function body. For example, the
     function
-    
+
     ```nix
     { x, y, z }: z + y + x
     ```
-    
+
     can only be called with a set containing exactly the attributes `x`,
     `y` and `z`. No other attributes are allowed. If you want to allow
     additional arguments, you can use an ellipsis (`...`):
-    
+
     ```nix
     { x, y, z, ... }: z + y + x
     ```
-    
+
     This works on any set that contains at least the three named
     attributes.
-    
+
     It is possible to provide *default values* for attributes, in
     which case they are allowed to be missing. A default value is
     specified by writing `name ?  e`, where *e* is an arbitrary
     expression. For example,
-    
+
     ```nix
     { x, y ? "foo", z ? "bar" }: z + y + x
     ```
-    
+
     specifies a function that only requires an attribute named `x`, but
     optionally accepts `y` and `z`.
 
   - An `@`-pattern provides a means of referring to the whole value
     being matched:
-    
+
     ```nix
     args@{ x, y, z, ... }: z + y + x + args.a
     ```
-    
+
     but can also be written as:
-    
+
     ```nix
     { x, y, z, ... } @ args: z + y + x + args.a
     ```

From 2eb59c34b531f03a85f67b9246ccaf0ff5fcad23 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 12 Nov 2023 03:02:02 +0100
Subject: [PATCH 151/294] Value: extract Value::StringWithContext

---
 src/libexpr/value.hh | 54 +++++++++++++++++++++++---------------------
 1 file changed, 28 insertions(+), 26 deletions(-)

diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 191cc30ba..0f8cef418 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -158,37 +158,39 @@ public:
     inline bool isPrimOp() const { return internalType == tPrimOp; };
     inline bool isPrimOpApp() const { return internalType == tPrimOpApp; };
 
+    /**
+     * Strings in the evaluator carry a so-called `context` which
+     * is a list of strings representing store paths.  This is to
+     * allow users to write things like
+     *
+     *   "--with-freetype2-library=" + freetype + "/lib"
+     *
+     * where `freetype` is a derivation (or a source to be copied
+     * to the store).  If we just concatenated the strings without
+     * keeping track of the referenced store paths, then if the
+     * string is used as a derivation attribute, the derivation
+     * will not have the correct dependencies in its inputDrvs and
+     * inputSrcs.
+
+     * The semantics of the context is as follows: when a string
+     * with context C is used as a derivation attribute, then the
+     * derivations in C will be added to the inputDrvs of the
+     * derivation, and the other store paths in C will be added to
+     * the inputSrcs of the derivations.
+
+     * For canonicity, the store paths should be in sorted order.
+     */
+    struct StringWithContext {
+        const char * c_str;
+        const char * * context; // must be in sorted order
+    };
+
     union
     {
         NixInt integer;
         bool boolean;
 
-        /**
-         * Strings in the evaluator carry a so-called `context` which
-         * is a list of strings representing store paths.  This is to
-         * allow users to write things like
-
-         *   "--with-freetype2-library=" + freetype + "/lib"
-
-         * where `freetype` is a derivation (or a source to be copied
-         * to the store).  If we just concatenated the strings without
-         * keeping track of the referenced store paths, then if the
-         * string is used as a derivation attribute, the derivation
-         * will not have the correct dependencies in its inputDrvs and
-         * inputSrcs.
-
-         * The semantics of the context is as follows: when a string
-         * with context C is used as a derivation attribute, then the
-         * derivations in C will be added to the inputDrvs of the
-         * derivation, and the other store paths in C will be added to
-         * the inputSrcs of the derivations.
-
-         * For canonicity, the store paths should be in sorted order.
-         */
-        struct {
-            const char * c_str;
-            const char * * context; // must be in sorted order
-        } string;
+        StringWithContext string;
 
         struct {
             InputAccessor * accessor;

From d8ff5cfe8eba34c8b4b5cc53f3b40cd3dfd84224 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 12 Nov 2023 03:03:37 +0100
Subject: [PATCH 152/294] Value: extract Value::Path

---
 src/libexpr/value.hh | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 0f8cef418..34f994997 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -185,6 +185,11 @@ public:
         const char * * context; // must be in sorted order
     };
 
+    struct Path {
+        InputAccessor * accessor;
+        const char * path;
+    };
+
     union
     {
         NixInt integer;
@@ -192,10 +197,7 @@ public:
 
         StringWithContext string;
 
-        struct {
-            InputAccessor * accessor;
-            const char * path;
-        } _path;
+        Path _path;
 
         Bindings * attrs;
         struct {

From b55203e874f8e4b2fc5289129efba791937c23d0 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 12 Nov 2023 03:06:04 +0100
Subject: [PATCH 153/294] Value: extract Value::ClosureThunk

---
 src/libexpr/value.hh | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 34f994997..4c51c52e4 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -190,6 +190,11 @@ public:
         const char * path;
     };
 
+    struct ClosureThunk {
+        Env * env;
+        Expr * expr;
+    };
+
     union
     {
         NixInt integer;
@@ -205,10 +210,7 @@ public:
             Value * * elems;
         } bigList;
         Value * smallList[2];
-        struct {
-            Env * env;
-            Expr * expr;
-        } thunk;
+        ClosureThunk thunk;
         struct {
             Value * left, * right;
         } app;

From 6af1d9f7b94da454252b62f0cfff4ce800c5a46b Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 12 Nov 2023 03:06:55 +0100
Subject: [PATCH 154/294] Value: extract Value::FunctionApplicationThunk

---
 src/libexpr/value.hh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 4c51c52e4..cfb3f5276 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -195,6 +195,10 @@ public:
         Expr * expr;
     };
 
+    struct FunctionApplicationThunk {
+        Value * left, * right;
+    };
+
     union
     {
         NixInt integer;
@@ -211,17 +215,13 @@ public:
         } bigList;
         Value * smallList[2];
         ClosureThunk thunk;
-        struct {
-            Value * left, * right;
-        } app;
+        FunctionApplicationThunk app;
         struct {
             Env * env;
             ExprLambda * fun;
         } lambda;
         PrimOp * primOp;
-        struct {
-            Value * left, * right;
-        } primOpApp;
+        FunctionApplicationThunk primOpApp;
         ExternalValueBase * external;
         NixFloat fpoint;
     };

From 7055c6528532fdd3b0ce9b8f5282b002fc011470 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 12 Nov 2023 03:07:32 +0100
Subject: [PATCH 155/294] Value: extract Value::Lambda

---
 src/libexpr/value.hh | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index cfb3f5276..93ccdbc2e 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -199,6 +199,11 @@ public:
         Value * left, * right;
     };
 
+    struct Lambda {
+        Env * env;
+        ExprLambda * fun;
+    };
+
     union
     {
         NixInt integer;
@@ -216,10 +221,7 @@ public:
         Value * smallList[2];
         ClosureThunk thunk;
         FunctionApplicationThunk app;
-        struct {
-            Env * env;
-            ExprLambda * fun;
-        } lambda;
+        Lambda lambda;
         PrimOp * primOp;
         FunctionApplicationThunk primOpApp;
         ExternalValueBase * external;

From 260c6147625e95e4772ccdee80d6463d242c7b64 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 12 Nov 2023 03:31:45 +0100
Subject: [PATCH 156/294] Value: use std::span, change use of const

**`Value` and `const`**

These two deserve some explanation. We'll get to lists later.

Values can normally be thought of as immutable, except they are
are also the vehicle for call by need, which must be implemented
using mutation.

This circumstance makes a `const Value` a rather useless thing:

 - If it's a thunk, you can't evaluate it, except by copying, but
   that would not be call by need.

 - If it's not a thunk, you know the type, so the method that
   acquired it for you should have returned something more specific,
   such as a `const Bindings &` (which actually does make sense
   because that's an immutable span of pointers to mutable `Value`s.

 - If you don't care about the type yet, you might establish the
   convention that `const Value` means `deepSeq`-ed data, but
   this is hardly useful and not actually as safe as you would
   supposedly want to trust it to be - just convention.

**Lists**

`std::span` is a tuple of pointer and size - just what we need.

We don't return them as `const Value`, because considering the
first bullet point we discussed before, we'd have to force all
the list values, which isn't what we want.

So what we end up with is a nice representation of a list in
weak head normal form: the spine is immutable, but the
items may need some evaluation later.
---
 src/libexpr/value.hh | 37 ++++++++-----------------------------
 1 file changed, 8 insertions(+), 29 deletions(-)

diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 93ccdbc2e..bcff8ae55 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -3,6 +3,7 @@
 
 #include <cassert>
 #include <climits>
+#include <span>
 
 #include "symbol-table.hh"
 #include "value/context.hh"
@@ -395,7 +396,13 @@ public:
         return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems;
     }
 
-    const Value * const * listElems() const
+    std::span<Value * const> listItems() const
+    {
+        assert(isList());
+        return std::span<Value * const>(listElems(), listSize());
+    }
+
+    Value * const * listElems() const
     {
         return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems;
     }
@@ -414,34 +421,6 @@ public:
      */
     bool isTrivial() const;
 
-    auto listItems()
-    {
-        struct ListIterable
-        {
-            typedef Value * const * iterator;
-            iterator _begin, _end;
-            iterator begin() const { return _begin; }
-            iterator end() const { return _end; }
-        };
-        assert(isList());
-        auto begin = listElems();
-        return ListIterable { begin, begin + listSize() };
-    }
-
-    auto listItems() const
-    {
-        struct ConstListIterable
-        {
-            typedef const Value * const * iterator;
-            iterator _begin, _end;
-            iterator begin() const { return _begin; }
-            iterator end() const { return _end; }
-        };
-        assert(isList());
-        auto begin = listElems();
-        return ConstListIterable { begin, begin + listSize() };
-    }
-
     SourcePath path() const
     {
         assert(internalType == tPath);

From 121665f3773bc46ca6df0dda6f66b1a86e7d9e72 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 12 Nov 2023 17:51:09 +0100
Subject: [PATCH 157/294] nix-env: Use state.mkList, required for correct stats

---
 src/nix-env/nix-env.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 213a20d93..ab1d8f713 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -172,7 +172,7 @@ static void loadSourceExpr(EvalState & state, const SourcePath & path, Value & v
        directory). */
     else if (st.type == InputAccessor::tDirectory) {
         auto attrs = state.buildBindings(maxAttrs);
-        attrs.alloc("_combineChannels").mkList(0);
+        state.mkList(attrs.alloc("_combineChannels"), 0);
         StringSet seen;
         getAllExprs(state, path, seen, attrs);
         v.mkAttrs(attrs);

From 7b0e8c5c2c09146722349d3fd2dd69211d8b8945 Mon Sep 17 00:00:00 2001
From: Robert Hensing <roberth@users.noreply.github.com>
Date: Fri, 17 Nov 2023 10:56:23 +0100
Subject: [PATCH 158/294] Apply suggestions from code review

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
---
 doc/manual/src/language/constructs.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/manual/src/language/constructs.md b/doc/manual/src/language/constructs.md
index bd7bd92a5..a82ec5960 100644
--- a/doc/manual/src/language/constructs.md
+++ b/doc/manual/src/language/constructs.md
@@ -132,7 +132,7 @@ a = src-set.a; b = src-set.b; c = src-set.c;
 when used while defining local variables in a let-expression or while
 defining a set.
 
-in a let expression, inherit can be used to selectively bring specific attributes of a set into scope. For example
+In a `let` expression, `inherit` can be used to selectively bring specific attributes of a set into scope. For example
 
 
 ```nix
@@ -156,7 +156,7 @@ in
 }
 ```
 
-both resolve to `{ names = [ "a" "b" ]; }`.
+both evaluate to `{ names = [ "a" "b" ]; }`.
 
 ## Functions
 

From f7d59d0dda5e4a793e701bc8fb9136b3ef22948c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 17 Nov 2023 14:21:17 +0100
Subject: [PATCH 159/294] Release notes

---
 doc/manual/src/SUMMARY.md.in            |  1 +
 doc/manual/src/release-notes/rl-2.19.md | 77 +++++++++++++++++++++++++
 doc/manual/src/release-notes/rl-next.md | 75 ------------------------
 3 files changed, 78 insertions(+), 75 deletions(-)
 create mode 100644 doc/manual/src/release-notes/rl-2.19.md

diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in
index 794f78a07..8dc464abd 100644
--- a/doc/manual/src/SUMMARY.md.in
+++ b/doc/manual/src/SUMMARY.md.in
@@ -115,6 +115,7 @@
   - [C++ style guide](contributing/cxx.md)
 - [Release Notes](release-notes/release-notes.md)
   - [Release X.Y (202?-??-??)](release-notes/rl-next.md)
+  - [Release 2.19 (2023-11-17)](release-notes/rl-2.19.md)
   - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md)
   - [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md)
   - [Release 2.16 (2023-05-31)](release-notes/rl-2.16.md)
diff --git a/doc/manual/src/release-notes/rl-2.19.md b/doc/manual/src/release-notes/rl-2.19.md
new file mode 100644
index 000000000..4eecaf929
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-2.19.md
@@ -0,0 +1,77 @@
+# Release 2.19 (2023-11-17)
+
+- The experimental `nix` command can now act as a [shebang interpreter](@docroot@/command-ref/new-cli/nix.md#shebang-interpreter)
+  by appending the contents of any `#! nix` lines and the script's location into a single call.
+
+- [URL flake references](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) now support [percent-encoded](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1) characters.
+
+- [Path-like flake references](@docroot@/command-ref/new-cli/nix3-flake.md#path-like-syntax) now accept arbitrary unicode characters (except `#` and `?`).
+
+- The experimental feature `repl-flake` is no longer needed, as its functionality is now part of the `flakes` experimental feature. To get the previous behavior, use the `--file/--expr` flags accordingly.
+
+- There is a new flake installable syntax `flakeref#.attrPath` where the "." prefix specifies that `attrPath` is interpreted from the root of the flake outputs, with no searching of default attribute prefixes like `packages.<SYSTEM>` or `legacyPackages.<SYSTEM>`.
+
+- Nix adds `apple-virt` to the default system features on macOS systems that support virtualization. This is similar to what's done for the `kvm` system feature on Linux hosts.
+
+- Add a new built-in function [`builtins.convertHash`](@docroot@/language/builtins.md#builtins-convertHash).
+
+- `nix-shell` shebang lines now support single-quoted arguments.
+
+- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree).
+  As described in the documentation for that feature, this is because we anticipate polishing it and then stabilizing it before the rest of flakes.
+
+- The interface for creating and updating lock files has been overhauled:
+
+  - [`nix flake lock`](@docroot@/command-ref/new-cli/nix3-flake-lock.md) only creates lock files and adds missing inputs now.
+    It will *never* update existing inputs.
+
+  - [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) does the same, but *will* update inputs.
+    - Passing no arguments will update all inputs of the current flake, just like it already did.
+    - Passing input names as arguments will ensure only those are updated. This replaces the functionality of `nix flake lock --update-input`
+    - To operate on a flake outside the current directory, you must now pass `--flake path/to/flake`.
+
+  - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables.
+    They are superceded by `nix flake update`.
+
+- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches).
+
+- [`nix path-info --json`](@docroot@/command-ref/new-cli/nix3-path-info.md)
+  (experimental) now returns a JSON map rather than JSON list.
+  The `path` field of each object has instead become the key in the outer map, since it is unique.
+  The `valid` field also goes away because we just use `null` instead.
+
+  - Old way:
+
+    ```json5
+    [
+      {
+        "path": "/nix/store/8fv91097mbh5049i9rglc73dx6kjg3qk-bash-5.2-p15",
+        "valid": true,
+        // ...
+      },
+      {
+        "path": "/nix/store/wffw7l0alvs3iw94cbgi1gmmbmw99sqb-home-manager-path",
+        "valid": false
+      }
+    ]
+    ```
+
+  - New way
+
+    ```json5
+    {
+      "/nix/store/8fv91097mbh5049i9rglc73dx6kjg3qk-bash-5.2-p15": {
+        // ...
+      },
+      "/nix/store/wffw7l0alvs3iw94cbgi1gmmbmw99sqb-home-manager-path": null,
+    }
+    ```
+
+  This makes it match `nix derivation show`, which also maps store paths to information.
+
+- When Nix is installed using the [binary installer](@docroot@/installation/installing-binary.md), in supported shells (Bash, Zsh, Fish)
+  [`XDG_DATA_DIRS`](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html#variables) is now populated with the path to the `/share` subdirectory of the current profile.
+  This means that command completion scripts, `.desktop` files, and similar artifacts installed via [`nix-env`](@docroot@/command-ref/nix-env.md) or [`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md)
+  (experimental) can be found by any program that follows the [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html).
+
+- A new command `nix store add` has been added. It replaces `nix store add-file` and `nix store add-path` which are now deprecated.
diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
index 422f1fce8..78ae99f4b 100644
--- a/doc/manual/src/release-notes/rl-next.md
+++ b/doc/manual/src/release-notes/rl-next.md
@@ -1,77 +1,2 @@
 # Release X.Y (202?-??-??)
 
-- The experimental nix command can now act as a [shebang interpreter](@docroot@/command-ref/new-cli/nix.md#shebang-interpreter)
-  by appending the contents of any `#! nix` lines and the script's location to a single call.
-
-- [URL flake references](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) now support [percent-encoded](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1) characters.
-
-- [Path-like flake references](@docroot@/command-ref/new-cli/nix3-flake.md#path-like-syntax) now accept arbitrary unicode characters (except `#` and `?`).
-
-- The experimental feature `repl-flake` is no longer needed, as its functionality is now part of the `flakes` experimental feature. To get the previous behavior, use the `--file/--expr` flags accordingly.
-
-- Introduce new flake installable syntax `flakeref#.attrPath` where the "." prefix denotes no searching of default attribute prefixes like `packages.<SYSTEM>` or `legacyPackages.<SYSTEM>`.
-
-- Nix adds `apple-virt` to the default system features on macOS systems that support virtualization. This is similar to what's done for the `kvm` system feature on Linux hosts.
-
-- Introduce a new built-in function [`builtins.convertHash`](@docroot@/language/builtins.md#builtins-convertHash).
-
-- `nix-shell` shebang lines now support single-quoted arguments.
-
-- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree).
-  As described in the document for that feature, this is because we anticipate polishing it and then stabilizing it before the rest of Flakes.
-
-- The interface for creating and updating lock files has been overhauled:
-
-  - [`nix flake lock`](@docroot@/command-ref/new-cli/nix3-flake-lock.md) only creates lock files and adds missing inputs now.
-    It will *never* update existing inputs.
-
-  - [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) does the same, but *will* update inputs.
-    - Passing no arguments will update all inputs of the current flake, just like it already did.
-    - Passing input names as arguments will ensure only those are updated. This replaces the functionality of `nix flake lock --update-input`
-    - To operate on a flake outside the current directory, you must now pass `--flake path/to/flake`.
-
-  - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables.
-    They are superceded by `nix flake update`.
-
-- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches).
-
-- [`nix path-info --json`](@docroot@/command-ref/new-cli/nix3-path-info.md)
-  (experimental) now returns a JSON map rather than JSON list.
-  The `path` field of each object has instead become the key in th outer map, since it is unique.
-  The `valid` field also goes away because we just use null instead.
-
-  - Old way:
-
-    ```json5
-    [
-      {
-        "path": "/nix/store/8fv91097mbh5049i9rglc73dx6kjg3qk-bash-5.2-p15",
-        "valid": true,
-        // ...
-      },
-      {
-        "path": "/nix/store/wffw7l0alvs3iw94cbgi1gmmbmw99sqb-home-manager-path",
-        "valid": false
-      }
-    ]
-    ```
-
-  - New way
-
-    ```json5
-    {
-      "/nix/store/8fv91097mbh5049i9rglc73dx6kjg3qk-bash-5.2-p15": {
-        // ...
-      },
-      "/nix/store/wffw7l0alvs3iw94cbgi1gmmbmw99sqb-home-manager-path": null,
-    }
-    ```
-
-  This makes it match `nix derivation show`, which also maps store paths to information.
-
-- When Nix is installed using the [binary installer](@docroot@/installation/installing-binary.md), in supported shells (Bash, Zsh, Fish)
-  [`XDG_DATA_DIRS`](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html#variables) is now populated with the path to the `/share` subdirectory of the current profile.
-  This means that command completion scripts, `.desktop` files, and similar artifacts installed via [`nix-env`](@docroot@/command-ref/nix-env.md) or [`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md)
-  (experimental) can be found by any program that follows the [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html).
-
-- A new command `nix store add` has been added. It replaces `nix store add-file` and `nix store add-path` which are now deprecated.

From 293ae592576bb9c48975466613fcba6a30d06f5e Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 17 Nov 2023 11:26:45 -0500
Subject: [PATCH 160/294] Fix `make check`

After 9c7749e13508996eb9df83b1692664cc8cdbf952, `libutil-tests_RUN`
doesn't exist. It needs to become `libutil-tests-exe_RUN`.
---
 src/libutil/tests/local.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk
index e6fc4e364..66886c45f 100644
--- a/src/libutil/tests/local.mk
+++ b/src/libutil/tests/local.mk
@@ -1,4 +1,4 @@
-check: libutil-tests_RUN
+check: libutil-tests-exe_RUN
 
 programs += libutil-tests-exe
 

From 4a539ac3eac90b2c2f839cae885df89a03240348 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 17 Nov 2023 17:38:08 +0100
Subject: [PATCH 161/294] Fix buildNoGc

Fixes https://hydra.nixos.org/build/241067941/nixlog/1

    src/libexpr/eval.cc:1776:54: error: variable 'boost::container::small_vector<nix::Value*, 4> vArgs' has initializer but incomplete type
---
 src/libexpr/eval.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 8b0ada517..e9b8cacfd 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -31,6 +31,7 @@
 
 #include <sys/resource.h>
 #include <nlohmann/json.hpp>
+#include <boost/container/small_vector.hpp>
 
 #if HAVE_BOEHMGC
 
@@ -42,7 +43,6 @@
 #include <boost/coroutine2/coroutine.hpp>
 #include <boost/coroutine2/protected_fixedsize_stack.hpp>
 #include <boost/context/stack_context.hpp>
-#include <boost/container/small_vector.hpp>
 
 #endif
 

From 251fb23aeab8f85afb4f8376c2e6fc3d8b229d23 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 19 Nov 2023 01:46:48 +0100
Subject: [PATCH 162/294] Shebang parser: add virtual destructor

Fixes:

    warning: destructor called on non-final 'nix::ParseUnquoted' that has virtual functions but non-virtual destructor [-Wdelete-non-abstract-non-virtual-dtor]
---
 src/libutil/args.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 4359c5e8e..4480a03f5 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -97,6 +97,8 @@ struct Parser {
     virtual void operator()(std::shared_ptr<Parser> & state, Strings & r) = 0;
 
     Parser(std::string_view s) : remaining(s) {};
+
+    virtual ~Parser() { };
 };
 
 struct ParseQuoted : public Parser {

From 70ddf298e0882075dcb1cf69562c629a195718f7 Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <silvan.mosberger@tweag.io>
Date: Sun, 19 Nov 2023 04:09:14 +0100
Subject: [PATCH 163/294] doc: Add link to filterSource from path

---
 src/libexpr/primops.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index e274c3c0c..dda409955 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -2374,7 +2374,7 @@ static RegisterPrimOp primop_path({
           like `@`.
 
         - filter\
-          A function of the type expected by `builtins.filterSource`,
+          A function of the type expected by [`builtins.filterSource`](#builtins-filterSource),
           with the same semantics.
 
         - recursive\

From fe4f573d49a5c47cf9ffd0bd3fe8868104550818 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 19 Nov 2023 11:38:47 +0100
Subject: [PATCH 164/294] flake.nix: Update nixpkgs: release-23.05 ->
 nixos-23.05-small
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nixpkgs':
    'github:NixOS/nixpkgs/9eb24edd6a0027fed010ccfe300a9734d029983c' (2023-11-01)
  → 'github:NixOS/nixpkgs/decdf666c833a325cb4417041a90681499e06a41' (2023-11-18)
---
 flake.lock | 8 ++++----
 flake.nix  | 4 +---
 2 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/flake.lock b/flake.lock
index 991cef1ee..825166717 100644
--- a/flake.lock
+++ b/flake.lock
@@ -34,16 +34,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1698876495,
-        "narHash": "sha256-nsQo2/mkDUFeAjuu92p0dEqhRvHHiENhkKVIV1y0/Oo=",
+        "lastModified": 1700342017,
+        "narHash": "sha256-HaibwlWH5LuqsaibW3sIVjZQtEM/jWtOHX4Nk93abGE=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "9eb24edd6a0027fed010ccfe300a9734d029983c",
+        "rev": "decdf666c833a325cb4417041a90681499e06a41",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "release-23.05",
+        "ref": "nixos-23.05-small",
         "repo": "nixpkgs",
         "type": "github"
       }
diff --git a/flake.nix b/flake.nix
index 05ab7b06d..f21b1a63f 100644
--- a/flake.nix
+++ b/flake.nix
@@ -1,9 +1,7 @@
 {
   description = "The purely functional package manager";
 
-  # FIXME go back to nixos-23.05-small once
-  # https://github.com/NixOS/nixpkgs/pull/264875 is included.
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-23.05";
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small";
   inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
   inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
   inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; };

From 796a7eb92d2b0caf75685126adc7460a4c39cfec Mon Sep 17 00:00:00 2001
From: DavHau <hsngrmpf+github@gmail.com>
Date: Sun, 19 Nov 2023 20:32:23 +0700
Subject: [PATCH 165/294] fetchTree: clarify docs for shallow flag

---
 src/libexpr/primops/fetchTree.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 8031bf809..383ec7c58 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -425,7 +425,8 @@ static RegisterPrimOp primop_fetchGit({
 
       - `shallow` (default: `false`)
 
-        A Boolean parameter that specifies whether fetching a shallow clone is allowed.
+        A Boolean parameter that specifies whether fetching from a shallow remote repository is allowed.
+        This still performs a full clone of what is available on the remote.
 
       - `allRefs`
 

From d5928085d5a542b19cc21b1f299392ee7a0c960b Mon Sep 17 00:00:00 2001
From: nicoo <nicoo@mur.at>
Date: Sun, 19 Nov 2023 19:57:07 +0100
Subject: [PATCH 166/294] builtins.concatMap: Fix typo in error message

---
 src/libexpr/primops.cc            | 2 +-
 src/libexpr/tests/error_traces.cc | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index dda409955..27f502830 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -3459,7 +3459,7 @@ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args,
     for (unsigned int n = 0; n < nrLists; ++n) {
         Value * vElem = args[1]->listElems()[n];
         state.callFunction(*args[0], *vElem, lists[n], pos);
-        state.forceList(lists[n], lists[n].determinePos(args[0]->determinePos(pos)), "while evaluating the return value of the function passed to buitlins.concatMap");
+        state.forceList(lists[n], lists[n].determinePos(args[0]->determinePos(pos)), "while evaluating the return value of the function passed to builtins.concatMap");
         len += lists[n].listSize();
     }
 
diff --git a/src/libexpr/tests/error_traces.cc b/src/libexpr/tests/error_traces.cc
index 139366bcd..81498f65a 100644
--- a/src/libexpr/tests/error_traces.cc
+++ b/src/libexpr/tests/error_traces.cc
@@ -906,12 +906,12 @@ namespace nix {
         ASSERT_TRACE2("concatMap (x: 1) [ \"foo\" ] # TODO",
                       TypeError,
                       hintfmt("value is %s while a list was expected", "an integer"),
-                      hintfmt("while evaluating the return value of the function passed to buitlins.concatMap"));
+                      hintfmt("while evaluating the return value of the function passed to builtins.concatMap"));
 
         ASSERT_TRACE2("concatMap (x: \"foo\") [ 1 2 ] # TODO",
                       TypeError,
                       hintfmt("value is %s while a list was expected", "a string"),
-                      hintfmt("while evaluating the return value of the function passed to buitlins.concatMap"));
+                      hintfmt("while evaluating the return value of the function passed to builtins.concatMap"));
 
     }
 

From 19993398a12069a868b0fb10b63f7d06f0f993e6 Mon Sep 17 00:00:00 2001
From: Qyriad <qyriad@qyriad.me>
Date: Mon, 20 Nov 2023 03:37:02 -0700
Subject: [PATCH 167/294] flakes: check for flake.nix before complaining that
 lstat on it fails

getFlake currently calls lstat (via isLink via canonPath) before it
performs the sanity check that a flake.nix exists in the first place.
This commit moves the check to before path canonicalization, so that
failed symlink check operations don't throw before the check does.
---
 src/libexpr/flake/flake.cc | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index 54de53e0b..b128de31e 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -212,8 +212,16 @@ static Flake getFlake(
     auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree(
         state, originalRef, allowLookup, flakeCache);
 
+    // We need to guard against symlink attacks, but before we start doing
+    // filesystem operations we should make sure there's a flake.nix in the
+    // first place.
+    auto unsafeFlakeDir = state.store->toRealPath(storePath) + "/" + lockedRef.subdir;
+    auto unsafeFlakeFile = unsafeFlakeDir + "/flake.nix";
+    if (!pathExists(unsafeFlakeFile))
+        throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir);
+
     // Guard against symlink attacks.
-    auto flakeDir = canonPath(state.store->toRealPath(storePath) + "/" + lockedRef.subdir, true);
+    auto flakeDir = canonPath(unsafeFlakeDir, true);
     auto flakeFile = canonPath(flakeDir + "/flake.nix", true);
     if (!isInDir(flakeFile, state.store->toRealPath(storePath)))
         throw Error("'flake.nix' file of flake '%s' escapes from '%s'",
@@ -226,9 +234,6 @@ static Flake getFlake(
         .storePath = storePath,
     };
 
-    if (!pathExists(flakeFile))
-        throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir);
-
     Value vInfo;
     state.evalFile(state.rootPath(CanonPath(flakeFile)), vInfo, true); // FIXME: symlink attack
 

From 1d6abec993a371091459d5e23f985c6d69621ce7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 20 Nov 2023 12:35:35 +0100
Subject: [PATCH 168/294] Revert use of boost::container::small_vector in the
 evaluator

It caused random crashes (https://hydra.nixos.org/build/241514506,
https://hydra.nixos.org/build/241443330) because the heap allocation
done by small_vector in the not-small case is not scanned for GC
roots.
---
 src/libexpr/eval.cc    | 11 +++++------
 src/libexpr/primops.cc |  7 ++++---
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index e9b8cacfd..46a49c891 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -31,7 +31,6 @@
 
 #include <sys/resource.h>
 #include <nlohmann/json.hpp>
-#include <boost/container/small_vector.hpp>
 
 #if HAVE_BOEHMGC
 
@@ -1710,7 +1709,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                 /* We have all the arguments, so call the primop with
                    the previous and new arguments. */
 
-                Value * vArgs[maxPrimOpArity];
+                Value * vArgs[arity];
                 auto n = argsDone;
                 for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->primOpApp.left)
                     vArgs[--n] = arg->primOpApp.right;
@@ -1773,11 +1772,11 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v)
     // 4: about 60
     // 5: under 10
     // This excluded attrset lambdas (`{...}:`). Contributions of mixed lambdas appears insignificant at ~150 total.
-    boost::container::small_vector<Value *, 4> vArgs(args.size());
+    Value * vArgs[args.size()];
     for (size_t i = 0; i < args.size(); ++i)
         vArgs[i] = args[i]->maybeThunk(state, env);
 
-    state.callFunction(vFun, args.size(), vArgs.data(), v, pos);
+    state.callFunction(vFun, args.size(), vArgs, v, pos);
 }
 
 
@@ -2016,8 +2015,8 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
         return result;
     };
 
-    boost::container::small_vector<Value, conservativeStackReservation> values(es->size());
-    Value * vTmpP = values.data();
+    Value values[es->size()];
+    Value * vTmpP = values;
 
     for (auto & [i_pos, i] : *es) {
         Value & vTmp = *vTmpP++;
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 27f502830..a8d44d8b7 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -2729,7 +2729,7 @@ static void prim_catAttrs(EvalState & state, const PosIdx pos, Value * * args, V
     auto attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.catAttrs"));
     state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.catAttrs");
 
-    boost::container::small_vector<Value *, nonRecursiveStackReservation> res(args[1]->listSize());
+    Value * res[args[1]->listSize()];
     size_t found = 0;
 
     for (auto v2 : args[1]->listItems()) {
@@ -3064,7 +3064,8 @@ static void prim_filter(EvalState & state, const PosIdx pos, Value * * args, Val
 
     state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filter");
 
-    boost::container::small_vector<Value *, nonRecursiveStackReservation> vs(args[1]->listSize());
+    // FIXME: putting this on the stack is risky.
+    Value * vs[args[1]->listSize()];
     size_t k = 0;
 
     bool same = true;
@@ -3453,7 +3454,7 @@ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args,
     state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.concatMap");
     auto nrLists = args[1]->listSize();
 
-    boost::container::small_vector<Value, conservativeStackReservation> lists(nrLists);
+    Value lists[nrLists];
     size_t len = 0;
 
     for (unsigned int n = 0; n < nrLists; ++n) {

From 7ac39ff05c8353c665174e8df61dd76a2b0b93db Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 20 Nov 2023 13:11:58 +0100
Subject: [PATCH 169/294] refactor Store::buildPaths: convert to string earlier

Preparation for RFC 92 dynamic derivations.
---
 src/libstore/build/entry-points.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc
index 13ff22f45..74eca63f3 100644
--- a/src/libstore/build/entry-points.cc
+++ b/src/libstore/build/entry-points.cc
@@ -15,7 +15,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
 
     worker.run(goals);
 
-    StorePathSet failed;
+    StringSet failed;
     std::optional<Error> ex;
     for (auto & i : goals) {
         if (i->ex) {
@@ -26,9 +26,9 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
         }
         if (i->exitCode != Goal::ecSuccess) {
             if (auto i2 = dynamic_cast<DerivationGoal *>(i.get()))
-                failed.insert(i2->drvPath);
+                failed.insert(std::string { i2->drvPath.to_string() });
             else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get()))
-                failed.insert(i2->storePath);
+                failed.insert(std::string { i2->storePath.to_string()});
         }
     }
 
@@ -37,7 +37,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
         throw std::move(*ex);
     } else if (!failed.empty()) {
         if (ex) logError(ex->info());
-        throw Error(worker.failingExitStatus(), "build of %s failed", showPaths(failed));
+        throw Error(worker.failingExitStatus(), "build of %s failed", concatStringsSep(", ", quoteStrings(failed)));
     }
 }
 

From a5e51a9e02efe2813170fdf0093c98b3d56aed84 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 20 Nov 2023 13:18:25 +0100
Subject: [PATCH 170/294] refactor Worker::childStarted/Terminated: use switch

Preparation for RFC 92 dynamic derivations.
---
 src/libstore/build/worker.cc | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc
index 01914e2d6..01f52e7ab 100644
--- a/src/libstore/build/worker.cc
+++ b/src/libstore/build/worker.cc
@@ -199,8 +199,16 @@ void Worker::childStarted(GoalPtr goal, const std::set<int> & fds,
     child.respectTimeouts = respectTimeouts;
     children.emplace_back(child);
     if (inBuildSlot) {
-        if (goal->jobCategory() == JobCategory::Substitution) nrSubstitutions++;
-        else nrLocalBuilds++;
+        switch (goal->jobCategory()) {
+        case JobCategory::Substitution:
+            nrSubstitutions++;
+            break;
+        case JobCategory::Build:
+            nrLocalBuilds++;
+            break;
+        default:
+            abort();
+        }
     }
 }
 
@@ -212,12 +220,17 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers)
     if (i == children.end()) return;
 
     if (i->inBuildSlot) {
-        if (goal->jobCategory() == JobCategory::Substitution) {
+        switch (goal->jobCategory()) {
+        case JobCategory::Substitution:
             assert(nrSubstitutions > 0);
             nrSubstitutions--;
-        } else {
+            break;
+        case JobCategory::Build:
             assert(nrLocalBuilds > 0);
             nrLocalBuilds--;
+            break;
+        default:
+            abort();
         }
     }
 

From 2a96445d7505cb0a82ed2a49c7210b3073ffd153 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 20 Nov 2023 15:06:28 +0100
Subject: [PATCH 171/294] Bump version

---
 .version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.version b/.version
index ef0f38abe..7329e21c3 100644
--- a/.version
+++ b/.version
@@ -1 +1 @@
-2.19.0
+2.20.0

From e2b6821ca0147f36bcb9404aab080f80746984c8 Mon Sep 17 00:00:00 2001
From: roblabla <unfiltered@roblab.la>
Date: Mon, 20 Nov 2023 15:41:38 +0100
Subject: [PATCH 172/294] Fix bad_format_string error when builder stdout
 contains %

---
 src/libutil/serialise.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index d7950b11b..f465bd0de 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -448,7 +448,7 @@ Error readError(Source & source)
     auto msg = readString(source);
     ErrorInfo info {
         .level = level,
-        .msg = hintformat(fmt("%s", msg)),
+        .msg = hintfmt(msg),
     };
     auto havePos = readNum<size_t>(source);
     assert(havePos == 0);
@@ -457,7 +457,7 @@ Error readError(Source & source)
         havePos = readNum<size_t>(source);
         assert(havePos == 0);
         info.traces.push_back(Trace {
-            .hint = hintformat(fmt("%s", readString(source)))
+            .hint = hintfmt(readString(source))
         });
     }
     return Error(std::move(info));

From e4066c04442f86a5a12d492d588e3e82b533053d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 15 Nov 2023 16:15:47 +0100
Subject: [PATCH 173/294] Fetch specific Git revisions

This is more efficient, and necessary when using shallow=1 with a rev.
---
 src/libfetchers/git.cc       | 5 ++++-
 tests/functional/fetchGit.sh | 4 +---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 7208a0b6d..2fd3fb41e 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -518,8 +518,11 @@ struct GitInputScheme : InputScheme
 
             if (doFetch) {
                 try {
-                    auto fetchRef = getAllRefsAttr(input)
+                    auto fetchRef =
+                        getAllRefsAttr(input)
                         ? "refs/*"
+                        : input.getRev()
+                        ? input.getRev()->gitRev()
                         : ref.compare(0, 5, "refs/") == 0
                         ? ref
                         : ref == "HEAD"
diff --git a/tests/functional/fetchGit.sh b/tests/functional/fetchGit.sh
index c38cd27eb..4985c7764 100644
--- a/tests/functional/fetchGit.sh
+++ b/tests/functional/fetchGit.sh
@@ -51,9 +51,7 @@ git -C $repo add differentbranch
 git -C $repo commit -m 'Test2'
 git -C $repo checkout master
 devrev=$(git -C $repo rev-parse devtest)
-out=$(nix eval --impure --raw --expr "builtins.fetchGit { url = file://$repo; rev = \"$devrev\"; }" 2>&1) || status=$?
-[[ $status == 1 ]]
-[[ $out =~ 'Cannot find Git revision' ]]
+nix eval --impure --raw --expr "builtins.fetchGit { url = file://$repo; rev = \"$devrev\"; }"
 
 [[ $(nix eval --raw --expr "builtins.readFile (builtins.fetchGit { url = file://$repo; rev = \"$devrev\"; allRefs = true; } + \"/differentbranch\")") = 'different file' ]]
 

From 071f14a0bb25ffa8e5aaf8ad37031d205f49ef7d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 15 Nov 2023 17:17:07 +0100
Subject: [PATCH 174/294] Don't do shallow fetches over ssh

---
 src/libfetchers/git-utils.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index f554dcc5f..19eae0e1d 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -381,7 +381,9 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         };
 
         git_fetch_options opts = GIT_FETCH_OPTIONS_INIT;
-        opts.depth = shallow ? 1 : GIT_FETCH_DEPTH_FULL;
+        // FIXME: for some reason, shallow fetching over ssh barfs
+        // with "could not read from remote repository".
+        opts.depth = shallow && parseURL(url).scheme != "ssh" ? 1 : GIT_FETCH_DEPTH_FULL;
         opts.callbacks.payload = &act;
         opts.callbacks.sideband_progress = sidebandProgressCallback;
         opts.callbacks.transfer_progress = transferProgressCallback;

From a0162d5732b23e7fdc1f65df28826611e3a424e5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 20 Nov 2023 18:54:36 +0100
Subject: [PATCH 175/294] Improve SourceAccessor path display
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Backported from lazy-trees. This allows SourceAccessors to show the
origin of the accessor. E.g. we now get

  copying '«git+https://github.com/blender/blender.git?ref=refs/heads/main&rev=4edc1389337dd3679ff66969c332d2aff52e1992»/' to the store

instead of

  copying '/' to the store
---
 src/libexpr/eval.cc                  | 3 +++
 src/libfetchers/fs-input-accessor.cc | 1 +
 src/libfetchers/git.cc               | 2 ++
 src/libutil/posix-source-accessor.hh | 2 +-
 src/libutil/source-accessor.cc       | 9 ++++++++-
 src/libutil/source-accessor.hh       | 4 ++++
 6 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 46a49c891..bf6b6f8c1 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -532,6 +532,9 @@ EvalState::EvalState(
     , baseEnv(allocEnv(128))
     , staticBaseEnv{std::make_shared<StaticEnv>(false, nullptr)}
 {
+    corepkgsFS->setPathDisplay("<nix", ">");
+    internalFS->setPathDisplay("«nix-internal»", "");
+
     countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0";
 
     assert(gcInitialised);
diff --git a/src/libfetchers/fs-input-accessor.cc b/src/libfetchers/fs-input-accessor.cc
index 81be64482..2efee932d 100644
--- a/src/libfetchers/fs-input-accessor.cc
+++ b/src/libfetchers/fs-input-accessor.cc
@@ -18,6 +18,7 @@ struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor
         , allowedPaths(std::move(allowedPaths))
         , makeNotAllowedError(std::move(makeNotAllowedError))
     {
+        displayPrefix = root.isRoot() ? "" : root.abs();
     }
 
     void readFile(
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 7208a0b6d..6bca87304 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -582,6 +582,8 @@ struct GitInputScheme : InputScheme
 
         auto accessor = repo->getAccessor(rev);
 
+        accessor->setPathDisplay("«" + input.to_string() + "»");
+
         /* If the repo has submodules, fetch them and return a mounted
            input accessor consisting of the accessor for the top-level
            repo and the accessors for the submodules. */
diff --git a/src/libutil/posix-source-accessor.hh b/src/libutil/posix-source-accessor.hh
index cf087d26e..a45d96bf8 100644
--- a/src/libutil/posix-source-accessor.hh
+++ b/src/libutil/posix-source-accessor.hh
@@ -7,7 +7,7 @@ namespace nix {
 /**
  * A source accessor that uses the Unix filesystem.
  */
-struct PosixSourceAccessor : SourceAccessor
+struct PosixSourceAccessor : virtual SourceAccessor
 {
     /**
      * The most recent mtime seen by lstat(). This is a hack to
diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc
index e2114e18f..7813433a7 100644
--- a/src/libutil/source-accessor.cc
+++ b/src/libutil/source-accessor.cc
@@ -7,6 +7,7 @@ static std::atomic<size_t> nextNumber{0};
 
 SourceAccessor::SourceAccessor()
     : number(++nextNumber)
+    , displayPrefix{"«unknown»"}
 {
 }
 
@@ -55,9 +56,15 @@ SourceAccessor::Stat SourceAccessor::lstat(const CanonPath & path)
         throw Error("path '%s' does not exist", showPath(path));
 }
 
+void SourceAccessor::setPathDisplay(std::string displayPrefix, std::string displaySuffix)
+{
+    this->displayPrefix = std::move(displayPrefix);
+    this->displaySuffix = std::move(displaySuffix);
+}
+
 std::string SourceAccessor::showPath(const CanonPath & path)
 {
-    return path.abs();
+    return displayPrefix + path.abs() + displaySuffix;
 }
 
 }
diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh
index 1a4e80361..264caab16 100644
--- a/src/libutil/source-accessor.hh
+++ b/src/libutil/source-accessor.hh
@@ -17,6 +17,8 @@ struct SourceAccessor
 {
     const size_t number;
 
+    std::string displayPrefix, displaySuffix;
+
     SourceAccessor();
 
     virtual ~SourceAccessor()
@@ -117,6 +119,8 @@ struct SourceAccessor
         return number < x.number;
     }
 
+    void setPathDisplay(std::string displayPrefix, std::string displaySuffix = "");
+
     virtual std::string showPath(const CanonPath & path);
 };
 

From 99d5204baaef211234d50f20610fa43d304888ce Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 20 Nov 2023 20:04:37 +0100
Subject: [PATCH 176/294] Persistently cache InputAccessor::fetchToStore()

This avoids repeated copying of the same source tree between Nix
invocations. It requires the accessor to have a "fingerprint" (e.g. a
Git revision) that uniquely determines its contents.
---
 src/libfetchers/fetchers.cc       |  5 +++++
 src/libfetchers/fetchers.hh       |  9 +++++++++
 src/libfetchers/git.cc            | 14 +++++++++++++-
 src/libfetchers/github.cc         |  8 ++++++++
 src/libfetchers/input-accessor.cc | 30 ++++++++++++++++++++++++++++++
 src/libfetchers/input-accessor.hh |  2 ++
 src/libfetchers/mercurial.cc      |  8 ++++++++
 src/libstore/content-address.hh   |  4 ++--
 8 files changed, 77 insertions(+), 3 deletions(-)

diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index c2513e076..60208619e 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -108,6 +108,11 @@ Input Input::fromAttrs(Attrs && attrs)
     return std::move(*res);
 }
 
+std::optional<std::string> Input::getFingerprint(ref<Store> store) const
+{
+    return scheme ? scheme->getFingerprint(store, *this) : std::nullopt;
+}
+
 ParsedURL Input::toURL() const
 {
     if (!scheme)
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index ce5aa4c69..5f3254b6d 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -113,6 +113,12 @@ public:
     std::optional<Hash> getRev() const;
     std::optional<uint64_t> getRevCount() const;
     std::optional<time_t> getLastModified() const;
+
+    /**
+     * For locked inputs, return a string that uniquely specifies the
+     * content of the input (typically a commit hash or content hash).
+     */
+    std::optional<std::string> getFingerprint(ref<Store> store) const;
 };
 
 
@@ -180,6 +186,9 @@ struct InputScheme
 
     virtual bool isDirect(const Input & input) const
     { return true; }
+
+    virtual std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const
+    { return std::nullopt; }
 };
 
 void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 7208a0b6d..6b461499b 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -700,10 +700,22 @@ struct GitInputScheme : InputScheme
 
         auto repoInfo = getRepoInfo(input);
 
-        return
+        auto [accessor, final] =
             input.getRef() || input.getRev() || !repoInfo.isLocal
             ? getAccessorFromCommit(store, repoInfo, std::move(input))
             : getAccessorFromWorkdir(store, repoInfo, std::move(input));
+
+        accessor->fingerprint = final.getFingerprint(store);
+
+        return {accessor, std::move(final)};
+    }
+
+    std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
+    {
+        if (auto rev = input.getRev())
+            return rev->gitRev() + (getSubmodulesAttr(input) ? ";s" : "");
+        else
+            return std::nullopt;
     }
 };
 
diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 6c9b29721..661ad4884 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -229,6 +229,14 @@ struct GitArchiveInputScheme : InputScheme
     {
         return Xp::Flakes;
     }
+
+    std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
+    {
+        if (auto rev = input.getRev())
+            return rev->gitRev();
+        else
+            return std::nullopt;
+    }
 };
 
 struct GitHubInputScheme : GitArchiveInputScheme
diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc
index d1d450cf7..53502c621 100644
--- a/src/libfetchers/input-accessor.cc
+++ b/src/libfetchers/input-accessor.cc
@@ -1,5 +1,6 @@
 #include "input-accessor.hh"
 #include "store-api.hh"
+#include "cache.hh"
 
 namespace nix {
 
@@ -11,6 +12,30 @@ StorePath InputAccessor::fetchToStore(
     PathFilter * filter,
     RepairFlag repair)
 {
+    // FIXME: add an optimisation for the case where the accessor is
+    // an FSInputAccessor pointing to a store path.
+
+    std::optional<fetchers::Attrs> cacheKey;
+
+    if (!filter && fingerprint) {
+        cacheKey = fetchers::Attrs{
+            {"_what", "fetchToStore"},
+            {"store", store->storeDir},
+            {"name", std::string(name)},
+            {"fingerprint", *fingerprint},
+            {"method", (uint8_t) method},
+            {"path", path.abs()}
+        };
+        if (auto res = fetchers::getCache()->lookup(*cacheKey)) {
+            StorePath storePath(fetchers::getStrAttr(*res, "storePath"));
+            if (store->isValidPath(storePath)) {
+                debug("store path cache hit for '%s'", showPath(path));
+                return storePath;
+            }
+        }
+    } else
+        debug("source path '%s' is uncacheable", showPath(path));
+
     Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", showPath(path)));
 
     auto source = sinkToSource([&](Sink & sink) {
@@ -25,6 +50,11 @@ StorePath InputAccessor::fetchToStore(
         ? store->computeStorePathFromDump(*source, name, method, htSHA256).first
         : store->addToStoreFromDump(*source, name, method, htSHA256, repair);
 
+    if (cacheKey)
+        fetchers::getCache()->upsert(
+            *cacheKey,
+            fetchers::Attrs{{"storePath", std::string(storePath.to_string())}});
+
     return storePath;
 }
 
diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh
index 9c688a234..26d17f064 100644
--- a/src/libfetchers/input-accessor.hh
+++ b/src/libfetchers/input-accessor.hh
@@ -18,6 +18,8 @@ class Store;
 
 struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this<InputAccessor>
 {
+    std::optional<std::string> fingerprint;
+
     /**
      * Return the maximum last-modified time of the files in this
      * tree, if available.
diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc
index 9244acf39..aa991a75d 100644
--- a/src/libfetchers/mercurial.cc
+++ b/src/libfetchers/mercurial.cc
@@ -339,6 +339,14 @@ struct MercurialInputScheme : InputScheme
 
         return makeResult(infoAttrs, std::move(storePath));
     }
+
+    std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
+    {
+        if (auto rev = input.getRev())
+            return rev->gitRev();
+        else
+            return std::nullopt;
+    }
 };
 
 static auto rMercurialInputScheme = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); });
diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh
index c4d619bdc..bdb558907 100644
--- a/src/libstore/content-address.hh
+++ b/src/libstore/content-address.hh
@@ -39,12 +39,12 @@ enum struct FileIngestionMethod : uint8_t {
     /**
      * Flat-file hashing. Directly ingest the contents of a single file
      */
-    Flat = false,
+    Flat = 0,
     /**
      * Recursive (or NAR) hashing. Serializes the file-system object in Nix
      * Archive format and ingest that
      */
-    Recursive = true
+    Recursive = 1
 };
 
 /**

From 64827360be35a3d16e818aa9d8426ca40b2c4dc2 Mon Sep 17 00:00:00 2001
From: Felix Uhl <github@mail.felix-uhl.de>
Date: Tue, 21 Nov 2023 14:49:48 +0100
Subject: [PATCH 177/294] Fix "unbound variable" errors in bash

Fixes #9414
---
 scripts/nix-profile-daemon.sh.in | 2 +-
 scripts/nix-profile.sh.in        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in
index c63db4648..d256b24ed 100644
--- a/scripts/nix-profile-daemon.sh.in
+++ b/scripts/nix-profile-daemon.sh.in
@@ -31,7 +31,7 @@ fi
 export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK"
 
 # Populate bash completions, .desktop files, etc
-if [ -z "$XDG_DATA_DIRS" ]; then
+if [ -z "${XDG_DATA_DIRS-}" ]; then
     # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default
     export XDG_DATA_DIRS="/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
 else
diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in
index 56e070ae1..44bc96e89 100644
--- a/scripts/nix-profile.sh.in
+++ b/scripts/nix-profile.sh.in
@@ -33,7 +33,7 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
     export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK"
 
     # Populate bash completions, .desktop files, etc
-    if [ -z "$XDG_DATA_DIRS" ]; then
+    if [ -z "${XDG_DATA_DIRS-}" ]; then
         # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default
         export XDG_DATA_DIRS="/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
     else

From 4d8decbd135a78389c463fbb4c844f1bf22aed69 Mon Sep 17 00:00:00 2001
From: Lin Jian <me@linj.tech>
Date: Tue, 21 Nov 2023 15:29:36 +0800
Subject: [PATCH 178/294] doc: fix number of template attributes

This number is not updated when welcomeText is added[1][2].

[1]: f3a2940e70dea2c35dcae3fca019e94bf8758b4d
[2]: https://github.com/NixOS/nix/pull/6103
---
 src/nix/flake-init.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/flake-init.md b/src/nix/flake-init.md
index fc1f4f805..ea274bf29 100644
--- a/src/nix/flake-init.md
+++ b/src/nix/flake-init.md
@@ -30,7 +30,7 @@ using `-t`.
 # Template definitions
 
 A flake can declare templates through its `templates` output
-attribute. A template has two attributes:
+attribute. A template has the following attributes:
 
 * `description`: A one-line description of the template, in CommonMark
   syntax.

From f880469173061a07f0b2a24734932c5a9ad633c6 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 19 Nov 2023 10:17:57 -0500
Subject: [PATCH 179/294] Put `canonicaliseTimestampAndPermissions` in its own
 header/file

It is not inherently tied to `LocalStore`, it could probably even go in
`libnixutil`. Functions not attached to `LocalStore` should not be
declared in `local-store.hh`.

I am moving it to facilitate experimenting for #9344. If
canonicalisation should be done client-side in client-side builds, there
wouldn't be a `LocalStore` at all so having to include that header to
get this freestanding function is cumbersome and wrong.

Perhaps canonicalisation should still be done server-side for security
reasons --- I don't mean to make that judgement call now --- but even if
so, this freestanding function still isn't connected to `LocalStore` so
while less urgent it is still better to move out of this header.
---
 src/libstore/build/local-derivation-goal.cc |   1 +
 src/libstore/local-store.cc                 | 159 +-----------------
 src/libstore/local-store.hh                 |  34 ----
 src/libstore/optimise-store.cc              |   1 +
 src/libstore/posix-fs-canonicalise.cc       | 169 ++++++++++++++++++++
 src/libstore/posix-fs-canonicalise.hh       |  45 ++++++
 src/nix-store/nix-store.cc                  |   1 +
 7 files changed, 218 insertions(+), 192 deletions(-)
 create mode 100644 src/libstore/posix-fs-canonicalise.cc
 create mode 100644 src/libstore/posix-fs-canonicalise.hh

diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index a9f930773..198402ff7 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -19,6 +19,7 @@
 #include "namespaces.hh"
 #include "child.hh"
 #include "unix-domain-socket.hh"
+#include "posix-fs-canonicalise.hh"
 
 #include <regex>
 #include <queue>
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 2a3582ad8..4ff75f528 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -11,6 +11,7 @@
 #include "finally.hh"
 #include "compression.hh"
 #include "signals.hh"
+#include "posix-fs-canonicalise.hh"
 
 #include <iostream>
 #include <algorithm>
@@ -581,164 +582,6 @@ void LocalStore::makeStoreWritable()
 }
 
 
-const time_t mtimeStore = 1; /* 1 second into the epoch */
-
-
-static void canonicaliseTimestampAndPermissions(const Path & path, const struct stat & st)
-{
-    if (!S_ISLNK(st.st_mode)) {
-
-        /* Mask out all type related bits. */
-        mode_t mode = st.st_mode & ~S_IFMT;
-
-        if (mode != 0444 && mode != 0555) {
-            mode = (st.st_mode & S_IFMT)
-                 | 0444
-                 | (st.st_mode & S_IXUSR ? 0111 : 0);
-            if (chmod(path.c_str(), mode) == -1)
-                throw SysError("changing mode of '%1%' to %2$o", path, mode);
-        }
-
-    }
-
-    if (st.st_mtime != mtimeStore) {
-        struct timeval times[2];
-        times[0].tv_sec = st.st_atime;
-        times[0].tv_usec = 0;
-        times[1].tv_sec = mtimeStore;
-        times[1].tv_usec = 0;
-#if HAVE_LUTIMES
-        if (lutimes(path.c_str(), times) == -1)
-            if (errno != ENOSYS ||
-                (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1))
-#else
-        if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)
-#endif
-            throw SysError("changing modification time of '%1%'", path);
-    }
-}
-
-
-void canonicaliseTimestampAndPermissions(const Path & path)
-{
-    canonicaliseTimestampAndPermissions(path, lstat(path));
-}
-
-
-static void canonicalisePathMetaData_(
-    const Path & path,
-    std::optional<std::pair<uid_t, uid_t>> uidRange,
-    InodesSeen & inodesSeen)
-{
-    checkInterrupt();
-
-#if __APPLE__
-    /* Remove flags, in particular UF_IMMUTABLE which would prevent
-       the file from being garbage-collected. FIXME: Use
-       setattrlist() to remove other attributes as well. */
-    if (lchflags(path.c_str(), 0)) {
-        if (errno != ENOTSUP)
-            throw SysError("clearing flags of path '%1%'", path);
-    }
-#endif
-
-    auto st = lstat(path);
-
-    /* Really make sure that the path is of a supported type. */
-    if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)))
-        throw Error("file '%1%' has an unsupported type", path);
-
-#if __linux__
-    /* Remove extended attributes / ACLs. */
-    ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0);
-
-    if (eaSize < 0) {
-        if (errno != ENOTSUP && errno != ENODATA)
-            throw SysError("querying extended attributes of '%s'", path);
-    } else if (eaSize > 0) {
-        std::vector<char> eaBuf(eaSize);
-
-        if ((eaSize = llistxattr(path.c_str(), eaBuf.data(), eaBuf.size())) < 0)
-            throw SysError("querying extended attributes of '%s'", path);
-
-        for (auto & eaName: tokenizeString<Strings>(std::string(eaBuf.data(), eaSize), std::string("\000", 1))) {
-            if (settings.ignoredAcls.get().count(eaName)) continue;
-            if (lremovexattr(path.c_str(), eaName.c_str()) == -1)
-                throw SysError("removing extended attribute '%s' from '%s'", eaName, path);
-        }
-     }
-#endif
-
-    /* Fail if the file is not owned by the build user.  This prevents
-       us from messing up the ownership/permissions of files
-       hard-linked into the output (e.g. "ln /etc/shadow $out/foo").
-       However, ignore files that we chown'ed ourselves previously to
-       ensure that we don't fail on hard links within the same build
-       (i.e. "touch $out/foo; ln $out/foo $out/bar"). */
-    if (uidRange && (st.st_uid < uidRange->first || st.st_uid > uidRange->second)) {
-        if (S_ISDIR(st.st_mode) || !inodesSeen.count(Inode(st.st_dev, st.st_ino)))
-            throw BuildError("invalid ownership on file '%1%'", path);
-        mode_t mode = st.st_mode & ~S_IFMT;
-        assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore));
-        return;
-    }
-
-    inodesSeen.insert(Inode(st.st_dev, st.st_ino));
-
-    canonicaliseTimestampAndPermissions(path, st);
-
-    /* Change ownership to the current uid.  If it's a symlink, use
-       lchown if available, otherwise don't bother.  Wrong ownership
-       of a symlink doesn't matter, since the owning user can't change
-       the symlink and can't delete it because the directory is not
-       writable.  The only exception is top-level paths in the Nix
-       store (since that directory is group-writable for the Nix build
-       users group); we check for this case below. */
-    if (st.st_uid != geteuid()) {
-#if HAVE_LCHOWN
-        if (lchown(path.c_str(), geteuid(), getegid()) == -1)
-#else
-        if (!S_ISLNK(st.st_mode) &&
-            chown(path.c_str(), geteuid(), getegid()) == -1)
-#endif
-            throw SysError("changing owner of '%1%' to %2%",
-                path, geteuid());
-    }
-
-    if (S_ISDIR(st.st_mode)) {
-        DirEntries entries = readDirectory(path);
-        for (auto & i : entries)
-            canonicalisePathMetaData_(path + "/" + i.name, uidRange, inodesSeen);
-    }
-}
-
-
-void canonicalisePathMetaData(
-    const Path & path,
-    std::optional<std::pair<uid_t, uid_t>> uidRange,
-    InodesSeen & inodesSeen)
-{
-    canonicalisePathMetaData_(path, uidRange, inodesSeen);
-
-    /* On platforms that don't have lchown(), the top-level path can't
-       be a symlink, since we can't change its ownership. */
-    auto st = lstat(path);
-
-    if (st.st_uid != geteuid()) {
-        assert(S_ISLNK(st.st_mode));
-        throw Error("wrong ownership of top-level store path '%1%'", path);
-    }
-}
-
-
-void canonicalisePathMetaData(const Path & path,
-    std::optional<std::pair<uid_t, uid_t>> uidRange)
-{
-    InodesSeen inodesSeen;
-    canonicalisePathMetaData(path, uidRange, inodesSeen);
-}
-
-
 void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs)
 {
     experimentalFeatureSettings.require(Xp::CaDerivations);
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 6d589bee5..8f0ffd2a2 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -371,38 +371,4 @@ private:
     friend struct DerivationGoal;
 };
 
-
-typedef std::pair<dev_t, ino_t> Inode;
-typedef std::set<Inode> InodesSeen;
-
-
-/**
- * "Fix", or canonicalise, the meta-data of the files in a store path
- * after it has been built.  In particular:
- *
- * - the last modification date on each file is set to 1 (i.e.,
- *   00:00:01 1/1/1970 UTC)
- *
- * - the permissions are set of 444 or 555 (i.e., read-only with or
- *   without execute permission; setuid bits etc. are cleared)
- *
- * - the owner and group are set to the Nix user and group, if we're
- *   running as root.
- *
- * If uidRange is not empty, this function will throw an error if it
- * encounters files owned by a user outside of the closed interval
- * [uidRange->first, uidRange->second].
- */
-void canonicalisePathMetaData(
-    const Path & path,
-    std::optional<std::pair<uid_t, uid_t>> uidRange,
-    InodesSeen & inodesSeen);
-void canonicalisePathMetaData(
-    const Path & path,
-    std::optional<std::pair<uid_t, uid_t>> uidRange);
-
-void canonicaliseTimestampAndPermissions(const Path & path);
-
-MakeError(PathInUse, Error);
-
 }
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index a4ac413b3..0fa977545 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -1,6 +1,7 @@
 #include "local-store.hh"
 #include "globals.hh"
 #include "signals.hh"
+#include "posix-fs-canonicalise.hh"
 
 #include <cstdlib>
 #include <cstring>
diff --git a/src/libstore/posix-fs-canonicalise.cc b/src/libstore/posix-fs-canonicalise.cc
new file mode 100644
index 000000000..cc3ab0b74
--- /dev/null
+++ b/src/libstore/posix-fs-canonicalise.cc
@@ -0,0 +1,169 @@
+#include <sys/xattr.h>
+
+#include "posix-fs-canonicalise.hh"
+#include "file-system.hh"
+#include "signals.hh"
+#include "util.hh"
+#include "globals.hh"
+#include "store-api.hh"
+
+namespace nix {
+
+const time_t mtimeStore = 1; /* 1 second into the epoch */
+
+
+static void canonicaliseTimestampAndPermissions(const Path & path, const struct stat & st)
+{
+    if (!S_ISLNK(st.st_mode)) {
+
+        /* Mask out all type related bits. */
+        mode_t mode = st.st_mode & ~S_IFMT;
+
+        if (mode != 0444 && mode != 0555) {
+            mode = (st.st_mode & S_IFMT)
+                 | 0444
+                 | (st.st_mode & S_IXUSR ? 0111 : 0);
+            if (chmod(path.c_str(), mode) == -1)
+                throw SysError("changing mode of '%1%' to %2$o", path, mode);
+        }
+
+    }
+
+    if (st.st_mtime != mtimeStore) {
+        struct timeval times[2];
+        times[0].tv_sec = st.st_atime;
+        times[0].tv_usec = 0;
+        times[1].tv_sec = mtimeStore;
+        times[1].tv_usec = 0;
+#if HAVE_LUTIMES
+        if (lutimes(path.c_str(), times) == -1)
+            if (errno != ENOSYS ||
+                (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1))
+#else
+        if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)
+#endif
+            throw SysError("changing modification time of '%1%'", path);
+    }
+}
+
+
+void canonicaliseTimestampAndPermissions(const Path & path)
+{
+    canonicaliseTimestampAndPermissions(path, lstat(path));
+}
+
+
+static void canonicalisePathMetaData_(
+    const Path & path,
+    std::optional<std::pair<uid_t, uid_t>> uidRange,
+    InodesSeen & inodesSeen)
+{
+    checkInterrupt();
+
+#if __APPLE__
+    /* Remove flags, in particular UF_IMMUTABLE which would prevent
+       the file from being garbage-collected. FIXME: Use
+       setattrlist() to remove other attributes as well. */
+    if (lchflags(path.c_str(), 0)) {
+        if (errno != ENOTSUP)
+            throw SysError("clearing flags of path '%1%'", path);
+    }
+#endif
+
+    auto st = lstat(path);
+
+    /* Really make sure that the path is of a supported type. */
+    if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)))
+        throw Error("file '%1%' has an unsupported type", path);
+
+#if __linux__
+    /* Remove extended attributes / ACLs. */
+    ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0);
+
+    if (eaSize < 0) {
+        if (errno != ENOTSUP && errno != ENODATA)
+            throw SysError("querying extended attributes of '%s'", path);
+    } else if (eaSize > 0) {
+        std::vector<char> eaBuf(eaSize);
+
+        if ((eaSize = llistxattr(path.c_str(), eaBuf.data(), eaBuf.size())) < 0)
+            throw SysError("querying extended attributes of '%s'", path);
+
+        for (auto & eaName: tokenizeString<Strings>(std::string(eaBuf.data(), eaSize), std::string("\000", 1))) {
+            if (settings.ignoredAcls.get().count(eaName)) continue;
+            if (lremovexattr(path.c_str(), eaName.c_str()) == -1)
+                throw SysError("removing extended attribute '%s' from '%s'", eaName, path);
+        }
+     }
+#endif
+
+    /* Fail if the file is not owned by the build user.  This prevents
+       us from messing up the ownership/permissions of files
+       hard-linked into the output (e.g. "ln /etc/shadow $out/foo").
+       However, ignore files that we chown'ed ourselves previously to
+       ensure that we don't fail on hard links within the same build
+       (i.e. "touch $out/foo; ln $out/foo $out/bar"). */
+    if (uidRange && (st.st_uid < uidRange->first || st.st_uid > uidRange->second)) {
+        if (S_ISDIR(st.st_mode) || !inodesSeen.count(Inode(st.st_dev, st.st_ino)))
+            throw BuildError("invalid ownership on file '%1%'", path);
+        mode_t mode = st.st_mode & ~S_IFMT;
+        assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore));
+        return;
+    }
+
+    inodesSeen.insert(Inode(st.st_dev, st.st_ino));
+
+    canonicaliseTimestampAndPermissions(path, st);
+
+    /* Change ownership to the current uid.  If it's a symlink, use
+       lchown if available, otherwise don't bother.  Wrong ownership
+       of a symlink doesn't matter, since the owning user can't change
+       the symlink and can't delete it because the directory is not
+       writable.  The only exception is top-level paths in the Nix
+       store (since that directory is group-writable for the Nix build
+       users group); we check for this case below. */
+    if (st.st_uid != geteuid()) {
+#if HAVE_LCHOWN
+        if (lchown(path.c_str(), geteuid(), getegid()) == -1)
+#else
+        if (!S_ISLNK(st.st_mode) &&
+            chown(path.c_str(), geteuid(), getegid()) == -1)
+#endif
+            throw SysError("changing owner of '%1%' to %2%",
+                path, geteuid());
+    }
+
+    if (S_ISDIR(st.st_mode)) {
+        DirEntries entries = readDirectory(path);
+        for (auto & i : entries)
+            canonicalisePathMetaData_(path + "/" + i.name, uidRange, inodesSeen);
+    }
+}
+
+
+void canonicalisePathMetaData(
+    const Path & path,
+    std::optional<std::pair<uid_t, uid_t>> uidRange,
+    InodesSeen & inodesSeen)
+{
+    canonicalisePathMetaData_(path, uidRange, inodesSeen);
+
+    /* On platforms that don't have lchown(), the top-level path can't
+       be a symlink, since we can't change its ownership. */
+    auto st = lstat(path);
+
+    if (st.st_uid != geteuid()) {
+        assert(S_ISLNK(st.st_mode));
+        throw Error("wrong ownership of top-level store path '%1%'", path);
+    }
+}
+
+
+void canonicalisePathMetaData(const Path & path,
+    std::optional<std::pair<uid_t, uid_t>> uidRange)
+{
+    InodesSeen inodesSeen;
+    canonicalisePathMetaData(path, uidRange, inodesSeen);
+}
+
+}
diff --git a/src/libstore/posix-fs-canonicalise.hh b/src/libstore/posix-fs-canonicalise.hh
new file mode 100644
index 000000000..35644af12
--- /dev/null
+++ b/src/libstore/posix-fs-canonicalise.hh
@@ -0,0 +1,45 @@
+#pragma once
+///@file
+
+#include <sys/stat.h>
+#include <sys/time.h>
+
+#include "types.hh"
+#include "error.hh"
+
+namespace nix {
+
+typedef std::pair<dev_t, ino_t> Inode;
+typedef std::set<Inode> InodesSeen;
+
+
+/**
+ * "Fix", or canonicalise, the meta-data of the files in a store path
+ * after it has been built.  In particular:
+ *
+ * - the last modification date on each file is set to 1 (i.e.,
+ *   00:00:01 1/1/1970 UTC)
+ *
+ * - the permissions are set of 444 or 555 (i.e., read-only with or
+ *   without execute permission; setuid bits etc. are cleared)
+ *
+ * - the owner and group are set to the Nix user and group, if we're
+ *   running as root.
+ *
+ * If uidRange is not empty, this function will throw an error if it
+ * encounters files owned by a user outside of the closed interval
+ * [uidRange->first, uidRange->second].
+ */
+void canonicalisePathMetaData(
+    const Path & path,
+    std::optional<std::pair<uid_t, uid_t>> uidRange,
+    InodesSeen & inodesSeen);
+void canonicalisePathMetaData(
+    const Path & path,
+    std::optional<std::pair<uid_t, uid_t>> uidRange);
+
+void canonicaliseTimestampAndPermissions(const Path & path);
+
+MakeError(PathInUse, Error);
+
+}
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 123283dfe..25f0107bc 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -14,6 +14,7 @@
 #include "graphml.hh"
 #include "legacy.hh"
 #include "path-with-outputs.hh"
+#include "posix-fs-canonicalise.hh"
 
 #include <iostream>
 #include <algorithm>

From 949f5841f8a8611d0f49793bd8c4963462d62e3a Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 21 Nov 2023 13:12:55 -0500
Subject: [PATCH 180/294] Add the `MountedSSHStore` experimental feature

It will be implemented in the subsequent commits of this PR.
---
 src/libutil/experimental-features.cc | 7 +++++++
 src/libutil/experimental-features.hh | 1 +
 2 files changed, 8 insertions(+)

diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc
index ac4d189e1..2418e3f4c 100644
--- a/src/libutil/experimental-features.cc
+++ b/src/libutil/experimental-features.cc
@@ -262,6 +262,13 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
             Allow the use of the [impure-env](@docroot@/command-ref/conf-file.md#conf-impure-env) setting.
         )",
     },
+    {
+        .tag = Xp::MountedSSHStore,
+        .name = "mounted-ssh-store",
+        .description = R"(
+            Allow the use of the [`mounted SSH store`](@docroot@/command-ref/new-cli/nix3-help-stores.html#experimental-ssh-store-with-filesytem-mounted).
+        )",
+    },
     {
         .tag = Xp::VerifiedFetches,
         .name = "verified-fetches",
diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh
index c355b8081..eae4fa9b8 100644
--- a/src/libutil/experimental-features.hh
+++ b/src/libutil/experimental-features.hh
@@ -34,6 +34,7 @@ enum struct ExperimentalFeature
     ParseTomlTimestamps,
     ReadOnlyLocalStore,
     ConfigurableImpureEnv,
+    MountedSSHStore,
     VerifiedFetches,
 };
 

From 9796ebd7ef8c9a23ee8128273d925acae00e43b0 Mon Sep 17 00:00:00 2001
From: Matej Urbas <matej.urbas@gmail.com>
Date: Mon, 17 Apr 2023 12:08:42 -0400
Subject: [PATCH 181/294] Add `--process-ops` flag to `nix-daemon`

---
 src/nix/daemon.cc | 33 +++++++++++++++++++++++++++++----
 1 file changed, 29 insertions(+), 4 deletions(-)

diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc
index 373dedf7c..4dada8e0e 100644
--- a/src/nix/daemon.cc
+++ b/src/nix/daemon.cc
@@ -443,16 +443,23 @@ static void processStdioConnection(ref<Store> store, TrustedFlag trustClient)
  *
  * @param forceTrustClientOpt See `daemonLoop()` and the parameter with
  * the same name over there for details.
+ *
+ * @param procesOps Whether to force processing ops even if the next
+ * store also is a remote store and could process it directly.
  */
-static void runDaemon(bool stdio, std::optional<TrustedFlag> forceTrustClientOpt)
+static void runDaemon(bool stdio, std::optional<TrustedFlag> forceTrustClientOpt, bool processOps)
 {
     if (stdio) {
         auto store = openUncachedStore();
 
+        std::shared_ptr<RemoteStore> remoteStore;
+
         // If --force-untrusted is passed, we cannot forward the connection and
         // must process it ourselves (before delegating to the next store) to
         // force untrusting the client.
-        if (auto remoteStore = store.dynamic_pointer_cast<RemoteStore>(); remoteStore && (!forceTrustClientOpt || *forceTrustClientOpt != NotTrusted))
+        processOps |= !forceTrustClientOpt || *forceTrustClientOpt != NotTrusted;
+
+        if (!processOps && (remoteStore = store.dynamic_pointer_cast<RemoteStore>()))
             forwardStdioConnection(*remoteStore);
         else
             // `Trusted` is passed in the auto (no override case) because we
@@ -468,6 +475,7 @@ static int main_nix_daemon(int argc, char * * argv)
     {
         auto stdio = false;
         std::optional<TrustedFlag> isTrustedOpt = std::nullopt;
+        auto processOps = false;
 
         parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
             if (*arg == "--daemon")
@@ -487,11 +495,14 @@ static int main_nix_daemon(int argc, char * * argv)
             } else if (*arg == "--default-trust") {
                 experimentalFeatureSettings.require(Xp::DaemonTrustOverride);
                 isTrustedOpt = std::nullopt;
+            } else if (*arg == "--process-ops") {
+                experimentalFeatureSettings.require(Xp::MountedSSHStore);
+                processOps = true;
             } else return false;
             return true;
         });
 
-        runDaemon(stdio, isTrustedOpt);
+        runDaemon(stdio, isTrustedOpt, processOps);
 
         return 0;
     }
@@ -503,6 +514,7 @@ struct CmdDaemon : StoreCommand
 {
     bool stdio = false;
     std::optional<TrustedFlag> isTrustedOpt = std::nullopt;
+    bool processOps = false;
 
     CmdDaemon()
     {
@@ -538,6 +550,19 @@ struct CmdDaemon : StoreCommand
             }},
             .experimentalFeature = Xp::DaemonTrustOverride,
         });
+
+        addFlag({
+            .longName = "process-ops",
+            .description = R"(
+              Forces the daemon to process received commands itself rather than forwarding the commands straight to the remote store.
+
+              This is useful for the `mounted-ssh://` store where some actions need to be performed on the remote end but as connected user, and not as the user of the underlying daemon on the remote end.
+            )",
+            .handler = {[&]() {
+                processOps = true;
+            }},
+            .experimentalFeature = Xp::MountedSSHStore,
+        });
     }
 
     std::string description() override
@@ -556,7 +581,7 @@ struct CmdDaemon : StoreCommand
 
     void run(ref<Store> store) override
     {
-        runDaemon(stdio, isTrustedOpt);
+        runDaemon(stdio, isTrustedOpt, processOps);
     }
 };
 

From 226b0f3956ef83ec51b6556d8f27e13a966b4ebf Mon Sep 17 00:00:00 2001
From: Matej Urbas <matej.urbas@gmail.com>
Date: Mon, 17 Apr 2023 12:04:18 -0400
Subject: [PATCH 182/294] Extend the worker protocol with `wopAddPermRoot`

---
 src/libstore/daemon.cc          | 15 +++++++++++++++
 src/libstore/worker-protocol.hh |  3 ++-
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 105d92f25..be9b0b0d3 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -657,6 +657,21 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
+    case WorkerProto::Op::AddPermRoot: {
+        if (!trusted)
+            throw Error(
+                "you are not privileged to create perm roots\n\n"
+                "hint: you can just do this client-side without special privileges, and probably want to do that instead.");
+        auto storePath = WorkerProto::Serialise<StorePath>::read(*store, rconn);
+        Path gcRoot = absPath(readString(from));
+        logger->startWork();
+        auto & localFSStore = require<LocalFSStore>(*store);
+        localFSStore.addPermRoot(storePath, gcRoot);
+        logger->stopWork();
+        to << gcRoot;
+        break;
+    }
+
     case WorkerProto::Op::AddIndirectRoot: {
         Path path = absPath(readString(from));
 
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 25d544ba7..8a26c09c5 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -9,7 +9,7 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION (1 << 8 | 35)
+#define PROTOCOL_VERSION (1 << 8 | 36)
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
@@ -161,6 +161,7 @@ enum struct WorkerProto::Op : uint64_t
     AddMultipleToStore = 44,
     AddBuildLog = 45,
     BuildPathsWithResults = 46,
+    AddPermRoot = 47,
 };
 
 /**

From 06b8902562089811e5724aa0b8d719f891ab73f2 Mon Sep 17 00:00:00 2001
From: Matej Urbas <matej.urbas@gmail.com>
Date: Sat, 15 Apr 2023 11:02:41 +0100
Subject: [PATCH 183/294] MountedSSHStore: stores on shared filesystems

---
 src/libstore/indirect-root-store.hh |  24 ++++++
 src/libstore/mounted-ssh-store.md   |  18 ++++
 src/libstore/ssh-store.cc           | 123 +++++++++++++++++++++++++++-
 3 files changed, 164 insertions(+), 1 deletion(-)
 create mode 100644 src/libstore/mounted-ssh-store.md

diff --git a/src/libstore/indirect-root-store.hh b/src/libstore/indirect-root-store.hh
index 59e45af45..c11679fe8 100644
--- a/src/libstore/indirect-root-store.hh
+++ b/src/libstore/indirect-root-store.hh
@@ -11,6 +11,30 @@ namespace nix {
  * reference.
  *
  * See methods for details on the operations it represents.
+ *
+ * @note
+ * To understand the purpose of this class, it might help to do some
+ * "closed-world" rather than "open-world" reasoning, and consider the
+ * problem it solved for us. This class was factored out from
+ * `LocalFSStore` in order to support the following table, which
+ * contains 4 concrete store types (non-abstract classes, exposed to the
+ * user), and how they implemented the two GC root methods:
+ *
+ * @note
+ * |                   | `addPermRoot()` | `addIndirectRoot()` |
+ * |-------------------|-----------------|---------------------|
+ * | `LocalStore`      | local           | local               |
+ * | `UDSRemoteStore`  | local           | remote              |
+ * | `SSHStore`        | doesn't have    | doesn't have        |
+ * | `MountedSSHStore` | remote          | doesn't have        |
+ *
+ * @note
+ * Note how only the local implementations of `addPermRoot()` need
+ * `addIndirectRoot()`; that is what this class enforces. Without it,
+ * and with `addPermRoot()` and `addIndirectRoot()` both `virtual`, we
+ * would accidentally be allowing for a combinatorial explosion of
+ * possible implementations many of which make no sense. Having this and
+ * that invariant enforced cuts down that space.
  */
 struct IndirectRootStore : public virtual LocalFSStore
 {
diff --git a/src/libstore/mounted-ssh-store.md b/src/libstore/mounted-ssh-store.md
new file mode 100644
index 000000000..1ebfe3081
--- /dev/null
+++ b/src/libstore/mounted-ssh-store.md
@@ -0,0 +1,18 @@
+R"(
+
+**Store URL format**: `mounted-ssh-ng://[username@]hostname`
+
+Experimental store type that allows full access to a Nix store on a remote machine,
+and additionally requires that store be mounted in the local file system.
+
+The mounting of that store is not managed by Nix, and must by managed manually.
+It could be accomplished with SSHFS or NFS, for example.
+
+The local file system is used to optimize certain operations.
+For example, rather than serializing Nix archives and sending over the Nix channel,
+we can directly access the file system data via the mount-point.
+
+The local file system is also used to make certain operations possible that wouldn't otherwise be.
+For example, persistent GC roots can be created if they reside on the same file system as the remote store:
+the remote side will create the symlinks necessary to avoid race conditions.
+)"
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index 4a6aad449..d4c8ab5b2 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -3,9 +3,10 @@
 #include "local-fs-store.hh"
 #include "remote-store.hh"
 #include "remote-store-connection.hh"
-#include "remote-fs-accessor.hh"
+#include "source-accessor.hh"
 #include "archive.hh"
 #include "worker-protocol.hh"
+#include "worker-protocol-impl.hh"
 #include "pool.hh"
 #include "ssh.hh"
 
@@ -78,6 +79,8 @@ protected:
 
     std::string host;
 
+    std::vector<std::string> extraRemoteProgramArgs;
+
     SSHMaster master;
 
     void setOptions(RemoteStore::Connection & conn) override
@@ -91,6 +94,121 @@ protected:
     };
 };
 
+struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfig
+{
+    using SSHStoreConfig::SSHStoreConfig;
+    using LocalFSStoreConfig::LocalFSStoreConfig;
+
+    MountedSSHStoreConfig(StringMap params)
+        : StoreConfig(params)
+        , RemoteStoreConfig(params)
+        , CommonSSHStoreConfig(params)
+        , SSHStoreConfig(params)
+        , LocalFSStoreConfig(params)
+    {
+    }
+
+    const std::string name() override { return "Experimental SSH Store with filesytem mounted"; }
+
+    std::string doc() override
+    {
+        return
+          #include "mounted-ssh-store.md"
+          ;
+    }
+
+    std::optional<ExperimentalFeature> experimentalFeature() const override
+    {
+        return ExperimentalFeature::MountedSSHStore;
+    }
+};
+
+/**
+ * The mounted ssh store assumes that filesystems on the remote host are
+ * shared with the local host. This means that the remote nix store is
+ * available locally and is therefore treated as a local filesystem
+ * store.
+ *
+ * MountedSSHStore is very similar to UDSRemoteStore --- ignoring the
+ * superficial differnce of SSH vs Unix domain sockets, they both are
+ * accessing remote stores, and they both assume the store will be
+ * mounted in the local filesystem.
+ *
+ * The difference lies in how they manage GC roots. See addPermRoot
+ * below for details.
+ */
+class MountedSSHStore : public virtual MountedSSHStoreConfig, public virtual SSHStore, public virtual LocalFSStore
+{
+public:
+
+    MountedSSHStore(const std::string & scheme, const std::string & host, const Params & params)
+        : StoreConfig(params)
+        , RemoteStoreConfig(params)
+        , CommonSSHStoreConfig(params)
+        , SSHStoreConfig(params)
+        , LocalFSStoreConfig(params)
+        , MountedSSHStoreConfig(params)
+        , Store(params)
+        , RemoteStore(params)
+        , SSHStore(scheme, host, params)
+        , LocalFSStore(params)
+    {
+        extraRemoteProgramArgs = {
+            "--process-ops",
+        };
+    }
+
+    static std::set<std::string> uriSchemes()
+    {
+        return {"mounted-ssh-ng"};
+    }
+
+    std::string getUri() override
+    {
+        return *uriSchemes().begin() + "://" + host;
+    }
+
+    void narFromPath(const StorePath & path, Sink & sink) override
+    {
+        return LocalFSStore::narFromPath(path, sink);
+    }
+
+    ref<SourceAccessor> getFSAccessor(bool requireValidPath) override
+    {
+        return LocalFSStore::getFSAccessor(requireValidPath);
+    }
+
+    std::optional<std::string> getBuildLogExact(const StorePath & path) override
+    {
+        return LocalFSStore::getBuildLogExact(path);
+    }
+
+    /**
+     * This is the key difference from UDSRemoteStore: UDSRemote store
+     * has the client create the direct root, and the remote side create
+     * the indirect root.
+     *
+     * We could also do that, but the race conditions (will the remote
+     * side see the direct root the client made?) seems bigger.
+     *
+     * In addition, the remote-side will have a process associated with
+     * the authenticating user handling the connection (even if there
+     * is a system-wide daemon or similar). This process can safely make
+     * the direct and indirect roots without there being such a risk of
+     * privilege escalation / symlinks in directories owned by the
+     * originating requester that they cannot delete.
+     */
+    Path addPermRoot(const StorePath & path, const Path & gcRoot) override
+    {
+        auto conn(getConnection());
+        conn->to << WorkerProto::Op::AddPermRoot;
+        WorkerProto::write(*this, *conn, path);
+        WorkerProto::write(*this, *conn, gcRoot);
+        conn.processStderr();
+        return readString(conn->from);
+    }
+};
+
 ref<RemoteStore::Connection> SSHStore::openConnection()
 {
     auto conn = make_ref<Connection>();
@@ -98,6 +216,8 @@ ref<RemoteStore::Connection> SSHStore::openConnection()
     std::string command = remoteProgram + " --stdio";
     if (remoteStore.get() != "")
         command += " --store " + shellEscape(remoteStore.get());
+    for (auto & arg : extraRemoteProgramArgs)
+        command += " " + shellEscape(arg);
 
     conn->sshConn = master.startCommand(command);
     conn->to = FdSink(conn->sshConn->in.get());
@@ -106,5 +226,6 @@ ref<RemoteStore::Connection> SSHStore::openConnection()
 }
 
 static RegisterStoreImplementation<SSHStore, SSHStoreConfig> regSSHStore;
+static RegisterStoreImplementation<MountedSSHStore, MountedSSHStoreConfig> regMountedSSHStore;
 
 }

From b32b20a6d7cf3a9cf2c81a133c255e9c2ee8e308 Mon Sep 17 00:00:00 2001
From: mupdt <25388474+mupdt@users.noreply.github.com>
Date: Mon, 31 Jul 2023 09:07:43 -0400
Subject: [PATCH 184/294] release note entry for the `mounted-ssh-ng://` store

---
 doc/manual/src/release-notes/rl-next.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
index dbe2692f9..0e3d8b462 100644
--- a/doc/manual/src/release-notes/rl-next.md
+++ b/doc/manual/src/release-notes/rl-next.md
@@ -1,3 +1,6 @@
 # Release X.Y (202?-??-??)
 
-- Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set.
\ No newline at end of file
+- Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set.
+
+- Introduced the store [`mounted-ssh-ng://`](@docroot@/command-ref/new-cli/nix3-help-stores.md).
+  This store allows full access to a Nix store on a remote machine and additionally requires that the store be mounted in the local filesystem.

From 03c3af1bf97354e281230a82f76d84b2db65db91 Mon Sep 17 00:00:00 2001
From: Matej Urbas <matej.urbas@gmail.com>
Date: Tue, 29 Aug 2023 17:01:09 +0100
Subject: [PATCH 185/294] mounted-ssh-ng store: integration tests

---
 .../build-remote-with-mounted-ssh-ng.sh       | 22 +++++++++++++++++++
 tests/functional/local.mk                     |  1 +
 2 files changed, 23 insertions(+)
 create mode 100644 tests/functional/build-remote-with-mounted-ssh-ng.sh

diff --git a/tests/functional/build-remote-with-mounted-ssh-ng.sh b/tests/functional/build-remote-with-mounted-ssh-ng.sh
new file mode 100644
index 000000000..443acb6ca
--- /dev/null
+++ b/tests/functional/build-remote-with-mounted-ssh-ng.sh
@@ -0,0 +1,22 @@
+source common.sh
+
+requireSandboxSupport
+[[ $busybox =~ busybox ]] || skipTest "no busybox"
+
+enableFeatures mounted-ssh-store
+
+nix build -Lvf simple.nix \
+  --arg busybox $busybox \
+  --out-link $TEST_ROOT/result-from-remote \
+  --store mounted-ssh-ng://localhost
+
+nix build -Lvf simple.nix \
+  --arg busybox $busybox \
+  --out-link $TEST_ROOT/result-from-remote-new-cli \
+  --store 'mounted-ssh-ng://localhost?remote-program=nix daemon'
+
+# This verifies that the out link was actually created and valid. The ability
+# to create out links (permanent gc roots) is the distinguishing feature of
+# the mounted-ssh-ng store.
+cat $TEST_ROOT/result-from-remote/hello | grepQuiet 'Hello World!'
+cat $TEST_ROOT/result-from-remote-new-cli/hello | grepQuiet 'Hello World!'
diff --git a/tests/functional/local.mk b/tests/functional/local.mk
index 21dabca88..8d584142a 100644
--- a/tests/functional/local.mk
+++ b/tests/functional/local.mk
@@ -69,6 +69,7 @@ nix_tests = \
   build-remote-trustless-should-pass-2.sh \
   build-remote-trustless-should-pass-3.sh \
   build-remote-trustless-should-fail-0.sh \
+  build-remote-with-mounted-ssh-ng.sh \
   nar-access.sh \
   pure-eval.sh \
   eval.sh \

From 4e790efade0c3073292ff73be44351f29badd935 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 20 Nov 2023 13:38:52 +0100
Subject: [PATCH 186/294] Use boost::container::small_vector in place of VLAs

---
 boehmgc-traceable_allocator-public.diff | 12 +++++++
 flake.nix                               |  3 ++
 src/libexpr/eval.cc                     | 13 +++++---
 src/libexpr/gc-small-vector.hh          | 42 +++++++++++++++++++++++++
 src/libexpr/primops.cc                  |  9 +++---
 5 files changed, 70 insertions(+), 9 deletions(-)
 create mode 100644 boehmgc-traceable_allocator-public.diff
 create mode 100644 src/libexpr/gc-small-vector.hh

diff --git a/boehmgc-traceable_allocator-public.diff b/boehmgc-traceable_allocator-public.diff
new file mode 100644
index 000000000..903c707a6
--- /dev/null
+++ b/boehmgc-traceable_allocator-public.diff
@@ -0,0 +1,12 @@
+diff --git a/include/gc_allocator.h b/include/gc_allocator.h
+index 597c7f13..587286be 100644
+--- a/include/gc_allocator.h
++++ b/include/gc_allocator.h
+@@ -312,6 +312,7 @@ public:
+ 
+ template<>
+ class traceable_allocator<void> {
++public:
+   typedef size_t      size_type;
+   typedef ptrdiff_t   difference_type;
+   typedef void*       pointer;
diff --git a/flake.nix b/flake.nix
index 9030a74f7..570a099ab 100644
--- a/flake.nix
+++ b/flake.nix
@@ -230,6 +230,9 @@
             }).overrideAttrs(o: {
               patches = (o.patches or []) ++ [
                 ./boehmgc-coroutine-sp-fallback.diff
+
+                # https://github.com/ivmai/bdwgc/pull/586
+                ./boehmgc-traceable_allocator-public.diff
               ];
             })
             )
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 46a49c891..90f04d40a 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -16,6 +16,7 @@
 #include "fs-input-accessor.hh"
 #include "memory-input-accessor.hh"
 #include "signals.hh"
+#include "gc-small-vector.hh"
 
 #include <algorithm>
 #include <chrono>
@@ -31,6 +32,7 @@
 
 #include <sys/resource.h>
 #include <nlohmann/json.hpp>
+#include <boost/container/small_vector.hpp>
 
 #if HAVE_BOEHMGC
 
@@ -1709,7 +1711,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                 /* We have all the arguments, so call the primop with
                    the previous and new arguments. */
 
-                Value * vArgs[arity];
+                Value * vArgs[maxPrimOpArity];
                 auto n = argsDone;
                 for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->primOpApp.left)
                     vArgs[--n] = arg->primOpApp.right;
@@ -1772,11 +1774,11 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v)
     // 4: about 60
     // 5: under 10
     // This excluded attrset lambdas (`{...}:`). Contributions of mixed lambdas appears insignificant at ~150 total.
-    Value * vArgs[args.size()];
+    SmallValueVector<4> vArgs(args.size());
     for (size_t i = 0; i < args.size(); ++i)
         vArgs[i] = args[i]->maybeThunk(state, env);
 
-    state.callFunction(vFun, args.size(), vArgs, v, pos);
+    state.callFunction(vFun, args.size(), vArgs.data(), v, pos);
 }
 
 
@@ -2015,8 +2017,9 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
         return result;
     };
 
-    Value values[es->size()];
-    Value * vTmpP = values;
+    // List of returned strings. References to these Values must NOT be persisted.
+    SmallTemporaryValueVector<conservativeStackReservation> values(es->size());
+    Value * vTmpP = values.data();
 
     for (auto & [i_pos, i] : *es) {
         Value & vTmp = *vTmpP++;
diff --git a/src/libexpr/gc-small-vector.hh b/src/libexpr/gc-small-vector.hh
new file mode 100644
index 000000000..7f4f08fc7
--- /dev/null
+++ b/src/libexpr/gc-small-vector.hh
@@ -0,0 +1,42 @@
+#pragma once
+
+#include <boost/container/small_vector.hpp>
+
+#if HAVE_BOEHMGC
+
+#include <gc/gc.h>
+#include <gc/gc_cpp.h>
+#include <gc/gc_allocator.h>
+
+#endif
+
+namespace nix {
+
+struct Value;
+
+/**
+ * A GC compatible vector that may used a reserved portion of `nItems` on the stack instead of allocating on the heap.
+ */
+#if HAVE_BOEHMGC
+template <typename T, size_t nItems>
+using SmallVector = boost::container::small_vector<T, nItems, traceable_allocator<T>>;
+#else
+template <typename T, size_t nItems>
+using SmallVector = boost::container::small_vector<T, nItems>;
+#endif
+
+/**
+ * A vector of value pointers. See `SmallVector`.
+ */
+template <size_t nItems>
+using SmallValueVector = SmallVector<Value *, nItems>;
+
+/**
+ * A vector of values that must not be referenced after the vector is destroyed.
+ *
+ * See also `SmallValueVector`.
+ */
+template <size_t nItems>
+using SmallTemporaryValueVector = SmallVector<Value, nItems>;
+
+}
\ No newline at end of file
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index a8d44d8b7..54a5da817 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -4,6 +4,7 @@
 #include "eval-inline.hh"
 #include "eval.hh"
 #include "eval-settings.hh"
+#include "gc-small-vector.hh"
 #include "globals.hh"
 #include "json-to-value.hh"
 #include "names.hh"
@@ -2729,7 +2730,7 @@ static void prim_catAttrs(EvalState & state, const PosIdx pos, Value * * args, V
     auto attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.catAttrs"));
     state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.catAttrs");
 
-    Value * res[args[1]->listSize()];
+    SmallValueVector<nonRecursiveStackReservation> res(args[1]->listSize());
     size_t found = 0;
 
     for (auto v2 : args[1]->listItems()) {
@@ -3064,8 +3065,7 @@ static void prim_filter(EvalState & state, const PosIdx pos, Value * * args, Val
 
     state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filter");
 
-    // FIXME: putting this on the stack is risky.
-    Value * vs[args[1]->listSize()];
+    SmallValueVector<nonRecursiveStackReservation> vs(args[1]->listSize());
     size_t k = 0;
 
     bool same = true;
@@ -3454,7 +3454,8 @@ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args,
     state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.concatMap");
     auto nrLists = args[1]->listSize();
 
-    Value lists[nrLists];
+    // List of returned lists before concatenation. References to these Values must NOT be persisted.
+    SmallTemporaryValueVector<conservativeStackReservation> lists(nrLists);
     size_t len = 0;
 
     for (unsigned int n = 0; n < nrLists; ++n) {

From 46131567da96ffac298b9ec54016b37114b0dfd5 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 21 Nov 2023 23:19:25 -0500
Subject: [PATCH 187/294] Add missing `-lrapidcheck` fixing build with shared
 lib

https://github.com/NixOS/nixpkgs/pull/269064 makes rapidcheck be build
as a shared lib, but that broke Nix because the `-lrapidcheck` was
missing. This fixes that (and doesn't break Nix what the library is a
static archive as today).
---
 src/libexpr/tests/local.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/tests/local.mk b/src/libexpr/tests/local.mk
index 6d2a04aaf..7689a03e0 100644
--- a/src/libexpr/tests/local.mk
+++ b/src/libexpr/tests/local.mk
@@ -20,4 +20,4 @@ libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/l
 
 libexpr-tests_LIBS = libstore-tests libutils-tests libexpr libutil libstore libfetchers
 
-libexpr-tests_LDFLAGS := $(GTEST_LIBS) -lgmock
+libexpr-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock

From 61b76f5f34db7f863a6f22bd9083f677b339fcf6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 22 Nov 2023 11:26:12 +0100
Subject: [PATCH 188/294] Apply suggestion

Co-authored-by: John Ericson <git@JohnEricson.me>
---
 src/libfetchers/input-accessor.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc
index 53502c621..8e10cf2e2 100644
--- a/src/libfetchers/input-accessor.cc
+++ b/src/libfetchers/input-accessor.cc
@@ -27,7 +27,7 @@ StorePath InputAccessor::fetchToStore(
             {"path", path.abs()}
         };
         if (auto res = fetchers::getCache()->lookup(*cacheKey)) {
-            StorePath storePath(fetchers::getStrAttr(*res, "storePath"));
+            StorePath storePath{fetchers::getStrAttr(*res, "storePath")};
             if (store->isValidPath(storePath)) {
                 debug("store path cache hit for '%s'", showPath(path));
                 return storePath;

From 5292f364267eb74005a2e06dfe69c0d0dc8bd2a3 Mon Sep 17 00:00:00 2001
From: r-vdp <ramses@well-founded.dev>
Date: Wed, 22 Nov 2023 11:33:25 +0100
Subject: [PATCH 189/294] Fix compile warning due to unused variable binding.

We still need the check, since we don't have narinfo for locally built store paths.
---
 src/nix/path-info.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index 23198a120..080d6bbf1 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -54,7 +54,7 @@ static json pathInfoToJSON(
 
                 jsonObject["closureSize"] = getStoreObjectsTotalSize(store, closure);
 
-                if (auto * narInfo = dynamic_cast<const NarInfo *>(&*info)) {
+                if (dynamic_cast<const NarInfo *>(&*info)) {
                     uint64_t totalDownloadSize = 0;
                     for (auto & p : closure) {
                         auto depInfo = store.queryPathInfo(p);

From b1ab592f28f08da5dc7c060e5c3b19dc66dbc111 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 22 Nov 2023 11:44:02 +0100
Subject: [PATCH 190/294] Use the StorePath-based cache interface

---
 src/libfetchers/input-accessor.cc | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc
index 8e10cf2e2..85dc4609f 100644
--- a/src/libfetchers/input-accessor.cc
+++ b/src/libfetchers/input-accessor.cc
@@ -26,12 +26,9 @@ StorePath InputAccessor::fetchToStore(
             {"method", (uint8_t) method},
             {"path", path.abs()}
         };
-        if (auto res = fetchers::getCache()->lookup(*cacheKey)) {
-            StorePath storePath{fetchers::getStrAttr(*res, "storePath")};
-            if (store->isValidPath(storePath)) {
-                debug("store path cache hit for '%s'", showPath(path));
-                return storePath;
-            }
+        if (auto res = fetchers::getCache()->lookup(store, *cacheKey)) {
+            debug("store path cache hit for '%s'", showPath(path));
+            return res->second;
         }
     } else
         debug("source path '%s' is uncacheable", showPath(path));
@@ -51,9 +48,7 @@ StorePath InputAccessor::fetchToStore(
         : store->addToStoreFromDump(*source, name, method, htSHA256, repair);
 
     if (cacheKey)
-        fetchers::getCache()->upsert(
-            *cacheKey,
-            fetchers::Attrs{{"storePath", std::string(storePath.to_string())}});
+        fetchers::getCache()->add(store, *cacheKey, {}, storePath, true);
 
     return storePath;
 }

From 2ce8c9650b3e714f28d8685e48996141cba2df2c Mon Sep 17 00:00:00 2001
From: Philipp Schuster <phip1611@gmail.com>
Date: Thu, 23 Nov 2023 22:02:20 +0100
Subject: [PATCH 191/294] doc: primops: add more info for foldl (#9254)

* doc: primops: add more info for foldl

From the existing doc it is not obvious whether the first or the
second argument is the accumulator. This is however relevant to
know, as for certain scenarios, this might change the behavior.

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
---
 src/libexpr/primops.cc | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index a8d44d8b7..7c0561413 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -3179,9 +3179,16 @@ static RegisterPrimOp primop_foldlStrict({
     .doc = R"(
       Reduce a list by applying a binary operator, from left to right,
       e.g. `foldl' op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2)
-      ...`. For example, `foldl' (x: y: x + y) 0 [1 2 3]` evaluates to 6.
-      The return value of each application of `op` is evaluated immediately,
-      even for intermediate values.
+      ...`.
+
+      For example, `foldl' (acc: elem: acc + elem) 0 [1 2 3]` evaluates
+      to `6` and `foldl' (acc: elem: { "${elem}" = elem; } // acc) {}
+      ["a" "b"]` evaluates to `{ a = "a"; b = "b"; }`.
+
+      The first argument of `op` is the accumulator wheres the second
+      argument is the current element being processed. The return value
+      of each application of `op` is evaluated immediately, even for
+      intermediate values.
     )",
     .fun = prim_foldlStrict,
 });

From 5be0e6b314c216b0b51499fc488ca08272297469 Mon Sep 17 00:00:00 2001
From: Philipp Schuster <phip1611@gmail.com>
Date: Fri, 24 Nov 2023 10:50:01 +0100
Subject: [PATCH 192/294] doc: primops: fix typo

---
 src/libexpr/primops.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 7c0561413..ba735b435 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -3185,7 +3185,7 @@ static RegisterPrimOp primop_foldlStrict({
       to `6` and `foldl' (acc: elem: { "${elem}" = elem; } // acc) {}
       ["a" "b"]` evaluates to `{ a = "a"; b = "b"; }`.
 
-      The first argument of `op` is the accumulator wheres the second
+      The first argument of `op` is the accumulator whereas the second
       argument is the current element being processed. The return value
       of each application of `op` is evaluated immediately, even for
       intermediate values.

From 6a94755b1240be654cadb463a9f528eeccf3787c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <theophane.hufschmitt@tweag.io>
Date: Fri, 24 Nov 2023 11:45:37 +0100
Subject: [PATCH 193/294] Allow user input in `git commit`

We occasionnally commit to git repositories (like with `nix flake update --commit-lock-file`).
This shells out to `git commit`, which might wait for user input (for a signing key passphrase for instance).
Disable the progress bar while this is running to make sure that the
user can enter it.
---
 src/libfetchers/git.cc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 2c5b70f53..8cd74057c 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -11,6 +11,8 @@
 #include "fs-input-accessor.hh"
 #include "mounted-input-accessor.hh"
 #include "git-utils.hh"
+#include "logging.hh"
+#include "finally.hh"
 
 #include "fetch-settings.hh"
 
@@ -314,6 +316,9 @@ struct GitInputScheme : InputScheme
         runProgram("git", true,
             { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) });
 
+        // Pause the logger to allow for user input (such as a gpg passphrase) in `git commit`
+        logger->pause();
+        Finally restoreLogger([]() { logger->resume(); });
         if (commitMsg)
             runProgram("git", true,
                 { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "commit", std::string(path.rel()), "-m", *commitMsg });

From b7982372d234b1fd15bab01d09093471c1870bb4 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 19 Nov 2023 15:05:21 +0100
Subject: [PATCH 194/294] Compile hand-written release notes with changelog-d

---
 .gitignore                             |  1 +
 doc/manual/local.mk                    |  5 +++-
 doc/manual/rl-next/config              |  2 ++
 doc/manual/src/contributing/hacking.md | 38 ++++++++++++++++++++++++++
 flake.nix                              | 11 +++++++-
 5 files changed, 55 insertions(+), 2 deletions(-)
 create mode 100644 doc/manual/rl-next/config

diff --git a/.gitignore b/.gitignore
index 04d96ca2c..38e7e2b5a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -27,6 +27,7 @@ perl/Makefile.config
 /doc/manual/src/contributing/experimental-feature-descriptions.md
 /doc/manual/src/language/builtins.md
 /doc/manual/src/language/builtin-constants.md
+/doc/manual/src/release-notes/rl-next.md
 
 # /scripts/
 /scripts/nix-profile.sh
diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index db3daf252..74a4103b3 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -144,6 +144,9 @@ $(d)/language.json: $(bindir)/nix
 	$(trace-gen) $(dummy-env) $(bindir)/nix __dump-language > $@.tmp
 	@mv $@.tmp $@
 
+$(d)/src/release-notes/rl-next.md: $(d)/rl-next/*
+	$(trace-gen) changelog-d doc/manual/rl-next > $@
+
 # Generate the HTML manual.
 .PHONY: manual-html
 manual-html: $(docdir)/manual/index.html
@@ -177,7 +180,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli
 # `@docroot@` is to be preserved for documenting the mechanism
 # FIXME: maybe contributing guides should live right next to the code
 # instead of in the manual
-$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md
+$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md
 	$(trace-gen) \
 		tmp="$$(mktemp -d)"; \
 		cp -r doc/manual "$$tmp"; \
diff --git a/doc/manual/rl-next/config b/doc/manual/rl-next/config
new file mode 100644
index 000000000..b3c2e868f
--- /dev/null
+++ b/doc/manual/rl-next/config
@@ -0,0 +1,2 @@
+organization: NixOS
+repository: nix
diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md
index fe08ceb94..855900d7a 100644
--- a/doc/manual/src/contributing/hacking.md
+++ b/doc/manual/src/contributing/hacking.md
@@ -220,3 +220,41 @@ Configure your editor to use the `clangd` from the shell, either by running it i
 > For some editors (e.g. Visual Studio Code), you may need to install a [special extension](https://open-vsx.org/extension/llvm-vs-code-extensions/vscode-clangd) for the editor to interact with `clangd`.
 > Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim).
 > Editor-specific setup is typically opinionated, so we will not cover it here in more detail.
+
+## Add a release note
+
+`doc/manual/rl-next` contains release notes entries for all unreleased changes.
+
+User-visible changes should come with a release note.
+
+### Add an entry
+
+Here's what a complete entry looks like. The file name is not incorporated in the document.
+
+```
+synopsis: Basically a title
+issues: #1234
+prs: #1238
+description: {
+
+Here's one or more paragraphs that describe the change.
+
+- It's markdown
+- Add references to the manual using @docroot@
+
+}
+```
+
+Significant changes should add the following header, which moves them to the top.
+
+```
+significance: significant
+```
+
+<!-- Keep an eye on https://codeberg.org/fgaz/changelog-d/issues/1 -->
+See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog).
+
+### Build process
+
+Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`.
+Non-releases build the release notes on the fly.
diff --git a/flake.nix b/flake.nix
index 9030a74f7..ca37948a7 100644
--- a/flake.nix
+++ b/flake.nix
@@ -173,6 +173,10 @@
           "--enable-internal-api-docs"
         ];
 
+        # TODO: after backport of https://github.com/NixOS/nixpkgs/pull/268487, remove `haskellPackages.` -
+        #       vastly improves output closure, and adds shell completions
+        changelog-d = pkgs.buildPackages.haskellPackages.changelog-d;
+
         nativeBuildDeps =
           [
             buildPackages.bison
@@ -190,7 +194,10 @@
             buildPackages.jq # Also for custom mdBook preprocessor.
             buildPackages.openssh # only needed for tests (ssh-keygen)
           ]
-          ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)];
+          ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)]
+          # Official releases don't have rl-next, so we don't need to compile a changelog
+          ++ lib.optional (!officialRelease) changelog-d
+          ;
 
         buildDeps =
           [ curl
@@ -727,6 +734,8 @@
               ++ lib.optional
                 (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform)
                 pkgs.buildPackages.clang-tools
+              # We want changelog-d in the shell even if it's an official release
+              ++ lib.optional officialRelease changelog-d
               ;
 
             buildInputs = buildDeps ++ propagatedDeps

From b26038c517ed10feae751ad6733244c00b715d34 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 19 Nov 2023 16:08:34 +0100
Subject: [PATCH 195/294] doc: Rename 2X.XX to "Upcoming release", and only
 generate if applicable

---
 .gitignore                   |  1 +
 doc/manual/local.mk          | 13 +++++++++++--
 doc/manual/src/SUMMARY.md.in |  2 +-
 3 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/.gitignore b/.gitignore
index 38e7e2b5a..5c359d7a0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -21,6 +21,7 @@ perl/Makefile.config
 /doc/manual/language.json
 /doc/manual/xp-features.json
 /doc/manual/src/SUMMARY.md
+/doc/manual/src/SUMMARY-rl-next.md
 /doc/manual/src/command-ref/new-cli
 /doc/manual/src/command-ref/conf-file.md
 /doc/manual/src/command-ref/experimental-features-shortlist.md
diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index 74a4103b3..265a4649d 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -92,7 +92,7 @@ $(d)/nix-profiles.5: $(d)/src/command-ref/files/profiles.md
 	$(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@
 	@rm $^.tmp
 
-$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md
+$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md
 	@cp $< $@
 	@$(call process-includes,$@,$@)
 
@@ -144,9 +144,18 @@ $(d)/language.json: $(bindir)/nix
 	$(trace-gen) $(dummy-env) $(bindir)/nix __dump-language > $@.tmp
 	@mv $@.tmp $@
 
-$(d)/src/release-notes/rl-next.md: $(d)/rl-next/*
+# Generate "Upcoming release" notes (or clear it and remove from menu)
+$(d)/src/release-notes/rl-next.md: $(d)/rl-next $(d)/rl-next/*
 	$(trace-gen) changelog-d doc/manual/rl-next > $@
 
+$(d)/src/SUMMARY-rl-next.md: $(d)/src/release-notes/rl-next.md
+	$(trace-gen) true
+	@if [ -s $< ]; then \
+		echo '  - [Upcoming release](release-notes/rl-next.md)' > $@; \
+	else \
+	  true > $@; \
+	fi
+
 # Generate the HTML manual.
 .PHONY: manual-html
 manual-html: $(docdir)/manual/index.html
diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in
index 8dc464abd..8e7b4eeab 100644
--- a/doc/manual/src/SUMMARY.md.in
+++ b/doc/manual/src/SUMMARY.md.in
@@ -114,7 +114,7 @@
   - [CLI guideline](contributing/cli-guideline.md)
   - [C++ style guide](contributing/cxx.md)
 - [Release Notes](release-notes/release-notes.md)
-  - [Release X.Y (202?-??-??)](release-notes/rl-next.md)
+{{#include ./SUMMARY-rl-next.md}}
   - [Release 2.19 (2023-11-17)](release-notes/rl-2.19.md)
   - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md)
   - [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md)

From 9aa63f70d7d861ba74764188410c0add730cd48d Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Fri, 24 Nov 2023 15:32:02 +0100
Subject: [PATCH 196/294] fricklerhandwerk: subscribe to documentation changes
 (#9422)

* fricklerhandwerk: subscribe to documentation changes
---
 .github/CODEOWNERS | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index ab5908649..39d595199 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -14,5 +14,12 @@
 /doc @fricklerhandwerk
 *.md @fricklerhandwerk
 
+# Documentation of built-in functions
+src/libexpr/primops.cc @fricklerhandwerk @roberth
+# Documentation on experimental features
+src/libutil/experimental-features.cc @fricklerhandwerk
+# Documentation on configuration settings
+src/libstore/globals.hh @fricklerhandwerk
+
 # Libstore layer
 /src/libstore @thufschmitt

From 2a538c571b13877fa426f2cff2749cf17d140216 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 19 Nov 2023 16:49:11 +0100
Subject: [PATCH 197/294] Add scripts/release-notes

---
 maintainers/release-process.md |  45 ++++------
 scripts/release-notes          | 148 +++++++++++++++++++++++++++++++++
 2 files changed, 166 insertions(+), 27 deletions(-)
 create mode 100755 scripts/release-notes

diff --git a/maintainers/release-process.md b/maintainers/release-process.md
index d85266b81..e542d8be5 100644
--- a/maintainers/release-process.md
+++ b/maintainers/release-process.md
@@ -24,34 +24,23 @@ release:
 * In a checkout of the Nix repo, make sure you're on `master` and run
   `git pull`.
 
-* Move the contents of `doc/manual/src/release-notes/rl-next.md`
-  (except the first line) to
-  `doc/manual/src/release-notes/rl-$VERSION.md` (where `$VERSION` is
-  the contents of `.version` *without* the patch level, e.g. `2.12`
-  rather than `2.12.0`).
-
-* Add a header to `doc/manual/src/release-notes/rl-$VERSION.md` like
-
-  ```
-  # Release 2.12 (2022-12-06)
-  ```
-
-* Proof-read / edit / rearrange the release notes. Breaking changes
-  and highlights should go to the top.
-
-* Add a link to the release notes to `doc/manual/src/SUMMARY.md.in`
-  (*not* `SUMMARY.md`), e.g.
-
-  ```
-  - [Release 2.12 (2022-12-06)](release-notes/rl-2.12.md)
-  ```
-
-* Run
+* Compile the release notes by running
 
   ```console
   $ git checkout -b release-notes
-  $ git add doc/manual/src/release-notes/rl-$VERSION.md
-  $ git commit -a -m 'Release notes'
+  $ VERSION=X.YY ./scripts/release-notes
+  ```
+
+  where `X.YY` is *without* the patch level, e.g. `2.12` rather than ~~`2.12.0`~~.
+
+  A commit is created.
+
+* Proof-read / edit / rearrange the release notes if needed. Breaking changes
+  and highlights should go to the top.
+
+* Push.
+
+  ```console
   $ git push --set-upstream $REMOTE release-notes
   ```
 
@@ -67,15 +56,17 @@ release:
   $ git checkout -b $VERSION-maintenance
   ```
 
-* Mark the release as stable:
+* Mark the release as official:
 
   ```console
-  $ git cherry-pick f673551e71942a52b6d7ae66af8b67140904a76a
+  $ sed -e 's/officialRelease = false;/officialRelease = true;/' -i flake.nix
   ```
 
   This removes the link to `rl-next.md` from the manual and sets
   `officialRelease = true` in `flake.nix`.
 
+* Commit
+
 * Push the release branch:
 
   ```console
diff --git a/scripts/release-notes b/scripts/release-notes
new file mode 100755
index 000000000..e5ee39d11
--- /dev/null
+++ b/scripts/release-notes
@@ -0,0 +1,148 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash ../shell.nix -I nixpkgs=channel:nixos-unstable-small
+#                                   ^^^^^^^
+#                Only used for bash. shell.nix goes to the flake.
+
+# --- CONFIGURATION ---
+
+# This does double duty for
+# - including rl-next
+# - marking where to insert new links (right after)
+SUMMARY_MARKER_LINE='{{#include ./SUMMARY-rl-next.md}}'
+
+# --- LIB ---
+
+log() {
+  echo 1>&2 "release-notes:" "$@"
+}
+logcmd() {
+  local cmd="$1"
+  shift
+  logcmd2 "$cmd" "${*@Q}" "$cmd" "$@"
+}
+logcmd2() {
+  local fakecmd="$1"
+  local fakeargs="$2"
+  shift
+  shift
+  printf 1>&2 "release-notes: \033[34;1m$fakecmd\033[0m "
+  echo "$fakeargs" 1>&2
+  "$@"
+}
+die() {
+  # ANSI red
+  printf 1>&2 "release-notes: \033[31;1merror:\033[0m"
+  echo 1>&2 "" "$@"
+  exit 1
+}
+confirm() {
+  local answer
+  echo 1>&2 "$@" "[y/n]"
+  read -r answer
+  case "$answer" in
+    y|Y|yes|Yes|YES)
+      return 0
+      ;;
+    n|N|no|No|NO)
+      return 1
+      ;;
+    *)
+      echo 1>&2 "please answer y or n"
+      confirm "$@"
+      ;;
+  esac
+}
+report_done() {
+  logcmd2 "git" "show" git -c pager.show=false show
+  printf 1>&2 "release-notes: \033[32;1mdone\033[0m\n"
+}
+
+# --- PARSE ARGS ---
+
+if [[ $# -gt 0 ]]; then
+  die "Release notes takes no arguments, but make sure to set VERSION."
+fi
+
+# --- CHECKS ---
+
+if [[ ! -e flake.nix ]] || [[ ! -e .git ]]; then
+  die "must run in repo root"
+  exit 1
+fi
+
+# repo must be clean
+if ! git diff --quiet; then
+  die "repo is dirty, please commit or stash changes"
+fi
+
+if ! git diff --quiet --cached; then
+  die "repo has staged changes, please commit or stash them"
+fi
+
+if ! grep "$SUMMARY_MARKER_LINE" doc/manual/src/SUMMARY.md.in >/dev/null; then
+  # would have been nice to catch this early, but won't be worth the extra infra
+  die "SUMMARY.md.in is missing the marker line '$SUMMARY_MARKER_LINE', which would be used for inserting a new release notes page. Please fix the script."
+fi
+
+if [[ ! -n "${VERSION:-}" ]]; then
+  die "please set the VERSION environment variable before invoking this script"
+  exit 1
+fi
+
+case "$VERSION" in
+  # FIXME: accepts "." without any real digits
+  [[:digit:]]*.[[:digit:]]*)
+    ;;
+  *)
+    die "VERSION must be MAJOR.MINOR, where each is a number, e.g. 2.20 (VERSION was set to $VERSION)"
+    ;;
+esac
+
+# --- DEFAULTS ---
+
+if [[ ! -n "${DATE:-}" ]]; then
+  DATE="$(date +%Y-%m-%d)"
+  log "DATE not set, using $DATE"
+fi
+
+case "$DATE" in
+  [[:digit:]]*-[[:digit:]]*-[[:digit:]]*)
+    ;;
+  *)
+    die "DATE must be YYYY-MM-DD, e.g. 2021-12-31 (DATE was set to $DATE)"
+    ;;
+esac
+
+# --- DO THE WORK ---
+
+basename=rl-$VERSION.md
+file=doc/manual/src/release-notes/$basename
+title="Release $VERSION ($DATE)"
+
+(
+  # TODO add minor number, and append?
+  echo "# $title"
+  echo
+  changelog-d doc/manual/rl-next | sed -e 's/ *$//'
+) > $file
+
+log "Wrote $file"
+
+NEW_SUMMARY_LINE="  - [$title](release-notes/$basename)"
+
+# find the marker line, insert new link after it
+escaped_marker="$(echo "$SUMMARY_MARKER_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')"
+escaped_line="$(echo "$NEW_SUMMARY_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')"
+logcmd sed -i -e "/$escaped_marker/a $escaped_line" doc/manual/src/SUMMARY.md.in
+
+for f in doc/manual/rl-next/*.md; do
+  if [[ config != "$(basename $f)" ]]; then
+    logcmd git rm $f
+  fi
+done
+
+logcmd git add $file doc/manual/src/SUMMARY.md.in
+logcmd git status
+logcmd git commit -m "release notes: $VERSION"
+
+report_done

From b1ea30f21d24df9afc4eb1635eee9a080e4f81f3 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 19 Nov 2023 18:13:59 +0100
Subject: [PATCH 198/294] scripts/release-notes: Support patch releases

This also fixes the broken case statement, which has globs, not regexes.
---
 maintainers/release-process.md | 24 +++++++++++++
 scripts/release-notes          | 65 ++++++++++++++++++++++++----------
 2 files changed, 71 insertions(+), 18 deletions(-)

diff --git a/maintainers/release-process.md b/maintainers/release-process.md
index e542d8be5..b1259a9bd 100644
--- a/maintainers/release-process.md
+++ b/maintainers/release-process.md
@@ -150,6 +150,30 @@ release:
 
 ## Creating a point release
 
+* Checkout.
+
+  ```console
+  $ git checkout XX.YY-maintenance
+  ```
+
+* Determine the next patch version.
+
+  ```console
+  $ export VERSION=XX.YY.ZZ
+  ```
+
+* Update release notes.
+
+  ```console
+  $ ./scripts/release-notes
+  ```
+
+* Push.
+
+  ```console
+  $ git push
+  ```
+
 * Wait for the desired evaluation of the maintenance jobset to finish
   building.
 
diff --git a/scripts/release-notes b/scripts/release-notes
index e5ee39d11..43f20a547 100755
--- a/scripts/release-notes
+++ b/scripts/release-notes
@@ -89,14 +89,40 @@ if [[ ! -n "${VERSION:-}" ]]; then
   exit 1
 fi
 
-case "$VERSION" in
-  # FIXME: accepts "." without any real digits
-  [[:digit:]]*.[[:digit:]]*)
-    ;;
-  *)
-    die "VERSION must be MAJOR.MINOR, where each is a number, e.g. 2.20 (VERSION was set to $VERSION)"
-    ;;
-esac
+# mutate/initialize:
+# VERSION:      MAJOR.MINOR
+# FULL_VERSION: MAJOR.MINOR.PATCH
+# IS_PATCH:     true if this is a patch release; append instead of create
+if grep -E '^[0-9]+\.[0-9]+$' <<< "$VERSION" >/dev/null; then
+  log 'is minor'
+  IS_PATCH=false
+  FULL_VERSION="$VERSION.0"
+elif grep -E '^[0-9]+\.[0-9]+\.0$' <<< "$VERSION" >/dev/null; then
+  log 'is minor (.0)'
+  IS_PATCH=false
+  FULL_VERSION="$VERSION"
+  VERSION="$(echo "$VERSION" | sed -e 's/\.0$//')"
+elif grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' <<< "$VERSION" >/dev/null; then
+  log 'is patch'
+  IS_PATCH=true
+  FULL_VERSION="$VERSION"
+  VERSION="$(echo "$VERSION" | sed -e 's/\.[0-9]*$//')"
+else
+  die "VERSION must be MAJOR.MINOR[.PATCH], where each is a number, e.g. 2.20 or 2.20.1 (VERSION was set to $VERSION)"
+fi
+
+log "VERSION=$VERSION"
+log "FULL_VERSION=$FULL_VERSION"
+log "IS_PATCH=$IS_PATCH"
+
+basename=rl-$VERSION.md
+file=doc/manual/src/release-notes/$basename
+
+if ! $IS_PATCH; then
+  if [[ -e $file ]]; then
+    die "release notes file $file already exists. If you'd like to make a minor release, pass a patch version, e.g. 2.20.1"
+  fi
+fi
 
 # --- DEFAULTS ---
 
@@ -106,7 +132,7 @@ if [[ ! -n "${DATE:-}" ]]; then
 fi
 
 case "$DATE" in
-  [[:digit:]]*-[[:digit:]]*-[[:digit:]]*)
+  [0-9]*-[0-9]*-[0-9]*)
     ;;
   *)
     die "DATE must be YYYY-MM-DD, e.g. 2021-12-31 (DATE was set to $DATE)"
@@ -115,25 +141,28 @@ esac
 
 # --- DO THE WORK ---
 
-basename=rl-$VERSION.md
-file=doc/manual/src/release-notes/$basename
+# menu
 title="Release $VERSION ($DATE)"
+# section on page
+section_title="Release $FULL_VERSION ($DATE)"
 
 (
   # TODO add minor number, and append?
-  echo "# $title"
+  echo "# $section_title"
   echo
   changelog-d doc/manual/rl-next | sed -e 's/ *$//'
-) > $file
+) | tee -a $file
 
 log "Wrote $file"
 
-NEW_SUMMARY_LINE="  - [$title](release-notes/$basename)"
+if ! $IS_PATCH; then
+  NEW_SUMMARY_LINE="  - [$title](release-notes/$basename)"
 
-# find the marker line, insert new link after it
-escaped_marker="$(echo "$SUMMARY_MARKER_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')"
-escaped_line="$(echo "$NEW_SUMMARY_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')"
-logcmd sed -i -e "/$escaped_marker/a $escaped_line" doc/manual/src/SUMMARY.md.in
+  # find the marker line, insert new link after it
+  escaped_marker="$(echo "$SUMMARY_MARKER_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')"
+  escaped_line="$(echo "$NEW_SUMMARY_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')"
+  logcmd sed -i -e "/$escaped_marker/a $escaped_line" doc/manual/src/SUMMARY.md.in
+fi
 
 for f in doc/manual/rl-next/*.md; do
   if [[ config != "$(basename $f)" ]]; then

From 7c4ee5c8135fae65602791f0b89d0dbae7e94f3e Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 19 Nov 2023 18:36:50 +0100
Subject: [PATCH 199/294] scripts/release-notes: Avoid mutating variables

---
 scripts/release-notes | 32 +++++++++++++++++---------------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/scripts/release-notes b/scripts/release-notes
index 43f20a547..2e1be64a4 100755
--- a/scripts/release-notes
+++ b/scripts/release-notes
@@ -89,33 +89,35 @@ if [[ ! -n "${VERSION:-}" ]]; then
   exit 1
 fi
 
-# mutate/initialize:
-# VERSION:      MAJOR.MINOR
-# FULL_VERSION: MAJOR.MINOR.PATCH
-# IS_PATCH:     true if this is a patch release; append instead of create
+# version_major_minor:      MAJOR.MINOR
+# version_full:             MAJOR.MINOR.PATCH
+# IS_PATCH:                 true if this is a patch release; append instead of create
 if grep -E '^[0-9]+\.[0-9]+$' <<< "$VERSION" >/dev/null; then
   log 'is minor'
   IS_PATCH=false
-  FULL_VERSION="$VERSION.0"
+  version_full="$VERSION.0"
+  version_major_minor="$VERSION"
 elif grep -E '^[0-9]+\.[0-9]+\.0$' <<< "$VERSION" >/dev/null; then
   log 'is minor (.0)'
   IS_PATCH=false
-  FULL_VERSION="$VERSION"
-  VERSION="$(echo "$VERSION" | sed -e 's/\.0$//')"
+  version_full="$VERSION"
+  version_major_minor="$(echo "$VERSION" | sed -e 's/\.0$//')"
 elif grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' <<< "$VERSION" >/dev/null; then
   log 'is patch'
   IS_PATCH=true
-  FULL_VERSION="$VERSION"
-  VERSION="$(echo "$VERSION" | sed -e 's/\.[0-9]*$//')"
+  version_full="$VERSION"
+  version_major_minor="$(echo "$VERSION" | sed -e 's/\.[0-9]*$//')"
 else
   die "VERSION must be MAJOR.MINOR[.PATCH], where each is a number, e.g. 2.20 or 2.20.1 (VERSION was set to $VERSION)"
 fi
 
-log "VERSION=$VERSION"
-log "FULL_VERSION=$FULL_VERSION"
+unset VERSION
+
+log "version_major_minor=$version_major_minor"
+log "version_full=$version_full"
 log "IS_PATCH=$IS_PATCH"
 
-basename=rl-$VERSION.md
+basename=rl-${version_major_minor}.md
 file=doc/manual/src/release-notes/$basename
 
 if ! $IS_PATCH; then
@@ -142,9 +144,9 @@ esac
 # --- DO THE WORK ---
 
 # menu
-title="Release $VERSION ($DATE)"
+title="Release $version_major_minor ($DATE)"
 # section on page
-section_title="Release $FULL_VERSION ($DATE)"
+section_title="Release $version_full ($DATE)"
 
 (
   # TODO add minor number, and append?
@@ -172,6 +174,6 @@ done
 
 logcmd git add $file doc/manual/src/SUMMARY.md.in
 logcmd git status
-logcmd git commit -m "release notes: $VERSION"
+logcmd git commit -m "release notes: $version_full"
 
 report_done

From 6971c4adc06d574cbe1e9ab6da19814e11e2ba6c Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 19 Nov 2023 19:08:42 +0100
Subject: [PATCH 200/294] maintainers/release-notes <- scripts/release-notes

---
 {scripts => maintainers}/release-notes | 0
 maintainers/release-process.md         | 4 ++--
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename {scripts => maintainers}/release-notes (100%)

diff --git a/scripts/release-notes b/maintainers/release-notes
similarity index 100%
rename from scripts/release-notes
rename to maintainers/release-notes
diff --git a/maintainers/release-process.md b/maintainers/release-process.md
index b1259a9bd..db8b064a5 100644
--- a/maintainers/release-process.md
+++ b/maintainers/release-process.md
@@ -28,7 +28,7 @@ release:
 
   ```console
   $ git checkout -b release-notes
-  $ VERSION=X.YY ./scripts/release-notes
+  $ VERSION=X.YY ./maintainers/release-notes
   ```
 
   where `X.YY` is *without* the patch level, e.g. `2.12` rather than ~~`2.12.0`~~.
@@ -165,7 +165,7 @@ release:
 * Update release notes.
 
   ```console
-  $ ./scripts/release-notes
+  $ ./maintainers/release-notes
   ```
 
 * Push.

From 857f9168f7b48aa491052f24fb571c21398f9826 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 24 Nov 2023 15:27:54 +0100
Subject: [PATCH 201/294] Migrate rl-next.md to doc/manual/rl-next directory

---
 doc/manual/rl-next/mounted-ssh-store.md     | 9 +++++++++
 doc/manual/rl-next/nix-env-json-drv-path.md | 9 +++++++++
 doc/manual/src/release-notes/rl-next.md     | 6 ------
 3 files changed, 18 insertions(+), 6 deletions(-)
 create mode 100644 doc/manual/rl-next/mounted-ssh-store.md
 create mode 100644 doc/manual/rl-next/nix-env-json-drv-path.md
 delete mode 100644 doc/manual/src/release-notes/rl-next.md

diff --git a/doc/manual/rl-next/mounted-ssh-store.md b/doc/manual/rl-next/mounted-ssh-store.md
new file mode 100644
index 000000000..39fac5283
--- /dev/null
+++ b/doc/manual/rl-next/mounted-ssh-store.md
@@ -0,0 +1,9 @@
+synopsis: Mounted SSH Store
+issues: #7890
+prs: #7912
+description: {
+
+Introduced the store [`mounted-ssh-ng://`](@docroot@/command-ref/new-cli/nix3-help-stores.md).
+This store allows full access to a Nix store on a remote machine and additionally requires that the store be mounted in the local filesystem.
+
+}
diff --git a/doc/manual/rl-next/nix-env-json-drv-path.md b/doc/manual/rl-next/nix-env-json-drv-path.md
new file mode 100644
index 000000000..fbe2b67d8
--- /dev/null
+++ b/doc/manual/rl-next/nix-env-json-drv-path.md
@@ -0,0 +1,9 @@
+synopsis: Fix `nix-env --query --drv-path --json`
+prs: #9257
+description: {
+
+Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set.
+
+}
+
+
diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
deleted file mode 100644
index 0e3d8b462..000000000
--- a/doc/manual/src/release-notes/rl-next.md
+++ /dev/null
@@ -1,6 +0,0 @@
-# Release X.Y (202?-??-??)
-
-- Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set.
-
-- Introduced the store [`mounted-ssh-ng://`](@docroot@/command-ref/new-cli/nix3-help-stores.md).
-  This store allows full access to a Nix store on a remote machine and additionally requires that the store be mounted in the local filesystem.

From 54b684765519dee1e74dca71605f8e7f6c8b0a25 Mon Sep 17 00:00:00 2001
From: ivan770 <ivan@ivan770.me>
Date: Fri, 24 Nov 2023 11:17:35 -0500
Subject: [PATCH 202/294] doc: fix machine-specific capabilities leaking

---
 src/libstore/globals.hh   | 14 +++++++++++---
 src/libstore/store-api.hh |  5 ++++-
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 27caf42c4..838d2aba2 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -183,7 +183,9 @@ public:
           command line switch and defaults to `1`. The value `0` means that
           the builder should use all available CPU cores in the system.
         )",
-        {"build-cores"}, false};
+        {"build-cores"},
+        // Don't document the machine-specific default value
+        false};
 
     /**
      * Read-only mode.  Don't copy stuff to the store, don't change
@@ -699,7 +701,10 @@ public:
 
           Build systems will usually detect the target platform to be the current physical system and therefore produce machine code incompatible with what may be intended in the derivation.
           You should design your derivation's `builder` accordingly and cross-check the results when using this option against natively-built versions of your derivation.
-        )", {}, false};
+        )",
+        {},
+        // Don't document the machine-specific default value
+        false};
 
     Setting<StringSet> systemFeatures{
         this,
@@ -744,7 +749,10 @@ public:
             [nspawn]: https://github.com/NixOS/nix/blob/67bcb99700a0da1395fa063d7c6586740b304598/tests/systemd-nspawn.nix.
 
             Included by default on Linux if the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting is enabled.
-        )", {}, false};
+        )",
+        {},
+        // Don't document the machine-specific default value
+        false};
 
     Setting<Strings> substituters{
         this,
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 32ad2aa44..8b6bf9aed 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -167,7 +167,10 @@ struct StoreConfig : public Config
           Optional [system features](@docroot@/command-ref/conf-file.md#conf-system-features) available on the system this store uses to build derivations.
 
           Example: `"kvm"`
-        )" };
+        )",
+        {},
+        // Don't document the machine-specific default value
+        false};
 };
 
 class Store : public std::enable_shared_from_this<Store>, public virtual StoreConfig

From d2f5e263e3c095dfe9d874387665b88c4bfff6f1 Mon Sep 17 00:00:00 2001
From: Yorick van Pelt <yorick@yorickvanpelt.nl>
Date: Mon, 6 Feb 2023 16:36:57 +0100
Subject: [PATCH 203/294] Switch from std::regex to boost::regex

---
 flake.nix              | 24 +++++++++++++++++-------
 src/libexpr/local.mk   |  2 +-
 src/libexpr/primops.cc | 35 +++++++++++++++++++++++------------
 3 files changed, 41 insertions(+), 20 deletions(-)

diff --git a/flake.nix b/flake.nix
index 9030a74f7..1f7e7b2af 100644
--- a/flake.nix
+++ b/flake.nix
@@ -154,7 +154,7 @@
 
         configureFlags =
           lib.optionals stdenv.isLinux [
-            "--with-boost=${boost}/lib"
+            "--with-boost=${boost-nix}/lib"
             "--with-sandbox-shell=${sh}/bin/busybox"
           ]
           ++ lib.optionals (stdenv.isLinux && !(isStatic && stdenv.system == "aarch64-linux")) [
@@ -202,7 +202,7 @@
               version = libgit2.lastModifiedDate;
               cmakeFlags = (attrs.cmakeFlags or []) ++ ["-DUSE_SSH=exec"];
             }))
-            boost
+            boost-nix
             lowdown-nix
             libsodium
           ]
@@ -423,14 +423,14 @@
 
             propagatedBuildInputs = propagatedDeps;
 
-            disallowedReferences = [ boost ];
+            disallowedReferences = [ boost-nix ];
 
             preConfigure = lib.optionalString (! currentStdenv.hostPlatform.isStatic)
               ''
                 # Copy libboost_context so we don't get all of Boost in our closure.
                 # https://github.com/NixOS/nixpkgs/issues/45462
                 mkdir -p $out/lib
-                cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
+                cp -pd ${boost-nix}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib
                 rm -f $out/lib/*.a
                 ${lib.optionalString currentStdenv.hostPlatform.isLinux ''
                   chmod u+w $out/lib/*.so.*
@@ -440,9 +440,9 @@
                   for LIB in $out/lib/*.dylib; do
                     chmod u+w $LIB
                     install_name_tool -id $LIB $LIB
-                    install_name_tool -delete_rpath ${boost}/lib/ $LIB || true
+                    install_name_tool -delete_rpath ${boost-nix}/lib/ $LIB || true
                   done
-                  install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib
+                  install_name_tool -change ${boost-nix}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib
                 ''}
               '';
 
@@ -470,9 +470,13 @@
               ''}
               ${lib.optionalString currentStdenv.isDarwin ''
               install_name_tool \
-                -change ${boost}/lib/libboost_context.dylib \
+                -change ${boost-nix}/lib/libboost_context.dylib \
                 $out/lib/libboost_context.dylib \
                 $out/lib/libnixutil.dylib
+              install_name_tool \
+                -change ${boost-nix}/lib/libboost_regex.dylib \
+                $out/lib/libboost_regex.dylib \
+                $out/lib/libnixexpr.dylib
               ''}
             '';
 
@@ -495,6 +499,12 @@
             meta.mainProgram = "nix";
           });
 
+          boost-nix = final.boost.override {
+            # enableIcu arg is not yet supported
+            # but will be with next nixpkgs update
+            enableIcu = false;
+          };
+
           lowdown-nix = with final; currentStdenv.mkDerivation rec {
             name = "lowdown-0.9.0";
 
diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk
index ed7bf9490..946059339 100644
--- a/src/libexpr/local.mk
+++ b/src/libexpr/local.mk
@@ -16,7 +16,7 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib
 
 libexpr_LIBS = libutil libstore libfetchers
 
-libexpr_LDFLAGS += -lboost_context -pthread
+libexpr_LDFLAGS += -lboost_context -lboost_regex -pthread
 ifdef HOST_LINUX
  libexpr_LDFLAGS += -ldl
 endif
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index ba735b435..0c34cb6e8 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -16,6 +16,7 @@
 #include "primops.hh"
 
 #include <boost/container/small_vector.hpp>
+#include <boost/regex.hpp>
 #include <nlohmann/json.hpp>
 
 #include <sys/types.h>
@@ -24,7 +25,6 @@
 
 #include <algorithm>
 #include <cstring>
-#include <regex>
 #include <dlfcn.h>
 
 #include <cmath>
@@ -3885,19 +3885,30 @@ static RegisterPrimOp primop_convertHash({
     .fun = prim_convertHash,
 });
 
+// regex aliases, switch between boost and std
+using regex = boost::regex;
+using regex_error = boost::regex_error;
+using cmatch = boost::cmatch;
+using cregex_iterator = boost::cregex_iterator;
+namespace regex_constants = boost::regex_constants;
+// overloaded function alias
+constexpr auto regex_match = [] (auto &&...args) {
+    return boost::regex_match(std::forward<decltype(args)>(args)...);
+ };
+
 struct RegexCache
 {
     // TODO use C++20 transparent comparison when available
-    std::unordered_map<std::string_view, std::regex> cache;
+    std::unordered_map<std::string_view, regex> cache;
     std::list<std::string> keys;
 
-    std::regex get(std::string_view re)
+    regex get(std::string_view re)
     {
         auto it = cache.find(re);
         if (it != cache.end())
             return it->second;
         keys.emplace_back(re);
-        return cache.emplace(keys.back(), std::regex(keys.back(), std::regex::extended)).first->second;
+        return cache.emplace(keys.back(), regex(keys.back(), regex::extended)).first->second;
     }
 };
 
@@ -3917,8 +3928,8 @@ void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v)
         NixStringContext context;
         const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.match");
 
-        std::cmatch match;
-        if (!std::regex_match(str.begin(), str.end(), match, regex)) {
+        cmatch match;
+        if (!regex_match(str.begin(), str.end(), match, regex)) {
             v.mkNull();
             return;
         }
@@ -3933,8 +3944,8 @@ void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v)
                 (v.listElems()[i] = state.allocValue())->mkString(match[i + 1].str());
         }
 
-    } catch (std::regex_error & e) {
-        if (e.code() == std::regex_constants::error_space) {
+    } catch (regex_error & e) {
+        if (e.code() == regex_constants::error_space) {
             // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
             state.debugThrowLastTrace(EvalError({
                 .msg = hintfmt("memory limit exceeded by regular expression '%s'", re),
@@ -3997,8 +4008,8 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v)
         NixStringContext context;
         const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.split");
 
-        auto begin = std::cregex_iterator(str.begin(), str.end(), regex);
-        auto end = std::cregex_iterator();
+        auto begin = cregex_iterator(str.begin(), str.end(), regex);
+        auto end = cregex_iterator();
 
         // Any matches results are surrounded by non-matching results.
         const size_t len = std::distance(begin, end);
@@ -4037,8 +4048,8 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 
         assert(idx == 2 * len + 1);
 
-    } catch (std::regex_error & e) {
-        if (e.code() == std::regex_constants::error_space) {
+    } catch (regex_error & e) {
+        if (e.code() == regex_constants::error_space) {
             // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
             state.debugThrowLastTrace(EvalError({
                 .msg = hintfmt("memory limit exceeded by regular expression '%s'", re),

From 213594721ace2b2e2dcf3d57615178a2c1690aa4 Mon Sep 17 00:00:00 2001
From: Peter Kolloch <info@eigenvalue.net>
Date: Sat, 25 Nov 2023 17:21:49 +0100
Subject: [PATCH 204/294] gitignore: Also ignore .DS_Store

This is a file that Finder on Mac OS loves to add into various folders.
---
 .gitignore | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.gitignore b/.gitignore
index 04d96ca2c..f7dcea422 100644
--- a/.gitignore
+++ b/.gitignore
@@ -144,3 +144,6 @@ result
 
 # clangd and possibly more
 .cache/
+
+# Mac OS
+.DS_Store

From f25c06d7a3289343887d09761c82356a3b6b441b Mon Sep 17 00:00:00 2001
From: Jan Tojnar <jtojnar@gmail.com>
Date: Sun, 26 Nov 2023 15:46:09 +0100
Subject: [PATCH 205/294] docs: Fix broken link

Link target definitions need to be in a separate paragraph to be collected.

Fixup for https://github.com/NixOS/nix/commit/217d863f7a251a4d8a08ff3294944b45146c61c9
---
 doc/manual/src/language/values.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md
index 0bb656746..aea68a441 100644
--- a/doc/manual/src/language/values.md
+++ b/doc/manual/src/language/values.md
@@ -116,6 +116,7 @@
   [store path]: ../glossary.md#gloss-store-path
 
   Paths can include [string interpolation] and can themselves be [interpolated in other expressions].
+
   [interpolated in other expressions]: ./string-interpolation.md#interpolated-expressions
 
   At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path.

From d63f72197cec6ff95d9ffc83aa8076acd86a3fd1 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 24 Nov 2023 15:48:41 +0100
Subject: [PATCH 206/294] Don't run changelog-d in the build

This way we lose the preview of release notes on master, as well
as on https://nixos.org/manual/nix/unstable/release-notes/rl-next
but we can come back to this.
---
 doc/manual/local.mk                    | 8 +++++++-
 doc/manual/src/contributing/hacking.md | 2 +-
 flake.nix                              | 9 ++++++---
 3 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index 265a4649d..f22dfa69e 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -146,7 +146,13 @@ $(d)/language.json: $(bindir)/nix
 
 # Generate "Upcoming release" notes (or clear it and remove from menu)
 $(d)/src/release-notes/rl-next.md: $(d)/rl-next $(d)/rl-next/*
-	$(trace-gen) changelog-d doc/manual/rl-next > $@
+	@if type -p changelog-d > /dev/null; then \
+		echo "  GEN   " $@; \
+		changelog-d doc/manual/rl-next > $@; \
+	else \
+		echo "  NULL  " $@; \
+		true > $@; \
+	fi
 
 $(d)/src/SUMMARY-rl-next.md: $(d)/src/release-notes/rl-next.md
 	$(trace-gen) true
diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md
index 855900d7a..3291d5a20 100644
--- a/doc/manual/src/contributing/hacking.md
+++ b/doc/manual/src/contributing/hacking.md
@@ -257,4 +257,4 @@ See also the [format documentation](https://github.com/haskell/cabal/blob/master
 ### Build process
 
 Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`.
-Non-releases build the release notes on the fly.
+Set `buildUnreleasedNotes = true;` in `flake.nix` to build the release notes on the fly.
diff --git a/flake.nix b/flake.nix
index ca37948a7..db913f062 100644
--- a/flake.nix
+++ b/flake.nix
@@ -14,6 +14,9 @@
 
       officialRelease = false;
 
+      # Set to true to build the release notes for the next release.
+      buildUnreleasedNotes = false;
+
       version = lib.fileContents ./.version + versionSuffix;
       versionSuffix =
         if officialRelease
@@ -196,7 +199,7 @@
           ]
           ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)]
           # Official releases don't have rl-next, so we don't need to compile a changelog
-          ++ lib.optional (!officialRelease) changelog-d
+          ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d
           ;
 
         buildDeps =
@@ -734,8 +737,8 @@
               ++ lib.optional
                 (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform)
                 pkgs.buildPackages.clang-tools
-              # We want changelog-d in the shell even if it's an official release
-              ++ lib.optional officialRelease changelog-d
+              # We want changelog-d in the shell even if the current build doesn't need it
+              ++ lib.optional (officialRelease || ! buildUnreleasedNotes) changelog-d
               ;
 
             buildInputs = buildDeps ++ propagatedDeps

From c5d49ec7ab7b9fb33f0336a909ac837e208be807 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 24 Nov 2023 16:18:27 +0100
Subject: [PATCH 207/294] flake.lock: Update
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nixpkgs':
    'github:NixOS/nixpkgs/decdf666c833a325cb4417041a90681499e06a41' (2023-11-18)
  → 'github:NixOS/nixpkgs/9ba29e2346bc542e9909d1021e8fd7d4b3f64db0' (2023-11-23)
---
 flake.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flake.lock b/flake.lock
index 1b3c0b6d5..f120d3b5f 100644
--- a/flake.lock
+++ b/flake.lock
@@ -50,11 +50,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1700342017,
-        "narHash": "sha256-HaibwlWH5LuqsaibW3sIVjZQtEM/jWtOHX4Nk93abGE=",
+        "lastModified": 1700748986,
+        "narHash": "sha256-/nqLrNU297h3PCw4QyDpZKZEUHmialJdZW2ceYFobds=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "decdf666c833a325cb4417041a90681499e06a41",
+        "rev": "9ba29e2346bc542e9909d1021e8fd7d4b3f64db0",
         "type": "github"
       },
       "original": {

From e7e21aa0c839460a62456fa44f31339c187077ff Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 24 Nov 2023 16:27:10 +0100
Subject: [PATCH 208/294] flake.nix: Use top level changelog-d

It is about 2 MB now, as only it and libffi (tiny) are new in the
build or shell closures.
---
 flake.nix | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/flake.nix b/flake.nix
index db913f062..a239226c0 100644
--- a/flake.nix
+++ b/flake.nix
@@ -176,9 +176,7 @@
           "--enable-internal-api-docs"
         ];
 
-        # TODO: after backport of https://github.com/NixOS/nixpkgs/pull/268487, remove `haskellPackages.` -
-        #       vastly improves output closure, and adds shell completions
-        changelog-d = pkgs.buildPackages.haskellPackages.changelog-d;
+        changelog-d = pkgs.buildPackages.changelog-d;
 
         nativeBuildDeps =
           [

From 2b7016cc56d12e67de9f1f25b18311866a26a5fe Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Mon, 27 Nov 2023 08:33:03 +0100
Subject: [PATCH 209/294] add path based redirects

up to now, those were managed outside of this repo, which as
unsurprisingly a real hassle to deal with if one wanted to prevent URLs
from breaking when moving pages around. this change removes a large part
of the friction involved in moving content in the Nix manual.

possible next steps for further automation:
- check for content that moved and warn if it's not reachable from
  links that were valid prior to a change
- create redirect rules automatically based on this information
---
 doc/manual/_redirects   | 30 ++++++++++++++++++++++++++++++
 doc/manual/redirects.js |  8 +++++---
 2 files changed, 35 insertions(+), 3 deletions(-)
 create mode 100644 doc/manual/_redirects

diff --git a/doc/manual/_redirects b/doc/manual/_redirects
new file mode 100644
index 000000000..4ea289d86
--- /dev/null
+++ b/doc/manual/_redirects
@@ -0,0 +1,30 @@
+# redirect rules for paths (server-side) to prevent link rot.
+# see ./redirects.js for redirects based on URL fragments (client-side)
+#
+# concrete user story this supports:
+# - user finds URL to the manual for Nix x.y
+# - Nix x.z (z > y) is the most recent release
+# - updating the version in the URL will show the right thing
+#
+# format documentation:
+# - https://docs.netlify.com/routing/redirects/#syntax-for-the-redirects-file
+# - https://docs.netlify.com/routing/redirects/redirect-options/
+#
+# conventions:
+# - always force (<CODE>!) since this allows re-using file names
+# - group related paths to ease readability
+# - always append new redirects to the end of the file
+# - redirects that should have been there but are missing can be inserted where they belong
+
+/expressions/expression-language /language/ 301!
+/expressions/language-values /language/values 301!
+/expressions/language-constructs /language/constructs 301!
+/expressions/language-operators /language/operators 301!
+/expressions/* /language/:splat 301!
+
+/package-management/basic-package-mgmt /command-ref/nix-env 301!
+
+/package-management/channels* /command-ref/nix-channel 301!
+
+/package-management/s3-substituter* /command-ref/new-cli/nix3-help-stores#s3-binary-cache-store 301!
+
diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js
index d1b10109d..3b507adf3 100644
--- a/doc/manual/redirects.js
+++ b/doc/manual/redirects.js
@@ -1,7 +1,9 @@
-// redirect rules for anchors ensure backwards compatibility of URLs.
-// this must be done on the client side, as web servers do not see the anchor part of the URL.
+// redirect rules for URL fragments (client-side) to prevent link rot.
+// this must be done on the client side, as web servers do not see the fragment part of the URL.
+// it will only work with JavaScript enabled in the browser, but this is the best we can do here.
+// see ./_redirects for path redirects (client-side)
 
-// redirections are declared as follows:
+// redirects are declared as follows:
 // each entry has as its key a path matching the requested URL path, relative to the mdBook document root.
 //
 //     IMPORTANT: it must specify the full path with file name and suffix

From f56401a114cb5504c3d74b893ce270ed28fd03e3 Mon Sep 17 00:00:00 2001
From: Moritz Angermann <moritz.angermann@gmail.com>
Date: Sat, 25 Nov 2023 11:26:57 +0800
Subject: [PATCH 210/294] `nix flake update` add deprecation warnings.

This builds on #8817, to add additional UX help for people with existing
muscle memory (or shell history) with --update-input and tries to gently
guide them towards the newly evolved CLI UI.

Co-authored-by: Cole Helbling <cole.e.helbling@outlook.com>
---
 src/libcmd/installables.cc | 24 ++++++++++++++++++++++++
 src/nix/flake.cc           |  8 +++++++-
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index 1c6103020..68287b445 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -47,6 +47,16 @@ MixFlakeOptions::MixFlakeOptions()
 {
     auto category = "Common flake-related options";
 
+    addFlag({
+        .longName = "recreate-lock-file",
+        .description = "Recreate the flake's lock file from scratch.",
+        .category = category,
+        .handler = {[&]() {
+            lockFlags.recreateLockFile = true;
+            warn("'--recreate-lock-file' is deprecated and will be removed in a future version; use 'nix flake update' instead.");
+        }}
+    });
+
     addFlag({
         .longName = "no-update-lock-file",
         .description = "Do not allow any updates to the flake's lock file.",
@@ -79,6 +89,20 @@ MixFlakeOptions::MixFlakeOptions()
         .handler = {&lockFlags.commitLockFile, true}
     });
 
+    addFlag({
+        .longName = "update-input",
+        .description = "Update a specific flake input (ignoring its previous entry in the lock file).",
+        .category = category,
+        .labels = {"input-path"},
+        .handler = {[&](std::string s) {
+            warn("'--update-input' is a deprecated alias for 'flake update' and will be removed in a future version.");
+            lockFlags.inputUpdates.insert(flake::parseInputPath(s));
+        }},
+        .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
+            completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix);
+        }}
+    });
+
     addFlag({
         .longName = "override-input",
         .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`). This implies `--no-write-lock-file`.",
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index 38938f09e..e0c67fdfa 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -89,7 +89,13 @@ public:
             .label="inputs",
             .optional=true,
             .handler={[&](std::string inputToUpdate){
-                auto inputPath = flake::parseInputPath(inputToUpdate);
+                InputPath inputPath;
+                try {
+                    inputPath = flake::parseInputPath(inputToUpdate);
+                } catch (Error & e) {
+                    warn("Invalid flake input '%s'. To update a specific flake, use 'nix flake update --flake %s' instead.", inputToUpdate, inputToUpdate);
+                    throw e;
+                }
                 if (lockFlags.inputUpdates.contains(inputPath))
                     warn("Input '%s' was specified multiple times. You may have done this by accident.");
                 lockFlags.inputUpdates.insert(inputPath);

From 75134b7513eb781074969fc8d6d865cc95063444 Mon Sep 17 00:00:00 2001
From: Sergei Trofimovich <slyich@gmail.com>
Date: Mon, 27 Nov 2023 08:56:24 +0000
Subject: [PATCH 211/294] libexpr: add missing dependency on
 'flake/call-flake.nix.gen.hh'

Without the change build for `eval.o` fails occasionally as:

    $ make src/libexpr/eval.o
      GEN    Makefile.config
      GEN    src/libexpr/primops/derivation.nix.gen.hh
      GEN    src/libexpr/fetchurl.nix.gen.hh
      GEN    src/libexpr/parser-tab.cc
      GEN    src/libexpr/lexer-tab.cc
    src/libexpr/lexer.l:314: warning, -s option given but default rule can be matched
      CXX    src/libexpr/eval.o
    src/libexpr/eval.cc:519:18: fatal error: flake/call-flake.nix.gen.hh: No such file or directory
      519 |         #include "flake/call-flake.nix.gen.hh"
          |                  ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    compilation terminated.
    make: *** [mk/patterns.mk:3: src/libexpr/eval.o] Error 1

Noticed in https://github.com/NixOS/nixpkgs/pull/269439
---
 src/libexpr/local.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk
index ed7bf9490..637f998b6 100644
--- a/src/libexpr/local.mk
+++ b/src/libexpr/local.mk
@@ -47,6 +47,6 @@ $(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh
 
 $(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh
 
-$(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh
+$(d)/eval.o: $(d)/flake/call-flake.nix.gen.hh
 
 src/libexpr/primops/fromTOML.o:	ERROR_SWITCH_ENUM =

From e986d20bedfc054663632255388bbd33fec99114 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <7226587+thufschmitt@users.noreply.github.com>
Date: Mon, 27 Nov 2023 10:25:18 +0100
Subject: [PATCH 212/294] Remove an obsolete comment

---
 flake.nix | 2 --
 1 file changed, 2 deletions(-)

diff --git a/flake.nix b/flake.nix
index 1f7e7b2af..887ef93b4 100644
--- a/flake.nix
+++ b/flake.nix
@@ -500,8 +500,6 @@
           });
 
           boost-nix = final.boost.override {
-            # enableIcu arg is not yet supported
-            # but will be with next nixpkgs update
             enableIcu = false;
           };
 

From 384ffb4443fd47d04f36c6bcc6ebf476274673ab Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Mon, 27 Nov 2023 09:18:56 +0100
Subject: [PATCH 213/294] add deprecation warnings in documentation

this is hacky, but can serve as a stopgap until we can do it
programmatically.
---
 src/libcmd/installables.cc | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index 68287b445..6e670efea 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -49,7 +49,13 @@ MixFlakeOptions::MixFlakeOptions()
 
     addFlag({
         .longName = "recreate-lock-file",
-        .description = "Recreate the flake's lock file from scratch.",
+        .description = R"(
+    Recreate the flake's lock file from scratch.
+
+    > **DEPRECATED**
+    >
+    > Use [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) instead.
+        )",
         .category = category,
         .handler = {[&]() {
             lockFlags.recreateLockFile = true;
@@ -73,8 +79,13 @@ MixFlakeOptions::MixFlakeOptions()
 
     addFlag({
         .longName = "no-registries",
-        .description =
-          "Don't allow lookups in the flake registries. This option is deprecated; use `--no-use-registries`.",
+        .description = R"(
+    Don't allow lookups in the flake registries.
+
+    > **DEPRECATED**
+    >
+    > Use [`--no-use-registries`](#opt-no-use-registries) instead.
+        )",
         .category = category,
         .handler = {[&]() {
             lockFlags.useRegistries = false;
@@ -91,7 +102,13 @@ MixFlakeOptions::MixFlakeOptions()
 
     addFlag({
         .longName = "update-input",
-        .description = "Update a specific flake input (ignoring its previous entry in the lock file).",
+        .description = R"(
+    Update a specific flake input (ignoring its previous entry in the lock file).
+
+    > **DEPRECATED**
+    >
+    > Use [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) instead.
+        )",
         .category = category,
         .labels = {"input-path"},
         .handler = {[&](std::string s) {

From f7bfec2806708573798d610cda101f27a24d9218 Mon Sep 17 00:00:00 2001
From: Robert Hensing <roberth@users.noreply.github.com>
Date: Mon, 27 Nov 2023 15:18:29 +0100
Subject: [PATCH 214/294] maintainers/release-notes: Improve DATE check

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
---
 maintainers/release-notes | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/maintainers/release-notes b/maintainers/release-notes
index 2e1be64a4..34cd85a56 100755
--- a/maintainers/release-notes
+++ b/maintainers/release-notes
@@ -134,7 +134,7 @@ if [[ ! -n "${DATE:-}" ]]; then
 fi
 
 case "$DATE" in
-  [0-9]*-[0-9]*-[0-9]*)
+  [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9])
     ;;
   *)
     die "DATE must be YYYY-MM-DD, e.g. 2021-12-31 (DATE was set to $DATE)"

From 68c48756fece5aee77f9b44607afa9248d75e67c Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 27 Nov 2023 15:50:45 +0100
Subject: [PATCH 215/294] libexpr/local.mk: Make eval compile deps regular

Dependency is now entirely through the eval.cc rule.
All gen.hh deps are now there.
---
 src/libexpr/local.mk | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk
index 637f998b6..b37fe6f1d 100644
--- a/src/libexpr/local.mk
+++ b/src/libexpr/local.mk
@@ -45,8 +45,6 @@ $(foreach i, $(wildcard src/libexpr/flake/*.hh), \
 
 $(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh
 
-$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh
-
-$(d)/eval.o: $(d)/flake/call-flake.nix.gen.hh
+$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/flake/call-flake.nix.gen.hh
 
 src/libexpr/primops/fromTOML.o:	ERROR_SWITCH_ENUM =

From 20cd5eb2b3668f5b95b6f020e1c258011c18ea33 Mon Sep 17 00:00:00 2001
From: Alois Wohlschlager <alois1@gmx-topmail.de>
Date: Mon, 27 Nov 2023 19:12:15 +0100
Subject: [PATCH 216/294] nix repl: Only hide the progress bar while waiting
 for user input

In commit 0d2163c6dcf03463fa91ec6d0d96c928ad907366, the progress bar was hidden
in nix repl because of a regression that caused it to interfere with user
input. Several users like(d) seeing the progress bar in the repl during builds.
Only hiding it while waiting for user input gives us the best of both worlds,
so do just that.
---
 src/libcmd/repl.cc | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc
index bf5643a5c..0986296ad 100644
--- a/src/libcmd/repl.cc
+++ b/src/libcmd/repl.cc
@@ -43,7 +43,6 @@ extern "C" {
 #include "finally.hh"
 #include "markdown.hh"
 #include "local-fs-store.hh"
-#include "progress-bar.hh"
 #include "print.hh"
 
 #if HAVE_BOEHMGC
@@ -262,13 +261,11 @@ void NixRepl::mainLoop()
     rl_set_list_possib_func(listPossibleCallback);
 #endif
 
-    /* Stop the progress bar because it interferes with the display of
-       the repl. */
-    stopProgressBar();
-
     std::string input;
 
     while (true) {
+        // Hide the progress bar while waiting for user input, so that it won't interfere.
+        logger->pause();
         // When continuing input from previous lines, don't print a prompt, just align to the same
         // number of chars as the prompt.
         if (!getLine(input, input.empty() ? "nix-repl> " : "          ")) {
@@ -278,6 +275,7 @@ void NixRepl::mainLoop()
             logger->cout("");
             break;
         }
+        logger->resume();
         try {
             if (!removeWhitespace(input).empty() && !processLine(input)) return;
         } catch (ParseError & e) {

From f300e11b056dea414d7d77bbc6e5a7dc5d9ddd41 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <theophane.hufschmitt@tweag.io>
Date: Mon, 27 Nov 2023 19:41:30 +0100
Subject: [PATCH 217/294] Rename `nix show-config` to `nix config show`

Part of #7672
---
 doc/manual/local.mk                       |  4 ++--
 doc/manual/rl-next/nix-config-show.md     |  8 +++++++
 src/nix/{show-config.cc => config.cc}     | 27 ++++++++++++++++++++---
 src/nix/main.cc                           |  1 +
 tests/functional/config.sh                | 12 +++++-----
 tests/functional/experimental-features.sh |  8 +++----
 6 files changed, 45 insertions(+), 15 deletions(-)
 create mode 100644 doc/manual/rl-next/nix-config-show.md
 rename src/nix/{show-config.cc => config.cc} (67%)

diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index f22dfa69e..d568681d4 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -24,7 +24,7 @@ man-pages += $(foreach subcommand, \
 clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8
 
 # Provide a dummy environment for nix, so that it will not access files outside the macOS sandbox.
-# Set cores to 0 because otherwise nix show-config resolves the cores based on the current machine
+# Set cores to 0 because otherwise `nix config show` resolves the cores based on the current machine
 dummy-env = env -i \
 	HOME=/dummy \
 	NIX_CONF_DIR=/dummy \
@@ -111,7 +111,7 @@ $(d)/nix.json: $(bindir)/nix
 	@mv $@.tmp $@
 
 $(d)/conf-file.json: $(bindir)/nix
-	$(trace-gen) $(dummy-env) $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp
+	$(trace-gen) $(dummy-env) $(bindir)/nix config show --json --experimental-features nix-command > $@.tmp
 	@mv $@.tmp $@
 
 $(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(bindir)/nix
diff --git a/doc/manual/rl-next/nix-config-show.md b/doc/manual/rl-next/nix-config-show.md
new file mode 100644
index 000000000..08ad207cb
--- /dev/null
+++ b/doc/manual/rl-next/nix-config-show.md
@@ -0,0 +1,8 @@
+synopsis: `nix config show`
+issues: #7672
+prs: #9477
+description: {
+
+`nix show-config` was renamed to `nix config show` to be more consistent with the rest of the command-line interface.
+
+}
diff --git a/src/nix/show-config.cc b/src/nix/config.cc
similarity index 67%
rename from src/nix/show-config.cc
rename to src/nix/config.cc
index 3530584f9..5b280d11d 100644
--- a/src/nix/show-config.cc
+++ b/src/nix/config.cc
@@ -7,11 +7,31 @@
 
 using namespace nix;
 
-struct CmdShowConfig : Command, MixJSON
+struct CmdConfig : virtual NixMultiCommand
+{
+    CmdConfig() : MultiCommand(RegisterCommand::getCommandsFor({"config"}))
+    { }
+
+    std::string description() override
+    {
+        return "manipulate the Nix configuration";
+    }
+
+    Category category() override { return catUtility; }
+
+    void run() override
+    {
+        if (!command)
+            throw UsageError("'nix config' requires a sub-command.");
+        command->second->run();
+    }
+};
+
+struct CmdConfigShow : Command, MixJSON
 {
     std::optional<std::string> name;
 
-    CmdShowConfig() {
+    CmdConfigShow() {
         expectArgs({
             .label = {"name"},
             .optional = true,
@@ -56,4 +76,5 @@ struct CmdShowConfig : Command, MixJSON
     }
 };
 
-static auto rShowConfig = registerCommand<CmdShowConfig>("show-config");
+static auto rCmdConfig = registerCommand<CmdConfig>("config");
+static auto rShowConfig = registerCommand2<CmdConfigShow>({"config", "show"});
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 73641f6d2..2a6c2f478 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -134,6 +134,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs
         {"ping-store", {"store", "ping"}},
         {"sign-paths", {"store", "sign"}},
         {"show-derivation", {"derivation", "show"}},
+        {"show-config", {"config", "show"}},
         {"to-base16", {"hash", "to-base16"}},
         {"to-base32", {"hash", "to-base32"}},
         {"to-base64", {"hash", "to-base64"}},
diff --git a/tests/functional/config.sh b/tests/functional/config.sh
index 0780c55d0..324fe95bd 100644
--- a/tests/functional/config.sh
+++ b/tests/functional/config.sh
@@ -40,20 +40,20 @@ files=$(nix-build --verbose --version | grep "User config" | cut -d ':' -f2- | x
 # Test that it's possible to load the config from a custom location
 here=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")
 export NIX_USER_CONF_FILES=$here/config/nix-with-substituters.conf
-var=$(nix show-config | grep '^substituters =' | cut -d '=' -f 2 | xargs)
+var=$(nix config show | grep '^substituters =' | cut -d '=' -f 2 | xargs)
 [[ $var == https://example.com ]]
 
 # Test that it's possible to load config from the environment
-prev=$(nix show-config | grep '^cores' | cut -d '=' -f 2 | xargs)
+prev=$(nix config show | grep '^cores' | cut -d '=' -f 2 | xargs)
 export NIX_CONFIG="cores = 4242"$'\n'"experimental-features = nix-command flakes"
-exp_cores=$(nix show-config | grep '^cores' | cut -d '=' -f 2 | xargs)
-exp_features=$(nix show-config | grep '^experimental-features' | cut -d '=' -f 2 | xargs)
+exp_cores=$(nix config show | grep '^cores' | cut -d '=' -f 2 | xargs)
+exp_features=$(nix config show | grep '^experimental-features' | cut -d '=' -f 2 | xargs)
 [[ $prev != $exp_cores ]]
 [[ $exp_cores == "4242" ]]
 # flakes implies fetch-tree
 [[ $exp_features == "fetch-tree flakes nix-command" ]]
 
 # Test that it's possible to retrieve a single setting's value
-val=$(nix show-config | grep '^warn-dirty' | cut -d '=' -f  2 | xargs)
-val2=$(nix show-config warn-dirty)
+val=$(nix config show | grep '^warn-dirty' | cut -d '=' -f  2 | xargs)
+val2=$(nix config show warn-dirty)
 [[ $val == $val2 ]]
diff --git a/tests/functional/experimental-features.sh b/tests/functional/experimental-features.sh
index 607bf0a8e..9ee4a53d4 100644
--- a/tests/functional/experimental-features.sh
+++ b/tests/functional/experimental-features.sh
@@ -31,7 +31,7 @@ source common.sh
 NIX_CONFIG='
   experimental-features = nix-command
   accept-flake-config = true
-' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr
+' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr
 grepQuiet "false" $TEST_ROOT/stdout
 grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr
 
@@ -39,7 +39,7 @@ grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature '
 NIX_CONFIG='
   accept-flake-config = true
   experimental-features = nix-command
-' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr
+' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr
 grepQuiet "false" $TEST_ROOT/stdout
 grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr
 
@@ -47,7 +47,7 @@ grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature '
 NIX_CONFIG='
   experimental-features = nix-command flakes
   accept-flake-config = true
-' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr
+' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr
 grepQuiet "true" $TEST_ROOT/stdout
 grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr
 
@@ -55,7 +55,7 @@ grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr
 NIX_CONFIG='
   accept-flake-config = true
   experimental-features = nix-command flakes
-' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr
+' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr
 grepQuiet "true" $TEST_ROOT/stdout
 grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr
 

From 6d1605818c12461edc9f4ee17a7929fdc8fe916c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <theophane.hufschmitt@tweag.io>
Date: Mon, 27 Nov 2023 19:46:05 +0100
Subject: [PATCH 218/294] Rename `nix doctor` to `nix config check`

Fix #7672
---
 doc/manual/src/contributing/cli-guideline.md | 2 +-
 src/nix/{doctor.cc => config-check.cc}       | 4 ++--
 src/nix/main.cc                              | 1 +
 3 files changed, 4 insertions(+), 3 deletions(-)
 rename src/nix/{doctor.cc => config-check.cc} (97%)

diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/contributing/cli-guideline.md
index e53d2d178..8dbac45b0 100644
--- a/doc/manual/src/contributing/cli-guideline.md
+++ b/doc/manual/src/contributing/cli-guideline.md
@@ -87,7 +87,7 @@ impacted the most by bad user experience.
       and [aligning of text](#text-alignment).
     - [Autocomplete](#shell-completion) of options.
 
-  Examples of such commands: `nix doctor`, `nix edit`, `nix eval`, ...
+  Examples of such commands: `nix edit`, `nix eval`, ...
 
 - **Utility and scripting commands**
 
diff --git a/src/nix/doctor.cc b/src/nix/config-check.cc
similarity index 97%
rename from src/nix/doctor.cc
rename to src/nix/config-check.cc
index 59f9e3e5d..410feca2f 100644
--- a/src/nix/doctor.cc
+++ b/src/nix/config-check.cc
@@ -38,7 +38,7 @@ void checkInfo(const std::string & msg) {
 
 }
 
-struct CmdDoctor : StoreCommand
+struct CmdConfigCheck : StoreCommand
 {
     bool success = true;
 
@@ -152,4 +152,4 @@ struct CmdDoctor : StoreCommand
     }
 };
 
-static auto rCmdDoctor = registerCommand<CmdDoctor>("doctor");
+static auto rCmdConfigCheck = registerCommand2<CmdConfigCheck>({ "config", "check" });
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 2a6c2f478..d715d0400 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -139,6 +139,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs
         {"to-base32", {"hash", "to-base32"}},
         {"to-base64", {"hash", "to-base64"}},
         {"verify", {"store", "verify"}},
+        {"doctor", {"config", "check"}},
     };
 
     bool aliasUsed = false;

From 52e0911302b20336c1600b60a98894423e110d7d Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sat, 25 Nov 2023 00:33:21 -0500
Subject: [PATCH 219/294] Use `buildprefix` in a few more places

`installcheck` doesn't yet work, but the rest of the build can now
happen mostly inside a separate build directory.

Progress on #9342

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
---
 Makefile                               |  6 ++++--
 doc/manual/src/contributing/hacking.md | 25 +++++++++++++++++++++++++
 mk/build-dir.mk                        | 10 ++++++++++
 mk/install-dirs.mk                     | 11 +++++++++++
 mk/lib.mk                              | 25 +++----------------------
 mk/templates.mk                        |  8 ++++----
 src/libcmd/local.mk                    |  2 +-
 src/libexpr/local.mk                   |  4 ++--
 src/libmain/local.mk                   |  2 +-
 src/libstore/local.mk                  |  2 +-
 tests/functional/local.mk              |  4 ++--
 11 files changed, 64 insertions(+), 35 deletions(-)
 create mode 100644 mk/build-dir.mk
 create mode 100644 mk/install-dirs.mk

diff --git a/Makefile b/Makefile
index 4f4ac0c6e..77974074c 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,7 @@
--include Makefile.config
-clean-files += Makefile.config
+include mk/build-dir.mk
+
+-include $(buildprefix)Makefile.config
+clean-files += $(buildprefix)Makefile.config
 
 ifeq ($(ENABLE_BUILD), yes)
 makefiles = \
diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md
index 3291d5a20..0a95334f7 100644
--- a/doc/manual/src/contributing/hacking.md
+++ b/doc/manual/src/contributing/hacking.md
@@ -146,6 +146,31 @@ $ nix build .#packages.aarch64-linux.default
 Cross-compiled builds are available for ARMv6 (`armv6l-linux`) and ARMv7 (`armv7l-linux`).
 Add more [system types](#system-type) to `crossSystems` in `flake.nix` to bootstrap Nix on unsupported platforms.
 
+### Building for multiple platforms at once
+
+It is useful to perform multiple cross and native builds on the same source tree,
+for example to ensure that better support for one platform doesn't break the build for another.
+In order to facilitate this, Nix has some support for being built out of tree – that is, placing build artefacts in a different directory than the source code:
+
+1. Create a directory for the build, e.g.
+
+   ```bash
+   mkdir build
+   ```
+
+2. Run the configure script from that directory, e.g.
+
+   ```bash
+   cd build
+   ../configure <configure flags>
+   ```
+
+3. Run make from the source directory, but with the build directory specified, e.g.
+
+   ```bash
+   make builddir=build <make flags>
+   ```
+
 ## System type
 
 Nix uses a string with he following format to identify the *system type* or *platform* it runs on:
diff --git a/mk/build-dir.mk b/mk/build-dir.mk
new file mode 100644
index 000000000..02f4cae60
--- /dev/null
+++ b/mk/build-dir.mk
@@ -0,0 +1,10 @@
+# Initialise support for build directories.
+builddir ?=
+
+ifdef builddir
+  buildprefix = $(builddir)/
+  buildprefixrel = $(builddir)
+else
+  buildprefix =
+  buildprefixrel = .
+endif
diff --git a/mk/install-dirs.mk b/mk/install-dirs.mk
new file mode 100644
index 000000000..732b0d6fc
--- /dev/null
+++ b/mk/install-dirs.mk
@@ -0,0 +1,11 @@
+# Default installation paths.
+prefix ?= /usr/local
+libdir ?= $(prefix)/lib
+bindir ?= $(prefix)/bin
+libexecdir ?= $(prefix)/libexec
+datadir ?= $(prefix)/share
+localstatedir ?= $(prefix)/var
+sysconfdir ?= $(prefix)/etc
+mandir ?= $(prefix)/share/man
+
+DESTDIR ?=
diff --git a/mk/lib.mk b/mk/lib.mk
index 49abe9862..3d503364f 100644
--- a/mk/lib.mk
+++ b/mk/lib.mk
@@ -43,27 +43,6 @@ define newline
 endef
 
 
-# Default installation paths.
-prefix ?= /usr/local
-libdir ?= $(prefix)/lib
-bindir ?= $(prefix)/bin
-libexecdir ?= $(prefix)/libexec
-datadir ?= $(prefix)/share
-localstatedir ?= $(prefix)/var
-sysconfdir ?= $(prefix)/etc
-mandir ?= $(prefix)/share/man
-
-
-# Initialise support for build directories.
-builddir ?=
-
-ifdef builddir
-  buildprefix = $(builddir)/
-else
-  buildprefix =
-endif
-
-
 # Pass -fPIC if we're building dynamic libraries.
 BUILD_SHARED_LIBS ?= 1
 
@@ -94,6 +73,8 @@ ifeq ($(BUILD_DEBUG), 1)
 endif
 
 
+include mk/build-dir.mk
+include mk/install-dirs.mk
 include mk/functions.mk
 include mk/tracing.mk
 include mk/clean.mk
@@ -112,7 +93,7 @@ define include-sub-makefile
   include $(1)
 endef
 
-$(foreach mf, $(makefiles), $(eval $(call include-sub-makefile, $(mf))))
+$(foreach mf, $(makefiles), $(eval $(call include-sub-makefile,$(mf))))
 
 
 # Instantiate stuff.
diff --git a/mk/templates.mk b/mk/templates.mk
index c7ac7afbf..866bdc17f 100644
--- a/mk/templates.mk
+++ b/mk/templates.mk
@@ -10,10 +10,10 @@ endef
 
 ifneq ($(MAKECMDGOALS), clean)
 
-%.h: %.h.in
-	$(trace-gen) rm -f $@ && ./config.status --quiet --header=$@
+$(buildprefix)%.h: %.h.in
+	$(trace-gen) rm -f $@ && cd $(buildprefixrel) && ./config.status --quiet --header=$(@:$(buildprefix)%=%)
 
-%: %.in
-	$(trace-gen) rm -f $@ && ./config.status --quiet --file=$@
+$(buildprefix)%: %.in
+	$(trace-gen) rm -f $@ && cd $(buildprefixrel) && ./config.status --quiet --file=$(@:$(buildprefix)%=%)
 
 endif
diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk
index 541a7d2ba..afd35af08 100644
--- a/src/libcmd/local.mk
+++ b/src/libcmd/local.mk
@@ -12,4 +12,4 @@ libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) -pthread
 
 libcmd_LIBS = libstore libutil libexpr libmain libfetchers
 
-$(eval $(call install-file-in, $(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644))
+$(eval $(call install-file-in, $(buildprefix)$(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644))
diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk
index 18d1bc95c..c07a18bb5 100644
--- a/src/libexpr/local.mk
+++ b/src/libexpr/local.mk
@@ -36,7 +36,7 @@ $(d)/lexer-tab.cc $(d)/lexer-tab.hh: $(d)/lexer.l
 
 clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh
 
-$(eval $(call install-file-in, $(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644))
+$(eval $(call install-file-in, $(buildprefix)$(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644))
 
 $(foreach i, $(wildcard src/libexpr/value/*.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix/value, 0644)))
@@ -47,4 +47,4 @@ $(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh
 
 $(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/flake/call-flake.nix.gen.hh
 
-src/libexpr/primops/fromTOML.o:	ERROR_SWITCH_ENUM =
+$(buildprefix)src/libexpr/primops/fromTOML.o:	ERROR_SWITCH_ENUM =
diff --git a/src/libmain/local.mk b/src/libmain/local.mk
index 99da95e27..5c7061863 100644
--- a/src/libmain/local.mk
+++ b/src/libmain/local.mk
@@ -14,4 +14,4 @@ libmain_LIBS = libstore libutil
 
 libmain_ALLOW_UNDEFINED = 1
 
-$(eval $(call install-file-in, $(d)/nix-main.pc, $(libdir)/pkgconfig, 0644))
+$(eval $(call install-file-in, $(buildprefix)$(d)/nix-main.pc, $(libdir)/pkgconfig, 0644))
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index 0be0bf310..68ccdc409 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -59,7 +59,7 @@ $(d)/build.cc:
 
 clean-files += $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh
 
-$(eval $(call install-file-in, $(d)/nix-store.pc, $(libdir)/pkgconfig, 0644))
+$(eval $(call install-file-in, $(buildprefix)$(d)/nix-store.pc, $(libdir)/pkgconfig, 0644))
 
 $(foreach i, $(wildcard src/libstore/builtins/*.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix/builtins, 0644)))
diff --git a/tests/functional/local.mk b/tests/functional/local.mk
index 8d584142a..10b399d75 100644
--- a/tests/functional/local.mk
+++ b/tests/functional/local.mk
@@ -140,9 +140,9 @@ ifeq ($(ENABLE_BUILD), yes)
 endif
 
 $(d)/test-libstoreconsumer.sh.test $(d)/test-libstoreconsumer.sh.test-debug: \
-  $(d)/test-libstoreconsumer/test-libstoreconsumer
+  $(buildprefix)$(d)/test-libstoreconsumer/test-libstoreconsumer
 $(d)/plugins.sh.test $(d)/plugins.sh.test-debug: \
-  $(d)/plugins/libplugintest.$(SO_EXT)
+  $(buildprefix)$(d)/plugins/libplugintest.$(SO_EXT)
 
 install-tests += $(foreach x, $(nix_tests), $(d)/$(x))
 

From 02bd821f2e71372d31bbe6700bd68086cc2ee70a Mon Sep 17 00:00:00 2001
From: Alex Ameen <alex.ameen.tx@gmail.com>
Date: Wed, 29 Nov 2023 19:26:39 -0600
Subject: [PATCH 220/294] fix: `nlohmann::adl_serializer` for `std::optional`
 (#9147)

This allows templates such as `NLOHMANN_DEFINE_TYPE_*` templates and other generators with things like `std::vector<std::optional<T>>`.

Co-authored-by: John Ericson <John.Ericson@Obsidian.Systems>
---
 src/libutil/json-utils.hh       | 19 ++++++++---
 src/libutil/tests/json-utils.cc | 58 +++++++++++++++++++++++++++++++++
 2 files changed, 72 insertions(+), 5 deletions(-)
 create mode 100644 src/libutil/tests/json-utils.cc

diff --git a/src/libutil/json-utils.hh b/src/libutil/json-utils.hh
index 77c63595c..06dd80cf7 100644
--- a/src/libutil/json-utils.hh
+++ b/src/libutil/json-utils.hh
@@ -78,20 +78,29 @@ namespace nlohmann {
  */
 template<typename T>
 struct adl_serializer<std::optional<T>> {
-    static std::optional<T> from_json(const json & json) {
+    /**
+     * @brief Convert a JSON type to an `optional<T>` treating
+     *        `null` as `std::nullopt`.
+     */
+    static void from_json(const json & json, std::optional<T> & t) {
         static_assert(
             nix::json_avoids_null<T>::value,
             "null is already in use for underlying type's JSON");
-        return json.is_null()
+        t = json.is_null()
             ? std::nullopt
-            : std::optional { adl_serializer<T>::from_json(json) };
+            : std::make_optional(json.template get<T>());
     }
-    static void to_json(json & json, std::optional<T> t) {
+
+    /**
+     *  @brief Convert an optional type to a JSON type  treating `std::nullopt`
+     *         as `null`.
+     */
+    static void to_json(json & json, const std::optional<T> & t) {
         static_assert(
             nix::json_avoids_null<T>::value,
             "null is already in use for underlying type's JSON");
         if (t)
-            adl_serializer<T>::to_json(json, *t);
+            json = *t;
         else
             json = nullptr;
     }
diff --git a/src/libutil/tests/json-utils.cc b/src/libutil/tests/json-utils.cc
new file mode 100644
index 000000000..f0ce15c93
--- /dev/null
+++ b/src/libutil/tests/json-utils.cc
@@ -0,0 +1,58 @@
+#include <vector>
+#include <optional>
+
+#include <gtest/gtest.h>
+
+#include "json-utils.hh"
+
+namespace nix {
+
+/* Test `to_json` and `from_json` with `std::optional` types.
+ * We are specifically interested in whether we can _nest_ optionals in STL
+ * containers so we that we can leverage existing adl_serializer templates. */
+
+TEST(to_json, optionalInt) {
+    std::optional<int> val = std::make_optional(420);
+    ASSERT_EQ(nlohmann::json(val), nlohmann::json(420));
+    val = std::nullopt;
+    ASSERT_EQ(nlohmann::json(val), nlohmann::json(nullptr));
+}
+
+TEST(to_json, vectorOfOptionalInts) {
+    std::vector<std::optional<int>> vals = {
+        std::make_optional(420),
+        std::nullopt,
+    };
+    ASSERT_EQ(nlohmann::json(vals), nlohmann::json::parse("[420,null]"));
+}
+
+TEST(to_json, optionalVectorOfInts) {
+    std::optional<std::vector<int>> val = std::make_optional(std::vector<int> {
+        -420,
+        420,
+    });
+    ASSERT_EQ(nlohmann::json(val), nlohmann::json::parse("[-420,420]"));
+    val = std::nullopt;
+    ASSERT_EQ(nlohmann::json(val), nlohmann::json(nullptr));
+}
+
+TEST(from_json, optionalInt) {
+    nlohmann::json json = 420;
+    std::optional<int> val = json;
+    ASSERT_TRUE(val.has_value());
+    ASSERT_EQ(*val, 420);
+    json = nullptr;
+    json.get_to(val);
+    ASSERT_FALSE(val.has_value());
+}
+
+TEST(from_json, vectorOfOptionalInts) {
+    nlohmann::json json = { 420, nullptr };
+    std::vector<std::optional<int>> vals = json;
+    ASSERT_EQ(vals.size(), 2);
+    ASSERT_TRUE(vals.at(0).has_value());
+    ASSERT_EQ(*vals.at(0), 420);
+    ASSERT_FALSE(vals.at(1).has_value());
+}
+
+} /* namespace nix */

From a7115a47ef0d83ea81b494f6bc5b11d8286e0672 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 29 Nov 2023 21:03:56 -0500
Subject: [PATCH 221/294] Improve ACL clearing support (fixing FreeBSD build)

The problem was that f880469173061a07f0b2a24734932c5a9ad633c6 forgot
that the `#include <sys/xattr.h>` was guarded by an `#ifdef __linux__`.

However, the build failure was only on FreeBSD --- turns out other
platforms have this header too!

The fix therefore uses a new configure check so we properly clear ACLs
on more platforms.
---
 configure.ac                          | 2 ++
 src/libstore/local-store.cc           | 1 -
 src/libstore/posix-fs-canonicalise.cc | 6 ++++--
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/configure.ac b/configure.ac
index 71e93feaa..f8b937eb5 100644
--- a/configure.ac
+++ b/configure.ac
@@ -282,6 +282,8 @@ case "$host_os" in
 esac
 AC_SUBST(HAVE_SECCOMP, [$have_seccomp])
 
+# Optional dependencies for better normalizing file system data
+AC_CHECK_HEADERS[sys/xattr.h]
 
 # Look for aws-cpp-sdk-s3.
 AC_LANG_PUSH(C++)
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 4ff75f528..9ed061b01 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -34,7 +34,6 @@
 #include <sys/statvfs.h>
 #include <sys/mount.h>
 #include <sys/ioctl.h>
-#include <sys/xattr.h>
 #endif
 
 #ifdef __CYGWIN__
diff --git a/src/libstore/posix-fs-canonicalise.cc b/src/libstore/posix-fs-canonicalise.cc
index cc3ab0b74..f38fa8369 100644
--- a/src/libstore/posix-fs-canonicalise.cc
+++ b/src/libstore/posix-fs-canonicalise.cc
@@ -1,4 +1,6 @@
-#include <sys/xattr.h>
+#if HAVE_SYS_XATTR_H
+# include <sys/xattr.h>
+#endif
 
 #include "posix-fs-canonicalise.hh"
 #include "file-system.hh"
@@ -76,7 +78,7 @@ static void canonicalisePathMetaData_(
     if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)))
         throw Error("file '%1%' has an unsupported type", path);
 
-#if __linux__
+#ifdef HAVE_SYS_XATTR_H
     /* Remove extended attributes / ACLs. */
     ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0);
 

From d536c57e878a04f795c1ef8ee3232a47035da2cf Mon Sep 17 00:00:00 2001
From: Federico Pellegrin <fede@evolware.org>
Date: Tue, 17 Aug 2021 04:26:41 +0200
Subject: [PATCH 222/294] Docs build: depend on locally built nix executable
 and not installed one

Previously many of the documentation targets were depending on
`$(bindir)/nix` which is the installed version. This meant that its
install rules would be triggered (which in chain would also trigger the
install of libraries, as reported in #5140). Therefore a build of the
documentation without an installation would not be possible (which apart
from doing unwanted operations it may also generate permission problems
for example).

The fix makes the rules depend on `$(nix_PATH)` instead, which is the
executable in the build tree.
---
 Makefile            | 11 ++++++++---
 doc/manual/local.mk | 35 ++++++++++++++++++++---------------
 2 files changed, 28 insertions(+), 18 deletions(-)

diff --git a/Makefile b/Makefile
index 4f4ac0c6e..957920686 100644
--- a/Makefile
+++ b/Makefile
@@ -19,9 +19,7 @@ makefiles = \
   misc/zsh/local.mk \
   misc/systemd/local.mk \
   misc/launchd/local.mk \
-  misc/upstart/local.mk \
-  doc/manual/local.mk \
-  doc/internal-api/local.mk
+  misc/upstart/local.mk
 endif
 
 ifeq ($(ENABLE_BUILD)_$(ENABLE_TESTS), yes_yes)
@@ -55,4 +53,11 @@ endif
 
 include mk/lib.mk
 
+# Must be included after `mk/lib.mk` so rules refer to variables defined
+# by the library. Rules are not "lazy" like variables, unfortunately.
+ifeq ($(ENABLE_BUILD), yes)
+$(eval $(call include-sub-makefile, doc/manual/local.mk))
+$(eval $(call include-sub-makefile, doc/internal-api/local.mk))
+endif
+
 GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++2a -I src
diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index d568681d4..fa9db9f02 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -1,5 +1,10 @@
 ifeq ($(doc_generate),yes)
 
+# The version of Nix used to generate the doc. Can also be
+# `$(nix_INSTALL_PATH)` or just `nix` (to grap ambient from the `PATH`),
+# if one prefers.
+doc_nix = $(nix_PATH)
+
 MANUAL_SRCS := \
 	$(call rwildcard, $(d)/src, *.md) \
 	$(call rwildcard, $(d)/src, */*.md)
@@ -32,7 +37,7 @@ dummy-env = env -i \
 	NIX_STATE_DIR=/dummy \
 	NIX_CONFIG='cores = 0'
 
-nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix=doc/manual --store dummy:// --impure --raw
+nix-eval = $(dummy-env) $(doc_nix) eval --experimental-features nix-command -I nix=doc/manual --store dummy:// --impure --raw
 
 # re-implement mdBook's include directive to make it usable for terminal output and for proper @docroot@ substitution
 define process-includes
@@ -96,52 +101,52 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src
 	@cp $< $@
 	@$(call process-includes,$@,$@)
 
-$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(d)/generate-settings.nix $(d)/generate-store-info.nix $(bindir)/nix
+$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(d)/generate-settings.nix $(d)/generate-store-info.nix $(doc_nix)
 	@rm -rf $@ $@.tmp
 	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix true (builtins.readFile $<)'
 	@mv $@.tmp $@
 
-$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/generate-settings.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix
+$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/generate-settings.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(doc_nix)
 	@cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp
 	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-settings.nix { prefix = "conf"; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
 	@mv $@.tmp $@
 
-$(d)/nix.json: $(bindir)/nix
-	$(trace-gen) $(dummy-env) $(bindir)/nix __dump-cli > $@.tmp
+$(d)/nix.json: $(doc_nix)
+	$(trace-gen) $(dummy-env) $(doc_nix) __dump-cli > $@.tmp
 	@mv $@.tmp $@
 
-$(d)/conf-file.json: $(bindir)/nix
-	$(trace-gen) $(dummy-env) $(bindir)/nix config show --json --experimental-features nix-command > $@.tmp
+$(d)/conf-file.json: $(doc_nix)
+	$(trace-gen) $(dummy-env) $(doc_nix) config show --json --experimental-features nix-command > $@.tmp
 	@mv $@.tmp $@
 
-$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(bindir)/nix
+$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(doc_nix)
 	@rm -rf $@ $@.tmp
 	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features.nix (builtins.fromJSON (builtins.readFile $<))'
 	@mv $@.tmp $@
 
-$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(bindir)/nix
+$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(doc_nix)
 	@rm -rf $@ $@.tmp
 	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features-shortlist.nix (builtins.fromJSON (builtins.readFile $<))'
 	@mv $@.tmp $@
 
-$(d)/xp-features.json: $(bindir)/nix
-	$(trace-gen) $(dummy-env) $(bindir)/nix __dump-xp-features > $@.tmp
+$(d)/xp-features.json: $(doc_nix)
+	$(trace-gen) $(dummy-env) $(doc_nix) __dump-xp-features > $@.tmp
 	@mv $@.tmp $@
 
-$(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix
+$(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(doc_nix)
 	@cat doc/manual/src/language/builtins-prefix.md > $@.tmp
 	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<)).builtins' >> $@.tmp;
 	@cat doc/manual/src/language/builtins-suffix.md >> $@.tmp
 	@mv $@.tmp $@
 
-$(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin-constants.nix $(d)/src/language/builtin-constants-prefix.md $(bindir)/nix
+$(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin-constants.nix $(d)/src/language/builtin-constants-prefix.md $(doc_nix)
 	@cat doc/manual/src/language/builtin-constants-prefix.md > $@.tmp
 	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtin-constants.nix (builtins.fromJSON (builtins.readFile $<)).constants' >> $@.tmp;
 	@cat doc/manual/src/language/builtin-constants-suffix.md >> $@.tmp
 	@mv $@.tmp $@
 
-$(d)/language.json: $(bindir)/nix
-	$(trace-gen) $(dummy-env) $(bindir)/nix __dump-language > $@.tmp
+$(d)/language.json: $(doc_nix)
+	$(trace-gen) $(dummy-env) $(doc_nix) __dump-language > $@.tmp
 	@mv $@.tmp $@
 
 # Generate "Upcoming release" notes (or clear it and remove from menu)

From 743232bf04d8ade18cfa2c791ed466ce48519878 Mon Sep 17 00:00:00 2001
From: Greg Pfeil <greg@technomadic.org>
Date: Thu, 30 Nov 2023 00:17:25 -0700
Subject: [PATCH 223/294] =?UTF-8?q?Don=E2=80=99t=20use=20`execvp`=20when?=
 =?UTF-8?q?=20we=20know=20the=20path?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/nix/develop.cc |  2 +-
 src/nix/fmt.cc     |  2 +-
 src/nix/run.cc     | 10 +++++++---
 src/nix/run.hh     |  1 +
 4 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 38482ed42..ae9be79a3 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -662,7 +662,7 @@ struct CmdDevelop : Common, MixEnvironment
             }
         }
 
-        runProgramInStore(store, shell, args, buildEnvironment.getSystem());
+        runProgramInStore(store, true, shell, args, buildEnvironment.getSystem());
     }
 };
 
diff --git a/src/nix/fmt.cc b/src/nix/fmt.cc
index c85eacded..396c93dbb 100644
--- a/src/nix/fmt.cc
+++ b/src/nix/fmt.cc
@@ -49,7 +49,7 @@ struct CmdFmt : SourceExprCommand {
             }
         }
 
-        runProgramInStore(store, app.program, programArgs);
+        runProgramInStore(store, false, app.program, programArgs);
     };
 };
 
diff --git a/src/nix/run.cc b/src/nix/run.cc
index ea0a17897..d531f712d 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -25,6 +25,7 @@ std::string chrootHelperName = "__run_in_chroot";
 namespace nix {
 
 void runProgramInStore(ref<Store> store,
+    bool search,
     const std::string & program,
     const Strings & args,
     std::optional<std::string_view> system)
@@ -58,7 +59,10 @@ void runProgramInStore(ref<Store> store,
     if (system)
         setPersonality(*system);
 
-    execvp(program.c_str(), stringsToCharPtrs(args).data());
+    if (search)
+        execvp(program.c_str(), stringsToCharPtrs(args).data());
+    else
+        execv(program.c_str(), stringsToCharPtrs(args).data());
 
     throw SysError("unable to execute '%s'", program);
 }
@@ -132,7 +136,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment
         Strings args;
         for (auto & arg : command) args.push_back(arg);
 
-        runProgramInStore(store, *command.begin(), args);
+        runProgramInStore(store, true, *command.begin(), args);
     }
 };
 
@@ -194,7 +198,7 @@ struct CmdRun : InstallableValueCommand
         Strings allArgs{app.program};
         for (auto & i : args) allArgs.push_back(i);
 
-        runProgramInStore(store, app.program, allArgs);
+        runProgramInStore(store, false, app.program, allArgs);
     }
 };
 
diff --git a/src/nix/run.hh b/src/nix/run.hh
index 97ddef19b..c62287e7e 100644
--- a/src/nix/run.hh
+++ b/src/nix/run.hh
@@ -6,6 +6,7 @@
 namespace nix {
 
 void runProgramInStore(ref<Store> store,
+    bool search,
     const std::string & program,
     const Strings & args,
     std::optional<std::string_view> system = std::nullopt);

From f99e468640ab0eac7f07ac2b328222eb45dee8d8 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 27 Nov 2023 08:48:11 -0500
Subject: [PATCH 224/294] Avoid `<name>/<name>` in documentation URLs

They are redundant and look weird.
---
 doc/manual/_redirects                          | 18 ++++++++++++++----
 doc/manual/src/SUMMARY.md.in                   | 14 +++++++-------
 .../{advanced-topics.md => index.md}           |  0
 doc/manual/src/architecture/architecture.md    |  2 +-
 .../command-ref/{command-ref.md => index.md}   |  0
 doc/manual/src/contributing/hacking.md         |  2 +-
 .../contributing/{contributing.md => index.md} |  0
 .../installation/{installation.md => index.md} |  0
 .../{package-management.md => index.md}        |  0
 .../src/protocols/{protocols.md => index.md}   |  0
 doc/manual/src/quick-start.md                  |  2 +-
 .../{release-notes.md => index.md}             |  0
 12 files changed, 24 insertions(+), 14 deletions(-)
 rename doc/manual/src/advanced-topics/{advanced-topics.md => index.md} (100%)
 rename doc/manual/src/command-ref/{command-ref.md => index.md} (100%)
 rename doc/manual/src/contributing/{contributing.md => index.md} (100%)
 rename doc/manual/src/installation/{installation.md => index.md} (100%)
 rename doc/manual/src/package-management/{package-management.md => index.md} (100%)
 rename doc/manual/src/protocols/{protocols.md => index.md} (100%)
 rename doc/manual/src/release-notes/{release-notes.md => index.md} (100%)

diff --git a/doc/manual/_redirects b/doc/manual/_redirects
index 4ea289d86..2038671d7 100644
--- a/doc/manual/_redirects
+++ b/doc/manual/_redirects
@@ -13,18 +13,28 @@
 # conventions:
 # - always force (<CODE>!) since this allows re-using file names
 # - group related paths to ease readability
-# - always append new redirects to the end of the file
+# - keep in alphabetical/wildcards-last order, which will reduce version control conflicts
 # - redirects that should have been there but are missing can be inserted where they belong
 
+/advanced-topics/advanced-topics /advanced-topics 301!
+
+/command-ref/command-ref /command-ref 301!
+
+/contributing/contributing /contributing 301!
+
 /expressions/expression-language /language/ 301!
-/expressions/language-values /language/values 301!
 /expressions/language-constructs /language/constructs 301!
 /expressions/language-operators /language/operators 301!
+/expressions/language-values /language/values 301!
 /expressions/* /language/:splat 301!
 
+/installation/installation /installation 301!
+
 /package-management/basic-package-mgmt /command-ref/nix-env 301!
-
 /package-management/channels* /command-ref/nix-channel 301!
-
+/package-management/package-management /package-management 301!
 /package-management/s3-substituter* /command-ref/new-cli/nix3-help-stores#s3-binary-cache-store 301!
 
+/protocols/protocols /protocols 301!
+
+/release-notes/release-notes /release-notes 301!
diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in
index 8e7b4eeab..686d3e8d7 100644
--- a/doc/manual/src/SUMMARY.md.in
+++ b/doc/manual/src/SUMMARY.md.in
@@ -2,7 +2,7 @@
 
 - [Introduction](introduction.md)
 - [Quick Start](quick-start.md)
-- [Installation](installation/installation.md)
+- [Installation](installation/index.md)
   - [Supported Platforms](installation/supported-platforms.md)
   - [Installing a Binary Distribution](installation/installing-binary.md)
   - [Installing Nix from Source](installation/installing-source.md)
@@ -31,11 +31,11 @@
     - [Import From Derivation](language/import-from-derivation.md)
   - [Built-in Constants](language/builtin-constants.md)
   - [Built-in Functions](language/builtins.md)
-- [Package Management](package-management/package-management.md)
+- [Package Management](package-management/index.md)
   - [Profiles](package-management/profiles.md)
   - [Garbage Collection](package-management/garbage-collection.md)
     - [Garbage Collector Roots](package-management/garbage-collector-roots.md)
-- [Advanced Topics](advanced-topics/advanced-topics.md)
+- [Advanced Topics](advanced-topics/index.md)
   - [Sharing Packages Between Machines](package-management/sharing-packages.md)
     - [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md)
     - [Copying Closures via SSH](package-management/copy-closure.md)
@@ -45,7 +45,7 @@
   - [Tuning Cores and Jobs](advanced-topics/cores-vs-jobs.md)
   - [Verifying Build Reproducibility](advanced-topics/diff-hook.md)
   - [Using the `post-build-hook`](advanced-topics/post-build-hook.md)
-- [Command Reference](command-ref/command-ref.md)
+- [Command Reference](command-ref/index.md)
   - [Common Options](command-ref/opt-common.md)
   - [Common Environment Variables](command-ref/env-common.md)
   - [Main Commands](command-ref/main-commands.md)
@@ -102,18 +102,18 @@
     - [Channels](command-ref/files/channels.md)
     - [Default Nix expression](command-ref/files/default-nix-expression.md)
 - [Architecture and Design](architecture/architecture.md)
-- [Protocols](protocols/protocols.md)
+- [Protocols](protocols/index.md)
   - [Serving Tarball Flakes](protocols/tarball-fetcher.md)
   - [Derivation "ATerm" file format](protocols/derivation-aterm.md)
 - [Glossary](glossary.md)
-- [Contributing](contributing/contributing.md)
+- [Contributing](contributing/index.md)
   - [Hacking](contributing/hacking.md)
   - [Testing](contributing/testing.md)
   - [Documentation](contributing/documentation.md)
   - [Experimental Features](contributing/experimental-features.md)
   - [CLI guideline](contributing/cli-guideline.md)
   - [C++ style guide](contributing/cxx.md)
-- [Release Notes](release-notes/release-notes.md)
+- [Release Notes](release-notes/index.md)
 {{#include ./SUMMARY-rl-next.md}}
   - [Release 2.19 (2023-11-17)](release-notes/rl-2.19.md)
   - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md)
diff --git a/doc/manual/src/advanced-topics/advanced-topics.md b/doc/manual/src/advanced-topics/index.md
similarity index 100%
rename from doc/manual/src/advanced-topics/advanced-topics.md
rename to doc/manual/src/advanced-topics/index.md
diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md
index 79429508f..2fec4ed20 100644
--- a/doc/manual/src/architecture/architecture.md
+++ b/doc/manual/src/architecture/architecture.md
@@ -52,7 +52,7 @@ The following [concept map] shows its main components (rectangles), the objects
                                             '---------------'
 ```
 
-At the top is the [command line interface](../command-ref/command-ref.md) that drives the underlying layers.
+At the top is the [command line interface](../command-ref/index.md) that drives the underlying layers.
 
 The [Nix language](../language/index.md) evaluator transforms Nix expressions into self-contained *build plans*, which are used to derive *build results* from referenced *build inputs*.
 
diff --git a/doc/manual/src/command-ref/command-ref.md b/doc/manual/src/command-ref/index.md
similarity index 100%
rename from doc/manual/src/command-ref/command-ref.md
rename to doc/manual/src/command-ref/index.md
diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md
index 0a95334f7..9de5ad39b 100644
--- a/doc/manual/src/contributing/hacking.md
+++ b/doc/manual/src/contributing/hacking.md
@@ -10,7 +10,7 @@ $ cd nix
 
 The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. If you don't have it installed, follow the [installation instructions].
 
-[installation instructions]: ../installation/installation.md
+[installation instructions]: ../installation/index.md
 
 ## Building Nix with flakes
 
diff --git a/doc/manual/src/contributing/contributing.md b/doc/manual/src/contributing/index.md
similarity index 100%
rename from doc/manual/src/contributing/contributing.md
rename to doc/manual/src/contributing/index.md
diff --git a/doc/manual/src/installation/installation.md b/doc/manual/src/installation/index.md
similarity index 100%
rename from doc/manual/src/installation/installation.md
rename to doc/manual/src/installation/index.md
diff --git a/doc/manual/src/package-management/package-management.md b/doc/manual/src/package-management/index.md
similarity index 100%
rename from doc/manual/src/package-management/package-management.md
rename to doc/manual/src/package-management/index.md
diff --git a/doc/manual/src/protocols/protocols.md b/doc/manual/src/protocols/index.md
similarity index 100%
rename from doc/manual/src/protocols/protocols.md
rename to doc/manual/src/protocols/index.md
diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md
index 1d2688ede..5f54abbde 100644
--- a/doc/manual/src/quick-start.md
+++ b/doc/manual/src/quick-start.md
@@ -13,7 +13,7 @@ to subsequent chapters.
    The install script will use `sudo`, so make sure you have sufficient rights.
    On Linux, `--daemon` can be omitted for a single-user install.
 
-   For other installation methods, see [here](installation/installation.md).
+   For other installation methods, see [here](installation/index.md).
 
 1. See what installable packages are currently available in the
    channel:
diff --git a/doc/manual/src/release-notes/release-notes.md b/doc/manual/src/release-notes/index.md
similarity index 100%
rename from doc/manual/src/release-notes/release-notes.md
rename to doc/manual/src/release-notes/index.md

From ea95327e72f5781295417b0eae46a5e351bebebd Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 30 Nov 2023 16:16:17 +0100
Subject: [PATCH 225/294] Move restricted/pure-eval access control out of the
 evaluator and into the accessor

---
 src/libcmd/installables.cc             |   7 +-
 src/libexpr/eval.cc                    | 103 +++++----------------
 src/libexpr/eval.hh                    |  25 +++---
 src/libexpr/parser.y                   |  19 +++-
 src/libexpr/primops.cc                 | 119 +++++++++++--------------
 src/nix-build/nix-build.cc             |   7 +-
 src/nix-instantiate/nix-instantiate.cc |   2 +-
 tests/functional/restricted.sh         |   4 +-
 8 files changed, 115 insertions(+), 171 deletions(-)

diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index 6e670efea..6b3c82374 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -260,9 +260,10 @@ void SourceExprCommand::completeInstallable(AddCompletions & completions, std::s
 
             evalSettings.pureEval = false;
             auto state = getEvalState();
-            Expr *e = state->parseExprFromFile(
-                resolveExprPath(state->checkSourcePath(lookupFileArg(*state, *file)))
-                );
+            auto e =
+                state->parseExprFromFile(
+                    resolveExprPath(
+                        lookupFileArg(*state, *file)));
 
             Value root;
             state->eval(e, root);
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 7e68e6f9b..23ac349fe 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -509,7 +509,18 @@ EvalState::EvalState(
     , sOutputSpecified(symbols.create("outputSpecified"))
     , repair(NoRepair)
     , emptyBindings(0)
-    , rootFS(makeFSInputAccessor(CanonPath::root))
+    , rootFS(
+        makeFSInputAccessor(
+            CanonPath::root,
+            evalSettings.restrictEval || evalSettings.pureEval
+            ? std::optional<std::set<CanonPath>>(std::set<CanonPath>())
+            : std::nullopt,
+            [](const CanonPath & path) -> RestrictedPathError {
+                auto modeInformation = evalSettings.pureEval
+                    ? "in pure evaluation mode (use '--impure' to override)"
+                    : "in restricted mode";
+                throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation);
+            }))
     , corepkgsFS(makeMemoryInputAccessor())
     , internalFS(makeMemoryInputAccessor())
     , derivationInternal{corepkgsFS->addFile(
@@ -551,28 +562,10 @@ EvalState::EvalState(
             searchPath.elements.emplace_back(SearchPath::Elem::parse(i));
     }
 
-    if (evalSettings.restrictEval || evalSettings.pureEval) {
-        allowedPaths = PathSet();
-
-        for (auto & i : searchPath.elements) {
-            auto r = resolveSearchPathPath(i.path);
-            if (!r) continue;
-
-            auto path = std::move(*r);
-
-            if (store->isInStore(path)) {
-                try {
-                    StorePathSet closure;
-                    store->computeFSClosure(store->toStorePath(path).first, closure);
-                    for (auto & path : closure)
-                        allowPath(path);
-                } catch (InvalidPath &) {
-                    allowPath(path);
-                }
-            } else
-                allowPath(path);
-        }
-    }
+    /* Allow access to all paths in the search path. */
+    if (rootFS->hasAccessControl())
+        for (auto & i : searchPath.elements)
+            resolveSearchPathPath(i.path, true);
 
     corepkgsFS->addFile(
         CanonPath("fetchurl.nix"),
@@ -590,14 +583,12 @@ EvalState::~EvalState()
 
 void EvalState::allowPath(const Path & path)
 {
-    if (allowedPaths)
-        allowedPaths->insert(path);
+    rootFS->allowPath(CanonPath(path));
 }
 
 void EvalState::allowPath(const StorePath & storePath)
 {
-    if (allowedPaths)
-        allowedPaths->insert(store->toRealPath(storePath));
+    rootFS->allowPath(CanonPath(store->toRealPath(storePath)));
 }
 
 void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & v)
@@ -607,54 +598,6 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value &
     mkStorePathString(storePath, v);
 }
 
-SourcePath EvalState::checkSourcePath(const SourcePath & path_)
-{
-    // Don't check non-rootFS accessors, they're in a different namespace.
-    if (path_.accessor != ref<InputAccessor>(rootFS)) return path_;
-
-    if (!allowedPaths) return path_;
-
-    auto i = resolvedPaths.find(path_.path.abs());
-    if (i != resolvedPaths.end())
-        return i->second;
-
-    bool found = false;
-
-    /* First canonicalize the path without symlinks, so we make sure an
-     * attacker can't append ../../... to a path that would be in allowedPaths
-     * and thus leak symlink targets.
-     */
-    Path abspath = canonPath(path_.path.abs());
-
-    for (auto & i : *allowedPaths) {
-        if (isDirOrInDir(abspath, i)) {
-            found = true;
-            break;
-        }
-    }
-
-    if (!found) {
-        auto modeInformation = evalSettings.pureEval
-            ? "in pure eval mode (use '--impure' to override)"
-            : "in restricted mode";
-        throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", abspath, modeInformation);
-    }
-
-    /* Resolve symlinks. */
-    debug("checking access to '%s'", abspath);
-    SourcePath path = rootPath(CanonPath(canonPath(abspath, true)));
-
-    for (auto & i : *allowedPaths) {
-        if (isDirOrInDir(path.path.abs(), i)) {
-            resolvedPaths.insert_or_assign(path_.path.abs(), path);
-            return path;
-        }
-    }
-
-    throw RestrictedPathError("access to canonical path '%1%' is forbidden in restricted mode", path);
-}
-
-
 void EvalState::checkURI(const std::string & uri)
 {
     if (!evalSettings.restrictEval) return;
@@ -674,12 +617,12 @@ void EvalState::checkURI(const std::string & uri)
     /* If the URI is a path, then check it against allowedPaths as
        well. */
     if (hasPrefix(uri, "/")) {
-        checkSourcePath(rootPath(CanonPath(uri)));
+        rootFS->checkAllowed(CanonPath(uri));
         return;
     }
 
     if (hasPrefix(uri, "file://")) {
-        checkSourcePath(rootPath(CanonPath(std::string(uri, 7))));
+        rootFS->checkAllowed(CanonPath(uri.substr(7)));
         return;
     }
 
@@ -1181,10 +1124,8 @@ Value * ExprPath::maybeThunk(EvalState & state, Env & env)
 }
 
 
-void EvalState::evalFile(const SourcePath & path_, Value & v, bool mustBeTrivial)
+void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial)
 {
-    auto path = checkSourcePath(path_);
-
     FileEvalCache::iterator i;
     if ((i = fileEvalCache.find(path)) != fileEvalCache.end()) {
         v = i->second;
@@ -1205,7 +1146,7 @@ void EvalState::evalFile(const SourcePath & path_, Value & v, bool mustBeTrivial
         e = j->second;
 
     if (!e)
-        e = parseExprFromFile(checkSourcePath(resolvedPath));
+        e = parseExprFromFile(resolvedPath);
 
     fileParseCache[resolvedPath] = e;
 
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 9a92992c1..ee7bdda0d 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -217,12 +217,6 @@ public:
      */
     RepairFlag repair;
 
-    /**
-     * The allowed filesystem paths in restricted or pure evaluation
-     * mode.
-     */
-    std::optional<PathSet> allowedPaths;
-
     Bindings emptyBindings;
 
     /**
@@ -396,12 +390,6 @@ public:
      */
     void allowAndSetStorePathString(const StorePath & storePath, Value & v);
 
-    /**
-     * Check whether access to a path is allowed and throw an error if
-     * not. Otherwise return the canonicalised path.
-     */
-    SourcePath checkSourcePath(const SourcePath & path);
-
     void checkURI(const std::string & uri);
 
     /**
@@ -445,13 +433,15 @@ public:
     SourcePath findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos);
 
     /**
-     * Try to resolve a search path value (not the optional key part)
+     * Try to resolve a search path value (not the optional key part).
      *
      * If the specified search path element is a URI, download it.
      *
      * If it is not found, return `std::nullopt`
      */
-    std::optional<std::string> resolveSearchPathPath(const SearchPath::Path & path);
+    std::optional<std::string> resolveSearchPathPath(
+        const SearchPath::Path & elem,
+        bool initAccessControl = false);
 
     /**
      * Evaluate an expression to normal form
@@ -756,6 +746,13 @@ public:
      */
     [[nodiscard]] StringMap realiseContext(const NixStringContext & context);
 
+    /* Call the binary path filter predicate used builtins.path etc. */
+    bool callPathFilter(
+        Value * filterFun,
+        const SourcePath & path,
+        std::string_view pathArg,
+        PosIdx pos);
+
 private:
 
     /**
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index f6cf1f689..58fc580fc 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -783,7 +783,7 @@ SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_
 }
 
 
-std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Path & value0)
+std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Path & value0, bool initAccessControl)
 {
     auto & value = value0.s;
     auto i = searchPathResolved.find(value);
@@ -800,7 +800,6 @@ std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Pa
             logWarning({
                 .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value)
             });
-            res = std::nullopt;
         }
     }
 
@@ -814,6 +813,20 @@ std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Pa
 
     else {
         auto path = absPath(value);
+
+        /* Allow access to paths in the search path. */
+        if (initAccessControl) {
+            allowPath(path);
+            if (store->isInStore(path)) {
+                try {
+                    StorePathSet closure;
+                    store->computeFSClosure(store->toStorePath(path).first, closure);
+                    for (auto & p : closure)
+                        allowPath(p);
+                } catch (InvalidPath &) { }
+            }
+        }
+
         if (pathExists(path))
             res = { path };
         else {
@@ -829,7 +842,7 @@ std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Pa
     else
         debug("failed to resolve search path element '%s'", value);
 
-    searchPathResolved[value] = res;
+    searchPathResolved.emplace(value, res);
     return res;
 }
 
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index ebf2549e4..0f7706563 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -15,6 +15,7 @@
 #include "value-to-json.hh"
 #include "value-to-xml.hh"
 #include "primops.hh"
+#include "fs-input-accessor.hh"
 
 #include <boost/container/small_vector.hpp>
 #include <boost/regex.hpp>
@@ -90,8 +91,8 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
         for (auto & [outputName, outputPath] : outputs) {
             /* Add the output of this derivations to the allowed
                paths. */
-            if (allowedPaths) {
-                allowPath(outputPath);
+            if (rootFS->hasAccessControl()) {
+                allowPath(store->toRealPath(outputPath));
             }
             /* Get all the output paths corresponding to the placeholders we had */
             if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
@@ -110,27 +111,19 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
     return res;
 }
 
-struct RealisePathFlags {
-    // Whether to check that the path is allowed in pure eval mode
-    bool checkForPureEval = true;
-};
-
-static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, const RealisePathFlags flags = {})
+static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v)
 {
     NixStringContext context;
 
     auto path = state.coerceToPath(noPos, v, context, "while realising the context of a path");
 
     try {
-        if (!context.empty()) {
+        if (!context.empty() && path.accessor == state.rootFS) {
             auto rewrites = state.realiseContext(context);
             auto realPath = state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context);
             return {path.accessor, CanonPath(realPath)};
-        }
-
-        return flags.checkForPureEval
-            ? state.checkSourcePath(path)
-            : path;
+        } else
+            return path;
     } catch (Error & e) {
         e.addTrace(state.positions[pos], "while realising the context of path '%s'", path);
         throw;
@@ -1493,7 +1486,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args,
         }));
 
     NixStringContext context;
-    auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'")).path;
+    auto path = state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'").path;
     /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink
        directly in the store.  The latter condition is necessary so
        e.g. nix-push does the right thing. */
@@ -1535,12 +1528,7 @@ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args,
 {
     auto & arg = *args[0];
 
-    /* We don’t check the path right now, because we don’t want to
-       throw if the path isn’t allowed, but just return false (and we
-       can’t just catch the exception here because we still want to
-       throw if something in the evaluation of `arg` tries to
-       access an unauthorized path). */
-    auto path = realisePath(state, pos, arg, { .checkForPureEval = false });
+    auto path = realisePath(state, pos, arg);
 
     /* SourcePath doesn't know about trailing slash. */
     auto mustBeDir = arg.type() == nString
@@ -1548,14 +1536,9 @@ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args,
             || arg.string_view().ends_with("/."));
 
     try {
-        auto checked = state.checkSourcePath(path);
-        auto st = checked.maybeLstat();
+        auto st = path.maybeLstat();
         auto exists = st && (!mustBeDir || st->type == SourceAccessor::tDirectory);
         v.mkBool(exists);
-    } catch (SysError & e) {
-        /* Don't give away info from errors while canonicalising
-           ‘path’ in restricted mode. */
-        v.mkBool(false);
     } catch (RestrictedPathError & e) {
         v.mkBool(false);
     }
@@ -1699,7 +1682,7 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V
 
     auto path = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.findFile");
 
-    v.mkPath(state.checkSourcePath(state.findFile(searchPath, path, pos)));
+    v.mkPath(state.findFile(searchPath, path, pos));
 }
 
 static RegisterPrimOp primop_findFile(PrimOp {
@@ -2178,11 +2161,35 @@ static RegisterPrimOp primop_toFile({
     .fun = prim_toFile,
 });
 
+bool EvalState::callPathFilter(
+    Value * filterFun,
+    const SourcePath & path,
+    std::string_view pathArg,
+    PosIdx pos)
+{
+    auto st = path.lstat();
+
+    /* Call the filter function.  The first argument is the path, the
+       second is a string indicating the type of the file. */
+    Value arg1;
+    arg1.mkString(pathArg);
+
+    Value arg2;
+    // assert that type is not "unknown"
+    arg2.mkString(fileTypeToString(st.type));
+
+    Value * args []{&arg1, &arg2};
+    Value res;
+    callFunction(*filterFun, 2, args, res, pos);
+
+    return forceBool(res, pos, "while evaluating the return value of the path filter function");
+}
+
 static void addPath(
     EvalState & state,
     const PosIdx pos,
     std::string_view name,
-    Path path,
+    SourcePath path,
     Value * filterFun,
     FileIngestionMethod method,
     const std::optional<Hash> expectedHash,
@@ -2190,48 +2197,29 @@ static void addPath(
     const NixStringContext & context)
 {
     try {
-        // FIXME: handle CA derivation outputs (where path needs to
-        // be rewritten to the actual output).
-        auto rewrites = state.realiseContext(context);
-        path = state.toRealPath(rewriteStrings(path, rewrites), context);
-
         StorePathSet refs;
 
-        if (state.store->isInStore(path)) {
+        if (path.accessor == state.rootFS && state.store->isInStore(path.path.abs())) {
+            // FIXME: handle CA derivation outputs (where path needs to
+            // be rewritten to the actual output).
+            auto rewrites = state.realiseContext(context);
+            path = {state.rootFS, CanonPath(state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context))};
+
             try {
-                auto [storePath, subPath] = state.store->toStorePath(path);
+                auto [storePath, subPath] = state.store->toStorePath(path.path.abs());
                 // FIXME: we should scanForReferences on the path before adding it
                 refs = state.store->queryPathInfo(storePath)->references;
-                path = state.store->toRealPath(storePath) + subPath;
+                path = {state.rootFS, CanonPath(state.store->toRealPath(storePath) + subPath)};
             } catch (Error &) { // FIXME: should be InvalidPathError
             }
         }
 
-        path = evalSettings.pureEval && expectedHash
-            ? path
-            : state.checkSourcePath(state.rootPath(CanonPath(path))).path.abs();
-
-        PathFilter filter = filterFun ? ([&](const Path & path) {
-            auto st = lstat(path);
-
-            /* Call the filter function.  The first argument is the path,
-               the second is a string indicating the type of the file. */
-            Value arg1;
-            arg1.mkString(path);
-
-            Value arg2;
-            arg2.mkString(
-                S_ISREG(st.st_mode) ? "regular" :
-                S_ISDIR(st.st_mode) ? "directory" :
-                S_ISLNK(st.st_mode) ? "symlink" :
-                "unknown" /* not supported, will fail! */);
-
-            Value * args []{&arg1, &arg2};
-            Value res;
-            state.callFunction(*filterFun, 2, args, res, pos);
-
-            return state.forceBool(res, pos, "while evaluating the return value of the path filter function");
-        }) : defaultPathFilter;
+        std::unique_ptr<PathFilter> filter;
+        if (filterFun)
+            filter = std::make_unique<PathFilter>([&](const Path & p) {
+                auto p2 = CanonPath(p);
+                return state.callPathFilter(filterFun, {path.accessor, p2}, p2.abs(), pos);
+            });
 
         std::optional<StorePath> expectedStorePath;
         if (expectedHash)
@@ -2242,7 +2230,7 @@ static void addPath(
             });
 
         if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
-            auto dstPath = state.rootPath(CanonPath(path)).fetchToStore(state.store, name, method, &filter, state.repair);
+            auto dstPath = path.fetchToStore(state.store, name, method, filter.get(), state.repair);
             if (expectedHash && expectedStorePath != dstPath)
                 state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path));
             state.allowAndSetStorePathString(dstPath, v);
@@ -2261,7 +2249,8 @@ static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * arg
     auto path = state.coerceToPath(pos, *args[1], context,
         "while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'");
     state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource");
-    addPath(state, pos, path.baseName(), path.path.abs(), args[0], FileIngestionMethod::Recursive, std::nullopt, v, context);
+
+    addPath(state, pos, path.baseName(), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v, context);
 }
 
 static RegisterPrimOp primop_filterSource({
@@ -2356,7 +2345,7 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value
     if (name.empty())
         name = path->baseName();
 
-    addPath(state, pos, name, path->path.abs(), filterFun, method, expectedHash, v, context);
+    addPath(state, pos, name, *path, filterFun, method, expectedHash, v, context);
 }
 
 static RegisterPrimOp primop_path({
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 75ce12a8c..e2986bfe0 100644
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -310,8 +310,11 @@ static void main_nix_build(int argc, char * * argv)
                 else
                     /* If we're in a #! script, interpret filenames
                        relative to the script. */
-                    exprs.push_back(state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state,
-                                        inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i)))));
+                    exprs.push_back(
+                        state->parseExprFromFile(
+                            resolveExprPath(
+                                lookupFileArg(*state,
+                                    inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i))));
             }
         }
 
diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc
index c67409e89..86b9be17d 100644
--- a/src/nix-instantiate/nix-instantiate.cc
+++ b/src/nix-instantiate/nix-instantiate.cc
@@ -183,7 +183,7 @@ static int main_nix_instantiate(int argc, char * * argv)
         for (auto & i : files) {
             Expr * e = fromArgs
                 ? state->parseExprFromString(i, state->rootPath(CanonPath::fromCwd()))
-                : state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state, i))));
+                : state->parseExprFromFile(resolveExprPath(lookupFileArg(*state, i)));
             processExpr(*state, attrPaths, parseOnly, strict, autoArgs,
                 evalOnly, outputKind, xmlOutputSourceLocation, e);
         }
diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh
index 197ae7a10..b8deceacc 100644
--- a/tests/functional/restricted.sh
+++ b/tests/functional/restricted.sh
@@ -14,8 +14,8 @@ nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I sr
 (! nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel')
 nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel' -I src=../../src
 
-(! nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in <foo>')
-nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in <foo>' -I src=.
+(! nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile <foo/simple.nix>')
+nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile <foo/simple.nix>' -I src=.
 
 p=$(nix eval --raw --expr "builtins.fetchurl file://$(pwd)/restricted.sh" --impure --restrict-eval --allowed-uris "file://$(pwd)")
 cmp $p restricted.sh

From 305939655a6cd680997981ca6077d4ce7f957984 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 30 Nov 2023 16:28:33 +0100
Subject: [PATCH 226/294] Remove superfluous use of hasAccessControl()

---
 src/libexpr/primops.cc | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 0f7706563..c442de986 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -91,9 +91,8 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
         for (auto & [outputName, outputPath] : outputs) {
             /* Add the output of this derivations to the allowed
                paths. */
-            if (rootFS->hasAccessControl()) {
-                allowPath(store->toRealPath(outputPath));
-            }
+            allowPath(store->toRealPath(outputPath));
+
             /* Get all the output paths corresponding to the placeholders we had */
             if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
                 res.insert_or_assign(

From 43d9fb6cf180c421be17b4247f5dd032cf4843f5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 30 Nov 2023 16:44:54 +0100
Subject: [PATCH 227/294] Remove InputAccessor::root()

---
 src/libexpr/value.hh              | 7 +++----
 src/libfetchers/fetchers.cc       | 2 +-
 src/libfetchers/input-accessor.cc | 7 +------
 src/libfetchers/input-accessor.hh | 7 +++++--
 4 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index bcff8ae55..72a3a2b32 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -424,10 +424,9 @@ public:
     SourcePath path() const
     {
         assert(internalType == tPath);
-        return SourcePath {
-            .accessor = ref(_path.accessor->shared_from_this()),
-            .path = CanonPath(CanonPath::unchecked_t(), _path.path)
-        };
+        return SourcePath(
+            ref(_path.accessor->shared_from_this()),
+            CanonPath(CanonPath::unchecked_t(), _path.path));
     }
 
     std::string_view string_view() const
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index 60208619e..5fd9e069f 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -374,7 +374,7 @@ void InputScheme::clone(const Input & input, const Path & destDir) const
 std::pair<StorePath, Input> InputScheme::fetch(ref<Store> store, const Input & input)
 {
     auto [accessor, input2] = getAccessor(store, input);
-    auto storePath = accessor->root().fetchToStore(store, input2.getName());
+    auto storePath = SourcePath(accessor).fetchToStore(store, input2.getName());
     return {storePath, input2};
 }
 
diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc
index 85dc4609f..f54a5a6fd 100644
--- a/src/libfetchers/input-accessor.cc
+++ b/src/libfetchers/input-accessor.cc
@@ -53,11 +53,6 @@ StorePath InputAccessor::fetchToStore(
     return storePath;
 }
 
-SourcePath InputAccessor::root()
-{
-    return {ref(shared_from_this()), CanonPath::root};
-}
-
 std::ostream & operator << (std::ostream & str, const SourcePath & path)
 {
     str << path.to_string();
@@ -88,7 +83,7 @@ SourcePath SourcePath::parent() const
 
 SourcePath SourcePath::resolveSymlinks() const
 {
-    auto res = accessor->root();
+    auto res = SourcePath(accessor);
 
     int linksAllowed = 1024;
 
diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh
index 26d17f064..d5ac238b1 100644
--- a/src/libfetchers/input-accessor.hh
+++ b/src/libfetchers/input-accessor.hh
@@ -36,8 +36,6 @@ struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this<Inpu
         FileIngestionMethod method = FileIngestionMethod::Recursive,
         PathFilter * filter = nullptr,
         RepairFlag repair = NoRepair);
-
-    SourcePath root();
 };
 
 /**
@@ -51,6 +49,11 @@ struct SourcePath
     ref<InputAccessor> accessor;
     CanonPath path;
 
+    SourcePath(ref<InputAccessor> accessor, CanonPath path = CanonPath::root)
+        : accessor(std::move(accessor))
+        , path(std::move(path))
+    { }
+
     std::string_view baseName() const;
 
     /**

From 44d21f6ef9783bed8812d39ff7b1a28a4883f84b Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Thu, 30 Nov 2023 20:36:50 +0100
Subject: [PATCH 228/294] keep generated documentation in a separate directory

- helps navigating the code as it highlights which files are generated
- makes it less error prone when working incrementally
  (although this should be just fixed by building out of tree)
---
 .gitignore                                    |  2 +-
 .../manual/src/store/types/index.md.in        |  4 ----
 src/nix/local.mk                              | 19 ++++++++++++++++---
 src/nix/main.cc                               |  2 +-
 src/nix/profile.md                            |  2 +-
 5 files changed, 19 insertions(+), 10 deletions(-)
 rename src/nix/help-stores.md => doc/manual/src/store/types/index.md.in (99%)

diff --git a/.gitignore b/.gitignore
index 13a4dfb75..bcf9c4a01 100644
--- a/.gitignore
+++ b/.gitignore
@@ -54,7 +54,7 @@ perl/Makefile.config
 
 /src/nix/nix
 
-/src/nix/doc
+/src/nix/generated-doc
 
 # /src/nix-env/
 /src/nix-env/nix-env
diff --git a/src/nix/help-stores.md b/doc/manual/src/store/types/index.md.in
similarity index 99%
rename from src/nix/help-stores.md
rename to doc/manual/src/store/types/index.md.in
index 47ba9b94d..bb166a1fc 100644
--- a/src/nix/help-stores.md
+++ b/doc/manual/src/store/types/index.md.in
@@ -1,5 +1,3 @@
-R"(
-
 Nix supports different types of stores. These are described below.
 
 ## Store URL format
@@ -42,5 +40,3 @@ store as follows:
 * Otherwise, use the [local store](#local-store) `/nix/store`.
 
 @stores@
-
-)"
diff --git a/src/nix/local.mk b/src/nix/local.mk
index 57f8259c4..a21aa705f 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -31,10 +31,23 @@ src/nix/develop.cc: src/nix/get-env.sh.gen.hh
 
 src/nix-channel/nix-channel.cc: src/nix-channel/unpack-channel.nix.gen.hh
 
-src/nix/main.cc: doc/manual/generate-manpage.nix.gen.hh doc/manual/utils.nix.gen.hh doc/manual/generate-settings.nix.gen.hh doc/manual/generate-store-info.nix.gen.hh
+src/nix/main.cc: \
+  doc/manual/generate-manpage.nix.gen.hh \
+  doc/manual/utils.nix.gen.hh doc/manual/generate-settings.nix.gen.hh \
+  doc/manual/generate-store-info.nix.gen.hh \
+  src/nix/generated-doc/help-stores.md
 
-src/nix/doc/files/%.md: doc/manual/src/command-ref/files/%.md
+src/nix/generated-doc/files/%.md: doc/manual/src/command-ref/files/%.md
 	@mkdir -p $$(dirname $@)
 	@cp $< $@
 
-src/nix/profile.cc: src/nix/profile.md src/nix/doc/files/profiles.md.gen.hh
+src/nix/profile.cc: src/nix/profile.md src/nix/generated-doc/files/profiles.md.gen.hh
+
+src/nix/generated-doc/help-stores.md: doc/manual/src/store/types/index.md.in
+	@mkdir -p $$(dirname $@)
+	@echo 'R"(' >> $@.tmp
+	@echo >> $@.tmp
+	@cat $^ >> $@.tmp
+	@echo >> $@.tmp
+	@echo ')"' >> $@.tmp
+	@mv $@.tmp $@
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 2a6c2f478..49e637fb0 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -297,7 +297,7 @@ struct CmdHelpStores : Command
     std::string doc() override
     {
         return
-          #include "help-stores.md"
+          #include "generated-doc/help-stores.md"
           ;
     }
 
diff --git a/src/nix/profile.md b/src/nix/profile.md
index bd13f906f..9b2f86f4a 100644
--- a/src/nix/profile.md
+++ b/src/nix/profile.md
@@ -11,7 +11,7 @@ them to be rolled back easily.
 
 )""
 
-#include "doc/files/profiles.md.gen.hh"
+#include "generated-doc/files/profiles.md.gen.hh"
 
 R""(
 

From d5ffc94f336fc4032dd4009c14c148e390e10e16 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Thu, 30 Nov 2023 21:41:47 +0100
Subject: [PATCH 229/294] use lookup paths in helper expressions consistently

this makes the files in question a bit more independent of source location.

to find where the value is set and how it's wired up:

    rg nix=doc/manual
---
 doc/manual/generate-builtin-constants.nix     |  2 +-
 doc/manual/generate-builtins.nix              |  2 +-
 doc/manual/generate-manpage.nix               | 30 +++++++++++++++----
 doc/manual/generate-settings.nix              |  2 +-
 doc/manual/generate-store-info.nix            |  4 +--
 doc/manual/generate-xp-features-shortlist.nix |  2 +-
 doc/manual/generate-xp-features.nix           |  2 +-
 7 files changed, 32 insertions(+), 12 deletions(-)

diff --git a/doc/manual/generate-builtin-constants.nix b/doc/manual/generate-builtin-constants.nix
index 8af80a02c..cccd1e279 100644
--- a/doc/manual/generate-builtin-constants.nix
+++ b/doc/manual/generate-builtin-constants.nix
@@ -1,6 +1,6 @@
 let
   inherit (builtins) concatStringsSep attrValues mapAttrs;
-  inherit (import ./utils.nix) optionalString squash;
+  inherit (import <nix/utils.nix>) optionalString squash;
 in
 
 builtinsInfo:
diff --git a/doc/manual/generate-builtins.nix b/doc/manual/generate-builtins.nix
index 813a287f5..05cae1c46 100644
--- a/doc/manual/generate-builtins.nix
+++ b/doc/manual/generate-builtins.nix
@@ -1,6 +1,6 @@
 let
   inherit (builtins) concatStringsSep attrValues mapAttrs;
-  inherit (import ./utils.nix) optionalString squash;
+  inherit (import <nix/utils.nix>) optionalString squash;
 in
 
 builtinsInfo:
diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix
index 14136016d..c4b9d1335 100644
--- a/doc/manual/generate-manpage.nix
+++ b/doc/manual/generate-manpage.nix
@@ -1,9 +1,29 @@
 let
   inherit (builtins)
-    attrNames attrValues fromJSON listToAttrs mapAttrs groupBy
-    concatStringsSep concatMap length lessThan replaceStrings sort;
-  inherit (import <nix/utils.nix>) attrsToList concatStrings optionalString filterAttrs trim squash unique;
-  showStoreDocs = import ./generate-store-info.nix;
+    attrNames
+    attrValues
+    concatMap
+    concatStringsSep
+    fromJSON
+    groupBy
+    length
+    lessThan
+    listToAttrs
+    mapAttrs
+    match
+    replaceStrings
+    sort
+    ;
+  inherit (import <nix/utils.nix>)
+    attrsToList
+    concatStrings
+    filterAttrs
+    optionalString
+    squash
+    trim
+    unique
+    ;
+  showStoreDocs = import <nix/generate-store-info.nix>;
 in
 
 inlineHTML: commandDump:
@@ -97,7 +117,7 @@ let
             ${optionalString (cat != "") "## ${cat}"}
 
             ${concatStringsSep "\n" (attrValues (mapAttrs showOption opts))}
-            '';
+          '';
           showOption = name: option:
             let
               result = trim ''
diff --git a/doc/manual/generate-settings.nix b/doc/manual/generate-settings.nix
index 8736bb793..3add10075 100644
--- a/doc/manual/generate-settings.nix
+++ b/doc/manual/generate-settings.nix
@@ -1,6 +1,6 @@
 let
   inherit (builtins) attrValues concatStringsSep isAttrs isBool mapAttrs;
-  inherit (import ./utils.nix) concatStrings indent optionalString squash;
+  inherit (import <nix/utils.nix>) concatStrings indent optionalString squash;
 in
 
 # `inlineHTML` is a hack to accommodate inconsistent output from `lowdown`
diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix
index 36215aadf..73defcb71 100644
--- a/doc/manual/generate-store-info.nix
+++ b/doc/manual/generate-store-info.nix
@@ -1,7 +1,7 @@
 let
   inherit (builtins) attrValues mapAttrs;
-  inherit (import ./utils.nix) concatStrings optionalString;
-  showSettings = import ./generate-settings.nix;
+  inherit (import <nix/utils.nix>) concatStrings optionalString;
+  showSettings = import <nix/generate-settings.nix>;
 in
 
 inlineHTML: storesInfo:
diff --git a/doc/manual/generate-xp-features-shortlist.nix b/doc/manual/generate-xp-features-shortlist.nix
index 30e211c96..ec09f4b75 100644
--- a/doc/manual/generate-xp-features-shortlist.nix
+++ b/doc/manual/generate-xp-features-shortlist.nix
@@ -1,5 +1,5 @@
 with builtins;
-with import ./utils.nix;
+with import <nix/utils.nix>;
 
 let
   showExperimentalFeature = name: doc:
diff --git a/doc/manual/generate-xp-features.nix b/doc/manual/generate-xp-features.nix
index adb94355c..fc7c7d4cf 100644
--- a/doc/manual/generate-xp-features.nix
+++ b/doc/manual/generate-xp-features.nix
@@ -1,5 +1,5 @@
 with builtins;
-with import ./utils.nix;
+with import <nix/utils.nix>;
 
 let
   showExperimentalFeature = name: doc:

From 8cafc754d845529a78595d1196769257ee23ca56 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 30 Nov 2023 21:54:53 +0100
Subject: [PATCH 230/294] Move access control from FSInputAccessor to
 FilteringInputAccessor

---
 src/libexpr/eval.cc                         | 23 +++---
 src/libexpr/eval.hh                         |  3 +-
 src/libfetchers/filtering-input-accessor.cc | 83 +++++++++++++++++++++
 src/libfetchers/filtering-input-accessor.hh | 73 ++++++++++++++++++
 src/libfetchers/fs-input-accessor.cc        | 77 +++----------------
 src/libfetchers/fs-input-accessor.hh        | 22 +-----
 src/libfetchers/git.cc                      |  6 +-
 7 files changed, 191 insertions(+), 96 deletions(-)
 create mode 100644 src/libfetchers/filtering-input-accessor.cc
 create mode 100644 src/libfetchers/filtering-input-accessor.hh

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 23ac349fe..841c223cd 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -14,6 +14,7 @@
 #include "profiles.hh"
 #include "print.hh"
 #include "fs-input-accessor.hh"
+#include "filtering-input-accessor.hh"
 #include "memory-input-accessor.hh"
 #include "signals.hh"
 #include "gc-small-vector.hh"
@@ -510,17 +511,15 @@ EvalState::EvalState(
     , repair(NoRepair)
     , emptyBindings(0)
     , rootFS(
-        makeFSInputAccessor(
-            CanonPath::root,
-            evalSettings.restrictEval || evalSettings.pureEval
-            ? std::optional<std::set<CanonPath>>(std::set<CanonPath>())
-            : std::nullopt,
+        evalSettings.restrictEval || evalSettings.pureEval
+        ? ref<InputAccessor>(AllowListInputAccessor::create(makeFSInputAccessor(CanonPath::root), {},
             [](const CanonPath & path) -> RestrictedPathError {
                 auto modeInformation = evalSettings.pureEval
                     ? "in pure evaluation mode (use '--impure' to override)"
                     : "in restricted mode";
                 throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation);
             }))
+        : makeFSInputAccessor(CanonPath::root))
     , corepkgsFS(makeMemoryInputAccessor())
     , internalFS(makeMemoryInputAccessor())
     , derivationInternal{corepkgsFS->addFile(
@@ -563,7 +562,7 @@ EvalState::EvalState(
     }
 
     /* Allow access to all paths in the search path. */
-    if (rootFS->hasAccessControl())
+    if (rootFS.dynamic_pointer_cast<AllowListInputAccessor>())
         for (auto & i : searchPath.elements)
             resolveSearchPathPath(i.path, true);
 
@@ -583,12 +582,14 @@ EvalState::~EvalState()
 
 void EvalState::allowPath(const Path & path)
 {
-    rootFS->allowPath(CanonPath(path));
+    if (auto rootFS2 = rootFS.dynamic_pointer_cast<AllowListInputAccessor>())
+        rootFS2->allowPath(CanonPath(path));
 }
 
 void EvalState::allowPath(const StorePath & storePath)
 {
-    rootFS->allowPath(CanonPath(store->toRealPath(storePath)));
+    if (auto rootFS2 = rootFS.dynamic_pointer_cast<AllowListInputAccessor>())
+        rootFS2->allowPath(CanonPath(store->toRealPath(storePath)));
 }
 
 void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & v)
@@ -617,12 +618,14 @@ void EvalState::checkURI(const std::string & uri)
     /* If the URI is a path, then check it against allowedPaths as
        well. */
     if (hasPrefix(uri, "/")) {
-        rootFS->checkAllowed(CanonPath(uri));
+        if (auto rootFS2 = rootFS.dynamic_pointer_cast<AllowListInputAccessor>())
+            rootFS2->checkAccess(CanonPath(uri));
         return;
     }
 
     if (hasPrefix(uri, "file://")) {
-        rootFS->checkAllowed(CanonPath(uri.substr(7)));
+        if (auto rootFS2 = rootFS.dynamic_pointer_cast<AllowListInputAccessor>())
+            rootFS2->checkAccess(CanonPath(uri.substr(7)));
         return;
     }
 
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index ee7bdda0d..f3f6d35b9 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -30,7 +30,6 @@ class EvalState;
 class StorePath;
 struct SingleDerivedPath;
 enum RepairFlag : bool;
-struct FSInputAccessor;
 struct MemoryInputAccessor;
 
 
@@ -222,7 +221,7 @@ public:
     /**
      * The accessor for the root filesystem.
      */
-    const ref<FSInputAccessor> rootFS;
+    const ref<InputAccessor> rootFS;
 
     /**
      * The in-memory filesystem for <nix/...> paths.
diff --git a/src/libfetchers/filtering-input-accessor.cc b/src/libfetchers/filtering-input-accessor.cc
new file mode 100644
index 000000000..5ae416fd3
--- /dev/null
+++ b/src/libfetchers/filtering-input-accessor.cc
@@ -0,0 +1,83 @@
+#include "filtering-input-accessor.hh"
+
+namespace nix {
+
+std::string FilteringInputAccessor::readFile(const CanonPath & path)
+{
+    checkAccess(path);
+    return next->readFile(prefix + path);
+}
+
+bool FilteringInputAccessor::pathExists(const CanonPath & path)
+{
+    return isAllowed(path) && next->pathExists(prefix + path);
+}
+
+std::optional<InputAccessor::Stat> FilteringInputAccessor::maybeLstat(const CanonPath & path)
+{
+    checkAccess(path);
+    return next->maybeLstat(prefix + path);
+}
+
+InputAccessor::DirEntries FilteringInputAccessor::readDirectory(const CanonPath & path)
+{
+    checkAccess(path);
+    DirEntries entries;
+    for (auto & entry : next->readDirectory(prefix + path)) {
+        if (isAllowed(path + entry.first))
+            entries.insert(std::move(entry));
+    }
+    return entries;
+}
+
+std::string FilteringInputAccessor::readLink(const CanonPath & path)
+{
+    checkAccess(path);
+    return next->readLink(prefix + path);
+}
+
+std::string FilteringInputAccessor::showPath(const CanonPath & path)
+{
+    return next->showPath(prefix + path);
+}
+
+void FilteringInputAccessor::checkAccess(const CanonPath & path)
+{
+    if (!isAllowed(path))
+        throw makeNotAllowedError
+            ? makeNotAllowedError(path)
+            : RestrictedPathError("access to path '%s' is forbidden", showPath(path));
+}
+
+struct AllowListInputAccessorImpl : AllowListInputAccessor
+{
+    std::set<CanonPath> allowedPaths;
+
+    AllowListInputAccessorImpl(
+        ref<InputAccessor> next,
+        std::set<CanonPath> && allowedPaths,
+        MakeNotAllowedError && makeNotAllowedError)
+        : AllowListInputAccessor(SourcePath(next), std::move(makeNotAllowedError))
+        , allowedPaths(std::move(allowedPaths))
+    { }
+
+    bool isAllowed(const CanonPath & path) override
+    {
+        return path.isAllowed(allowedPaths);
+    }
+
+    void allowPath(CanonPath path) override
+    {
+        allowedPaths.insert(std::move(path));
+    }
+};
+
+ref<AllowListInputAccessor> AllowListInputAccessor::create(
+    ref<InputAccessor> next,
+    std::set<CanonPath> && allowedPaths,
+    MakeNotAllowedError && makeNotAllowedError)
+{
+    return make_ref<AllowListInputAccessorImpl>(next, std::move(allowedPaths), std::move(makeNotAllowedError));
+}
+
+}
diff --git a/src/libfetchers/filtering-input-accessor.hh b/src/libfetchers/filtering-input-accessor.hh
new file mode 100644
index 000000000..209d26974
--- /dev/null
+++ b/src/libfetchers/filtering-input-accessor.hh
@@ -0,0 +1,73 @@
+#pragma once
+
+#include "input-accessor.hh"
+
+namespace nix {
+
+/**
+ * A function that should throw an exception of type
+ * `RestrictedPathError` explaining that access to `path` is
+ * forbidden.
+ */
+typedef std::function<RestrictedPathError(const CanonPath & path)> MakeNotAllowedError;
+
+/**
+ * An abstract wrapping `InputAccessor` that performs access
+ * control. Subclasses should override `checkAccess()` to implement an
+ * access control policy.
+ */
+struct FilteringInputAccessor : InputAccessor
+{
+    ref<InputAccessor> next;
+    CanonPath prefix;
+    MakeNotAllowedError makeNotAllowedError;
+
+    FilteringInputAccessor(const SourcePath & src, MakeNotAllowedError && makeNotAllowedError)
+        : next(src.accessor)
+        , prefix(src.path)
+        , makeNotAllowedError(std::move(makeNotAllowedError))
+    { }
+
+    std::string readFile(const CanonPath & path) override;
+
+    bool pathExists(const CanonPath & path) override;
+
+    std::optional<Stat> maybeLstat(const CanonPath & path) override;
+
+    DirEntries readDirectory(const CanonPath & path) override;
+
+    std::string readLink(const CanonPath & path) override;
+
+    std::string showPath(const CanonPath & path) override;
+
+    /**
+     * Call `makeNotAllowedError` to throw a `RestrictedPathError`
+     * exception if `isAllowed()` returns `false` for `path`.
+     */
+    void checkAccess(const CanonPath & path);
+
+    /**
+     * Return `true` iff access to path is allowed.
+     */
+    virtual bool isAllowed(const CanonPath & path) = 0;
+};
+
+/**
+ * A wrapping `InputAccessor` that checks paths against an allow-list.
+ */
+struct AllowListInputAccessor : public FilteringInputAccessor
+{
+    /**
+     * Grant access to the specified path.
+     */
+    virtual void allowPath(CanonPath path) = 0;
+
+    static ref<AllowListInputAccessor> create(
+        ref<InputAccessor> next,
+        std::set<CanonPath> && allowedPaths,
+        MakeNotAllowedError && makeNotAllowedError);
+
+    using FilteringInputAccessor::FilteringInputAccessor;
+};
+
+}
diff --git a/src/libfetchers/fs-input-accessor.cc b/src/libfetchers/fs-input-accessor.cc
index 2efee932d..c3d8d273c 100644
--- a/src/libfetchers/fs-input-accessor.cc
+++ b/src/libfetchers/fs-input-accessor.cc
@@ -4,19 +4,12 @@
 
 namespace nix {
 
-struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor
+struct FSInputAccessor : InputAccessor, PosixSourceAccessor
 {
     CanonPath root;
-    std::optional<std::set<CanonPath>> allowedPaths;
-    MakeNotAllowedError makeNotAllowedError;
 
-    FSInputAccessorImpl(
-        const CanonPath & root,
-        std::optional<std::set<CanonPath>> && allowedPaths,
-        MakeNotAllowedError && makeNotAllowedError)
+    FSInputAccessor(const CanonPath & root)
         : root(root)
-        , allowedPaths(std::move(allowedPaths))
-        , makeNotAllowedError(std::move(makeNotAllowedError))
     {
         displayPrefix = root.isRoot() ? "" : root.abs();
     }
@@ -27,39 +20,30 @@ struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor
         std::function<void(uint64_t)> sizeCallback) override
     {
         auto absPath = makeAbsPath(path);
-        checkAllowed(absPath);
         PosixSourceAccessor::readFile(absPath, sink, sizeCallback);
     }
 
     bool pathExists(const CanonPath & path) override
     {
-        auto absPath = makeAbsPath(path);
-        return isAllowed(absPath) && PosixSourceAccessor::pathExists(absPath);
+        return PosixSourceAccessor::pathExists(makeAbsPath(path));
     }
 
     std::optional<Stat> maybeLstat(const CanonPath & path) override
     {
-        auto absPath = makeAbsPath(path);
-        checkAllowed(absPath);
-        return PosixSourceAccessor::maybeLstat(absPath);
+        return PosixSourceAccessor::maybeLstat(makeAbsPath(path));
     }
 
     DirEntries readDirectory(const CanonPath & path) override
     {
-        auto absPath = makeAbsPath(path);
-        checkAllowed(absPath);
         DirEntries res;
-        for (auto & entry : PosixSourceAccessor::readDirectory(absPath))
-            if (isAllowed(absPath + entry.first))
-                res.emplace(entry);
+        for (auto & entry : PosixSourceAccessor::readDirectory(makeAbsPath(path)))
+            res.emplace(entry);
         return res;
     }
 
     std::string readLink(const CanonPath & path) override
     {
-        auto absPath = makeAbsPath(path);
-        checkAllowed(absPath);
-        return PosixSourceAccessor::readLink(absPath);
+        return PosixSourceAccessor::readLink(makeAbsPath(path));
     }
 
     CanonPath makeAbsPath(const CanonPath & path)
@@ -67,59 +51,22 @@ struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor
         return root + path;
     }
 
-    void checkAllowed(const CanonPath & absPath) override
-    {
-        if (!isAllowed(absPath))
-            throw makeNotAllowedError
-                ? makeNotAllowedError(absPath)
-                : RestrictedPathError("access to path '%s' is forbidden", absPath);
-    }
-
-    bool isAllowed(const CanonPath & absPath)
-    {
-        if (!absPath.isWithin(root))
-            return false;
-
-        if (allowedPaths) {
-            auto p = absPath.removePrefix(root);
-            if (!p.isAllowed(*allowedPaths))
-                return false;
-        }
-
-        return true;
-    }
-
-    void allowPath(CanonPath path) override
-    {
-        if (allowedPaths)
-            allowedPaths->insert(std::move(path));
-    }
-
-    bool hasAccessControl() override
-    {
-        return (bool) allowedPaths;
-    }
-
     std::optional<CanonPath> getPhysicalPath(const CanonPath & path) override
     {
         return makeAbsPath(path);
     }
 };
 
-ref<FSInputAccessor> makeFSInputAccessor(
-    const CanonPath & root,
-    std::optional<std::set<CanonPath>> && allowedPaths,
-    MakeNotAllowedError && makeNotAllowedError)
+ref<InputAccessor> makeFSInputAccessor(const CanonPath & root)
 {
-    return make_ref<FSInputAccessorImpl>(root, std::move(allowedPaths), std::move(makeNotAllowedError));
+    return make_ref<FSInputAccessor>(root);
 }
 
-ref<FSInputAccessor> makeStorePathAccessor(
+ref<InputAccessor> makeStorePathAccessor(
     ref<Store> store,
-    const StorePath & storePath,
-    MakeNotAllowedError && makeNotAllowedError)
+    const StorePath & storePath)
 {
-    return makeFSInputAccessor(CanonPath(store->toRealPath(storePath)), {}, std::move(makeNotAllowedError));
+    return makeFSInputAccessor(CanonPath(store->toRealPath(storePath)));
 }
 
 SourcePath getUnfilteredRootPath(CanonPath path)
diff --git a/src/libfetchers/fs-input-accessor.hh b/src/libfetchers/fs-input-accessor.hh
index 19a5211c8..ba5af5887 100644
--- a/src/libfetchers/fs-input-accessor.hh
+++ b/src/libfetchers/fs-input-accessor.hh
@@ -7,26 +7,12 @@ namespace nix {
 class StorePath;
 class Store;
 
-struct FSInputAccessor : InputAccessor
-{
-    virtual void checkAllowed(const CanonPath & absPath) = 0;
+ref<InputAccessor> makeFSInputAccessor(
+    const CanonPath & root);
 
-    virtual void allowPath(CanonPath path) = 0;
-
-    virtual bool hasAccessControl() = 0;
-};
-
-typedef std::function<RestrictedPathError(const CanonPath & path)> MakeNotAllowedError;
-
-ref<FSInputAccessor> makeFSInputAccessor(
-    const CanonPath & root,
-    std::optional<std::set<CanonPath>> && allowedPaths = {},
-    MakeNotAllowedError && makeNotAllowedError = {});
-
-ref<FSInputAccessor> makeStorePathAccessor(
+ref<InputAccessor> makeStorePathAccessor(
     ref<Store> store,
-    const StorePath & storePath,
-    MakeNotAllowedError && makeNotAllowedError = {});
+    const StorePath & storePath);
 
 SourcePath getUnfilteredRootPath(CanonPath path);
 
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 8cd74057c..ff4b1e823 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -9,6 +9,7 @@
 #include "processes.hh"
 #include "git.hh"
 #include "fs-input-accessor.hh"
+#include "filtering-input-accessor.hh"
 #include "mounted-input-accessor.hh"
 #include "git-utils.hh"
 #include "logging.hh"
@@ -639,7 +640,10 @@ struct GitInputScheme : InputScheme
                 repoInfo.workdirInfo.files.insert(submodule.path);
 
         ref<InputAccessor> accessor =
-            makeFSInputAccessor(CanonPath(repoInfo.url), repoInfo.workdirInfo.files, makeNotAllowedError(repoInfo.url));
+            AllowListInputAccessor::create(
+                makeFSInputAccessor(CanonPath(repoInfo.url)),
+                std::move(repoInfo.workdirInfo.files),
+                makeNotAllowedError(repoInfo.url));
 
         /* If the repo has submodules, return a mounted input accessor
            consisting of the accessor for the top-level repo and the

From cab41025d85d3b02f5175cf7ca2611c7a44c2cdd Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Thu, 30 Nov 2023 23:04:05 +0100
Subject: [PATCH 231/294] mention renaming of `nix doctor`

---
 doc/manual/rl-next/nix-config-show.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/rl-next/nix-config-show.md b/doc/manual/rl-next/nix-config-show.md
index 08ad207cb..b2ad3c666 100644
--- a/doc/manual/rl-next/nix-config-show.md
+++ b/doc/manual/rl-next/nix-config-show.md
@@ -3,6 +3,6 @@ issues: #7672
 prs: #9477
 description: {
 
-`nix show-config` was renamed to `nix config show` to be more consistent with the rest of the command-line interface.
+`nix show-config` was renamed to `nix config show`, and `nix doctor` was renamed to `nix config check`, to be more consistent with the rest of the command-line interface.
 
 }

From 39de819edaef2dc3e308a490bbb2b1622a932771 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Mon, 9 Oct 2023 10:08:22 +0200
Subject: [PATCH 232/294] rename debugging helper environment variable

---
 src/libutil/error.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 8488e7e21..72c346cb5 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -159,11 +159,11 @@ static std::string indent(std::string_view indentFirst, std::string_view indentR
 /**
  * A development aid for finding missing positions, to improve error messages. Example use:
  *
- *     NIX_DEVELOPER_SHOW_UNKNOWN_LOCATIONS=1 _NIX_TEST_ACCEPT=1 make tests/lang.sh.test
+ *     _NIX_EVAL_SHOW_UNKNOWN_LOCATIONS=1 _NIX_TEST_ACCEPT=1 make tests/lang.sh.test
  *     git diff -U20 tests
  *
  */
-static bool printUnknownLocations = getEnv("_NIX_DEVELOPER_SHOW_UNKNOWN_LOCATIONS").has_value();
+static bool printUnknownLocations = getEnv("_NIX_EVAL_SHOW_UNKNOWN_LOCATIONS").has_value();
 
 /**
  * Print a position, if it is known.

From 0301b8fc7354a94dc03b57f796bfa6e853758af8 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Thu, 30 Nov 2023 21:59:08 +0100
Subject: [PATCH 233/294] reword the experimental feature notice

- put the highlight box around all the relevant instructions
- simplify the wording
- make the link more prominent by using the whole phrase for the link text
---
 doc/manual/generate-settings.nix   | 21 ++++++-------
 doc/manual/generate-store-info.nix | 48 +++++++++++++++---------------
 2 files changed, 35 insertions(+), 34 deletions(-)

diff --git a/doc/manual/generate-settings.nix b/doc/manual/generate-settings.nix
index 3add10075..74446b70b 100644
--- a/doc/manual/generate-settings.nix
+++ b/doc/manual/generate-settings.nix
@@ -31,18 +31,19 @@ let
 
       experimentalFeatureNote = optionalString (experimentalFeature != null) ''
           > **Warning**
+          >
           > This setting is part of an
           > [experimental feature](@docroot@/contributing/experimental-features.md).
-
-          To change this setting, you need to make sure the corresponding experimental feature,
-          [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}),
-          is enabled.
-          For example, include the following in [`nix.conf`](#):
-
-          ```
-          extra-experimental-features = ${experimentalFeature}
-          ${setting} = ...
-          ```
+          >
+          > To change this setting, make sure the
+          > [`${experimentalFeature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature})
+          > is enabled.
+          > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md):
+          >
+          > ```
+          > extra-experimental-features = ${experimentalFeature}
+          > ${setting} = ...
+          > ```
         '';
 
       showDefault = documentDefault: defaultValue:
diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix
index 73defcb71..8e26edb65 100644
--- a/doc/manual/generate-store-info.nix
+++ b/doc/manual/generate-store-info.nix
@@ -1,6 +1,6 @@
 let
   inherit (builtins) attrValues mapAttrs;
-  inherit (import <nix/utils.nix>) concatStrings optionalString;
+  inherit (import <nix/utils.nix>) concatStrings optionalString squash;
   showSettings = import <nix/generate-settings.nix>;
 in
 
@@ -10,36 +10,36 @@ let
 
   showStore = name: { settings, doc, experimentalFeature }:
     let
+      result = squash ''
+        # ${name}
 
-    result = ''
-      ## ${name}
+        ${doc}
 
-      ${doc}
+        ${experimentalFeatureNote}
 
-      ${experimentalFeatureNote}
+        ## Settings
 
-      ### Settings
-
-      ${showSettings { prefix = "store-${slug}"; inherit inlineHTML; } settings}
-    '';
+        ${showSettings { prefix = "store-${slug}"; inherit inlineHTML; } settings}
+      '';
 
       # markdown doesn't like spaces in URLs
       slug = builtins.replaceStrings [ " " ] [ "-" ] name;
 
-    experimentalFeatureNote = optionalString (experimentalFeature != null) ''
-      > **Warning**
-      > This store is part of an
-      > [experimental feature](@docroot@/contributing/experimental-features.md).
-
-      To use this store, you need to make sure the corresponding experimental feature,
-      [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}),
-      is enabled.
-      For example, include the following in [`nix.conf`](#):
-
-      ```
-      extra-experimental-features = ${experimentalFeature}
-      ```
-    '';
-  in result;
+      experimentalFeatureNote = optionalString (experimentalFeature != null) ''
+        > **Warning**
+        >
+        > This store is part of an
+        > [experimental feature](@docroot@/contributing/experimental-features.md).
+        >
+        > To use this store, make sure the
+        > [`${experimentalFeature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature})
+        > is enabled.
+        > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md):
+        >
+        > ```
+        > extra-experimental-features = ${experimentalFeature}
+        > ```
+      '';
+    in result;
 
 in concatStrings (attrValues (mapAttrs showStore storesInfo))

From 02d9cf2d303e4e7e283dba2f3181f3e40843c354 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Fri, 1 Dec 2023 00:41:19 +0100
Subject: [PATCH 234/294] shorten the quick start chapter

this focuses on `nix-shell -p` and refers to search.nixos.org for
package search, which is currently the easiest and most effective way to
find program names.
---
 doc/manual/src/quick-start.md | 87 +++++++----------------------------
 1 file changed, 16 insertions(+), 71 deletions(-)

diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md
index 5f54abbde..04a0b7c96 100644
--- a/doc/manual/src/quick-start.md
+++ b/doc/manual/src/quick-start.md
@@ -1,10 +1,9 @@
 # Quick Start
 
-This chapter is for impatient people who don't like reading
-documentation.  For more in-depth information you are kindly referred
-to subsequent chapters.
+This chapter is for impatient people who don't like reading documentation.
+For more in-depth information you are kindly referred to subsequent chapters.
 
-1. Install Nix by running the following:
+1. Install Nix:
 
    ```console
    $ curl -L https://nixos.org/nix/install | sh
@@ -13,87 +12,33 @@ to subsequent chapters.
    The install script will use `sudo`, so make sure you have sufficient rights.
    On Linux, `--daemon` can be omitted for a single-user install.
 
-   For other installation methods, see [here](installation/index.md).
+   For other installation methods, see the detailed [installation instructions](installation/index.md).
 
-1. See what installable packages are currently available in the
-   channel:
+1. Run software without installing it permanently:
 
    ```console
-   $ nix-env --query --available --attr-path
-   nixpkgs.docbook_xml_dtd_43                    docbook-xml-4.3
-   nixpkgs.docbook_xml_dtd_45                    docbook-xml-4.5
-   nixpkgs.firefox                               firefox-33.0.2
-   nixpkgs.hello                                 hello-2.9
-   nixpkgs.libxslt                               libxslt-1.1.28
-   …
+   $ nix-shell --packages cowsay lolcat
    ```
 
-1. Install some packages from the channel:
+   This downloads the specified packages with all their dependencies, and drops you into a Bash shell where the commands provided by those packages are present.
+   This will not affect your normal environment:
 
    ```console
-   $ nix-env --install --attr nixpkgs.hello
+   [nix-shell:~]$ cowsay Hello, Nix! | lolcat
    ```
 
-   This should download pre-built packages; it should not build them
-   locally (if it does, something went wrong).
-
-1. Test that they work:
+   Exiting the shell will make the programs disappear again:
 
    ```console
-   $ which hello
-   /home/eelco/.nix-profile/bin/hello
-   $ hello
-   Hello, world!
-   ```
-
-1. Uninstall a package:
-
-   ```console
-   $ nix-env --uninstall hello
-   ```
-
-1. You can also test a package without installing it:
-
-   ```console
-   $ nix-shell --packages hello
-   ```
-
-   This builds or downloads GNU Hello and its dependencies, then drops
-   you into a Bash shell where the `hello` command is present, all
-   without affecting your normal environment:
-
-   ```console
-   [nix-shell:~]$ hello
-   Hello, world!
-
    [nix-shell:~]$ exit
-
-   $ hello
-   hello: command not found
+   $ lolcat
+   lolcat: command not found
    ```
 
-1. To keep up-to-date with the channel, do:
+1. Search for more packages on <search.nixos.org> to try them out.
+
+1. Free up storage space:
 
    ```console
-   $ nix-channel --update nixpkgs
-   $ nix-env --upgrade '*'
-   ```
-
-   The latter command will upgrade each installed package for which
-   there is a “newer” version (as determined by comparing the version
-   numbers).
-
-1. If you're unhappy with the result of a `nix-env` action (e.g., an
-   upgraded package turned out not to work properly), you can go back:
-
-   ```console
-   $ nix-env --rollback
-   ```
-
-1. You should periodically run the Nix garbage collector to get rid of
-   unused packages, since uninstalls or upgrades don't actually delete
-   them:
-
-   ```console
-   $ nix-collect-garbage --delete-old
+   $ nix-collect-garbage
    ```

From 908a011a4a2fe4e494e5b6e4c94f013f159f3616 Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <github@infinisil.com>
Date: Fri, 1 Dec 2023 00:50:20 +0100
Subject: [PATCH 235/294] Revert "Switch from std::regex to boost::regex"

---
 flake.nix              | 22 +++++++---------------
 src/libexpr/local.mk   |  2 +-
 src/libexpr/primops.cc | 35 ++++++++++++-----------------------
 3 files changed, 20 insertions(+), 39 deletions(-)

diff --git a/flake.nix b/flake.nix
index 33673575b..822b3d31e 100644
--- a/flake.nix
+++ b/flake.nix
@@ -157,7 +157,7 @@
 
         configureFlags =
           lib.optionals stdenv.isLinux [
-            "--with-boost=${boost-nix}/lib"
+            "--with-boost=${boost}/lib"
             "--with-sandbox-shell=${sh}/bin/busybox"
           ]
           ++ lib.optionals (stdenv.isLinux && !(isStatic && stdenv.system == "aarch64-linux")) [
@@ -210,7 +210,7 @@
               version = libgit2.lastModifiedDate;
               cmakeFlags = (attrs.cmakeFlags or []) ++ ["-DUSE_SSH=exec"];
             }))
-            boost-nix
+            boost
             lowdown-nix
             libsodium
           ]
@@ -434,14 +434,14 @@
 
             propagatedBuildInputs = propagatedDeps;
 
-            disallowedReferences = [ boost-nix ];
+            disallowedReferences = [ boost ];
 
             preConfigure = lib.optionalString (! currentStdenv.hostPlatform.isStatic)
               ''
                 # Copy libboost_context so we don't get all of Boost in our closure.
                 # https://github.com/NixOS/nixpkgs/issues/45462
                 mkdir -p $out/lib
-                cp -pd ${boost-nix}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib
+                cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
                 rm -f $out/lib/*.a
                 ${lib.optionalString currentStdenv.hostPlatform.isLinux ''
                   chmod u+w $out/lib/*.so.*
@@ -451,9 +451,9 @@
                   for LIB in $out/lib/*.dylib; do
                     chmod u+w $LIB
                     install_name_tool -id $LIB $LIB
-                    install_name_tool -delete_rpath ${boost-nix}/lib/ $LIB || true
+                    install_name_tool -delete_rpath ${boost}/lib/ $LIB || true
                   done
-                  install_name_tool -change ${boost-nix}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib
+                  install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib
                 ''}
               '';
 
@@ -481,13 +481,9 @@
               ''}
               ${lib.optionalString currentStdenv.isDarwin ''
               install_name_tool \
-                -change ${boost-nix}/lib/libboost_context.dylib \
+                -change ${boost}/lib/libboost_context.dylib \
                 $out/lib/libboost_context.dylib \
                 $out/lib/libnixutil.dylib
-              install_name_tool \
-                -change ${boost-nix}/lib/libboost_regex.dylib \
-                $out/lib/libboost_regex.dylib \
-                $out/lib/libnixexpr.dylib
               ''}
             '';
 
@@ -510,10 +506,6 @@
             meta.mainProgram = "nix";
           });
 
-          boost-nix = final.boost.override {
-            enableIcu = false;
-          };
-
           lowdown-nix = with final; currentStdenv.mkDerivation rec {
             name = "lowdown-0.9.0";
 
diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk
index c07a18bb5..ed6bc761a 100644
--- a/src/libexpr/local.mk
+++ b/src/libexpr/local.mk
@@ -16,7 +16,7 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib
 
 libexpr_LIBS = libutil libstore libfetchers
 
-libexpr_LDFLAGS += -lboost_context -lboost_regex -pthread
+libexpr_LDFLAGS += -lboost_context -pthread
 ifdef HOST_LINUX
  libexpr_LDFLAGS += -ldl
 endif
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index ebf2549e4..146a7603c 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -17,7 +17,6 @@
 #include "primops.hh"
 
 #include <boost/container/small_vector.hpp>
-#include <boost/regex.hpp>
 #include <nlohmann/json.hpp>
 
 #include <sys/types.h>
@@ -26,6 +25,7 @@
 
 #include <algorithm>
 #include <cstring>
+#include <regex>
 #include <dlfcn.h>
 
 #include <cmath>
@@ -3886,30 +3886,19 @@ static RegisterPrimOp primop_convertHash({
     .fun = prim_convertHash,
 });
 
-// regex aliases, switch between boost and std
-using regex = boost::regex;
-using regex_error = boost::regex_error;
-using cmatch = boost::cmatch;
-using cregex_iterator = boost::cregex_iterator;
-namespace regex_constants = boost::regex_constants;
-// overloaded function alias
-constexpr auto regex_match = [] (auto &&...args) {
-    return boost::regex_match(std::forward<decltype(args)>(args)...);
- };
-
 struct RegexCache
 {
     // TODO use C++20 transparent comparison when available
-    std::unordered_map<std::string_view, regex> cache;
+    std::unordered_map<std::string_view, std::regex> cache;
     std::list<std::string> keys;
 
-    regex get(std::string_view re)
+    std::regex get(std::string_view re)
     {
         auto it = cache.find(re);
         if (it != cache.end())
             return it->second;
         keys.emplace_back(re);
-        return cache.emplace(keys.back(), regex(keys.back(), regex::extended)).first->second;
+        return cache.emplace(keys.back(), std::regex(keys.back(), std::regex::extended)).first->second;
     }
 };
 
@@ -3929,8 +3918,8 @@ void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v)
         NixStringContext context;
         const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.match");
 
-        cmatch match;
-        if (!regex_match(str.begin(), str.end(), match, regex)) {
+        std::cmatch match;
+        if (!std::regex_match(str.begin(), str.end(), match, regex)) {
             v.mkNull();
             return;
         }
@@ -3945,8 +3934,8 @@ void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v)
                 (v.listElems()[i] = state.allocValue())->mkString(match[i + 1].str());
         }
 
-    } catch (regex_error & e) {
-        if (e.code() == regex_constants::error_space) {
+    } catch (std::regex_error & e) {
+        if (e.code() == std::regex_constants::error_space) {
             // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
             state.debugThrowLastTrace(EvalError({
                 .msg = hintfmt("memory limit exceeded by regular expression '%s'", re),
@@ -4009,8 +3998,8 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v)
         NixStringContext context;
         const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.split");
 
-        auto begin = cregex_iterator(str.begin(), str.end(), regex);
-        auto end = cregex_iterator();
+        auto begin = std::cregex_iterator(str.begin(), str.end(), regex);
+        auto end = std::cregex_iterator();
 
         // Any matches results are surrounded by non-matching results.
         const size_t len = std::distance(begin, end);
@@ -4049,8 +4038,8 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 
         assert(idx == 2 * len + 1);
 
-    } catch (regex_error & e) {
-        if (e.code() == regex_constants::error_space) {
+    } catch (std::regex_error & e) {
+        if (e.code() == std::regex_constants::error_space) {
             // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
             state.debugThrowLastTrace(EvalError({
                 .msg = hintfmt("memory limit exceeded by regular expression '%s'", re),

From 4781e7fa7048d2861172baaaa04a7be4b8a2b631 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Thu, 30 Nov 2023 23:07:09 +0100
Subject: [PATCH 236/294] Document each store type on its own page

This makes for more useful manual table of contents, that displays the
information at a glance.

The `nix help-stores` command is kept as-is, even though it will show up
in the manual with the same information as these pages due to the way it
is written as a "`--help`-style" command. Deciding what to do with that
command is left for a later PR.

This change also lists all store types at the top of the respective overview page.

Co-authored-by: John Ericson <John.Ericson@Obsidian.Systems
---
 .gitignore                                   |  2 +
 doc/manual/generate-manpage.nix              | 64 ++++++++++++++------
 doc/manual/generate-store-info.nix           | 28 ++++++---
 doc/manual/generate-store-types.nix          | 39 ++++++++++++
 doc/manual/generate-xp-features.nix          |  4 +-
 doc/manual/local.mk                          | 11 +++-
 doc/manual/src/SUMMARY.md.in                 |  2 +
 doc/manual/src/contributing/cli-guideline.md |  2 +-
 doc/manual/src/store/index.md                |  2 +-
 doc/manual/src/store/types/index.md.in       |  5 +-
 doc/manual/utils.nix                         |  8 +++
 src/libcmd/common-eval-args.cc               |  2 +-
 src/libexpr/primops.cc                       |  2 +-
 src/libstore/globals.hh                      | 11 ++--
 src/libstore/remote-store.cc                 |  2 +-
 src/libstore/ssh-store-config.hh             |  2 +-
 src/libstore/store-api.hh                    |  4 +-
 src/libutil/experimental-features.cc         |  2 +-
 src/nix/nix.md                               | 10 ++-
 19 files changed, 155 insertions(+), 47 deletions(-)
 create mode 100644 doc/manual/generate-store-types.nix

diff --git a/.gitignore b/.gitignore
index bcf9c4a01..767a5d6ed 100644
--- a/.gitignore
+++ b/.gitignore
@@ -22,6 +22,8 @@ perl/Makefile.config
 /doc/manual/xp-features.json
 /doc/manual/src/SUMMARY.md
 /doc/manual/src/SUMMARY-rl-next.md
+/doc/manual/src/store/types/*
+!/doc/manual/src/store/types/index.md.in
 /doc/manual/src/command-ref/new-cli
 /doc/manual/src/command-ref/conf-file.md
 /doc/manual/src/command-ref/experimental-features-shortlist.md
diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix
index c4b9d1335..365422af7 100644
--- a/doc/manual/generate-manpage.nix
+++ b/doc/manual/generate-manpage.nix
@@ -51,7 +51,7 @@ let
 
         ${maybeSubcommands}
 
-        ${maybeStoreDocs}
+        ${maybeProse}
 
         ${maybeOptions}
       '';
@@ -91,25 +91,55 @@ let
         * [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description}
       '';
 
-      # FIXME: this is a hack.
-      # store parameters should not be part of command documentation to begin
-      # with, but instead be rendered on separate pages.
-      maybeStoreDocs = optionalString (details ? doc)
-        (replaceStrings [ "@stores@" ] [ (showStoreDocs inlineHTML commandInfo.stores) ] details.doc);
+      maybeProse =
+        # FIXME: this is a horrible hack to keep `nix help-stores` working.
+        # the correct answer to this is to remove that command and replace it
+        # by statically generated manpages or the output of something like `nix
+        # store info <store type>`.
+        let
+          help-stores = ''
+            ${index}
 
-      maybeOptions = let
-        allVisibleOptions = filterAttrs
-          (_: o: ! o.hiddenCategory)
-          (details.flags // toplevel.flags);
-      in optionalString (allVisibleOptions != {}) ''
-        # Options
+            ${allStores}
+          '';
+          index = replaceStrings
+            [ "@store-types@" ] [ storesOverview ]
+            details.doc;
+          storesOverview =
+            let
+              showEntry = store:
+                "- [${store.name}](#${store.slug})";
+            in
+            concatStringsSep "\n" (map showEntry storesList) + "\n";
+          allStores = concatStringsSep "\n" (attrValues storePages);
+          storePages = listToAttrs
+            (map (s: { name = s.filename; value = s.page; }) storesList);
+          storesList = showStoreDocs {
+            storeInfo = commandInfo.stores;
+            inherit inlineHTML;
+          };
+        in
+        optionalString (details ? doc) (
+          if match "@store-types@" details.doc != [ ]
+          then help-stores
+          else details.doc
+        );
 
-        ${showOptions inlineHTML allVisibleOptions}
+      maybeOptions =
+        let
+          allVisibleOptions = filterAttrs
+            (_: o: ! o.hiddenCategory)
+            (details.flags // toplevel.flags);
+        in
+        optionalString (allVisibleOptions != { }) ''
+          # Options
 
-        > **Note**
-        >
-        > See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags.
-      '';
+          ${showOptions inlineHTML allVisibleOptions}
+
+          > **Note**
+          >
+          > See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags.
+        '';
 
       showOptions = inlineHTML: allOptions:
         let
diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix
index 8e26edb65..57247a181 100644
--- a/doc/manual/generate-store-info.nix
+++ b/doc/manual/generate-store-info.nix
@@ -1,14 +1,20 @@
 let
-  inherit (builtins) attrValues mapAttrs;
-  inherit (import <nix/utils.nix>) concatStrings optionalString squash;
+  inherit (builtins) attrNames listToAttrs concatStringsSep readFile replaceStrings;
+  inherit (import <nix/utils.nix>) optionalString filterAttrs trim squash toLower unique indent;
   showSettings = import <nix/generate-settings.nix>;
 in
 
-inlineHTML: storesInfo:
+{
+  # data structure describing all stores and their parameters
+  storeInfo,
+  # whether to add inline HTML tags
+  # `lowdown` does not eat those for one of the output modes
+  inlineHTML,
+}:
 
 let
 
-  showStore = name: { settings, doc, experimentalFeature }:
+  showStore = { name, slug }: { settings, doc, experimentalFeature }:
     let
       result = squash ''
         # ${name}
@@ -22,9 +28,6 @@ let
         ${showSettings { prefix = "store-${slug}"; inherit inlineHTML; } settings}
       '';
 
-      # markdown doesn't like spaces in URLs
-      slug = builtins.replaceStrings [ " " ] [ "-" ] name;
-
       experimentalFeatureNote = optionalString (experimentalFeature != null) ''
         > **Warning**
         >
@@ -42,4 +45,13 @@ let
       '';
     in result;
 
-in concatStrings (attrValues (mapAttrs showStore storesInfo))
+  storesList = map
+    (name: rec {
+      inherit name;
+      slug = replaceStrings [ " " ] [ "-" ] (toLower name);
+      filename = "${slug}.md";
+      page = showStore { inherit name slug; } storeInfo.${name};
+    })
+    (attrNames storeInfo);
+
+in storesList
diff --git a/doc/manual/generate-store-types.nix b/doc/manual/generate-store-types.nix
new file mode 100644
index 000000000..3b78a0e1b
--- /dev/null
+++ b/doc/manual/generate-store-types.nix
@@ -0,0 +1,39 @@
+let
+  inherit (builtins) attrNames listToAttrs concatStringsSep readFile replaceStrings;
+  showSettings = import <nix/generate-settings.nix>;
+  showStoreDocs = import <nix/generate-store-info.nix>;
+in
+
+storeInfo:
+
+let
+  storesList = showStoreDocs {
+    inherit storeInfo;
+    inlineHTML = true;
+  };
+
+  index =
+    let
+      showEntry = store:
+        "- [${store.name}](./${store.filename})";
+    in
+    concatStringsSep "\n" (map showEntry storesList);
+
+  "index.md" = replaceStrings
+    [ "@store-types@" ] [ index ]
+    (readFile ./src/store/types/index.md.in);
+
+  tableOfContents =
+    let
+      showEntry = store:
+        "    - [${store.name}](store/types/${store.filename})";
+    in
+    concatStringsSep "\n" (map showEntry storesList) + "\n";
+
+  "SUMMARY.md" = tableOfContents;
+
+  storePages = listToAttrs
+    (map (s: { name = s.filename; value = s.page; }) storesList);
+
+in
+storePages // { inherit "index.md" "SUMMARY.md"; }
diff --git a/doc/manual/generate-xp-features.nix b/doc/manual/generate-xp-features.nix
index fc7c7d4cf..0eec0e1da 100644
--- a/doc/manual/generate-xp-features.nix
+++ b/doc/manual/generate-xp-features.nix
@@ -8,4 +8,6 @@ let
 
       ${doc}
     '';
-in xps: (concatStringsSep "\n" (attrValues (mapAttrs showExperimentalFeature xps)))
+in
+
+xps: (concatStringsSep "\n" (attrValues (mapAttrs showExperimentalFeature xps)))
diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index fa9db9f02..456000d3d 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -97,10 +97,17 @@ $(d)/nix-profiles.5: $(d)/src/command-ref/files/profiles.md
 	$(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@
 	@rm $^.tmp
 
-$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md
+$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md
 	@cp $< $@
 	@$(call process-includes,$@,$@)
 
+$(d)/src/store/types: $(d)/nix.json $(d)/utils.nix $(d)/generate-store-info.nix  $(d)/generate-store-types.nix $(d)/src/store/types/index.md.in $(doc_nix)
+	@# FIXME: build out of tree!
+	@rm -rf $@.tmp
+	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-store-types.nix (builtins.fromJSON (builtins.readFile $<)).stores'
+	@# do not destroy existing contents
+	@mv $@.tmp/* $@/
+
 $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(d)/generate-settings.nix $(d)/generate-store-info.nix $(doc_nix)
 	@rm -rf $@ $@.tmp
 	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix true (builtins.readFile $<)'
@@ -200,7 +207,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli
 # `@docroot@` is to be preserved for documenting the mechanism
 # FIXME: maybe contributing guides should live right next to the code
 # instead of in the manual
-$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md
+$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md
 	$(trace-gen) \
 		tmp="$$(mktemp -d)"; \
 		cp -r doc/manual "$$tmp"; \
diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in
index 686d3e8d7..c67ddc6cb 100644
--- a/doc/manual/src/SUMMARY.md.in
+++ b/doc/manual/src/SUMMARY.md.in
@@ -20,6 +20,8 @@
   - [File System Object](store/file-system-object.md)
   - [Store Object](store/store-object.md)
   - [Store Path](store/store-path.md)
+  - [Store Types](store/types/index.md)
+{{#include ./store/types/SUMMARY.md}}
 - [Nix Language](language/index.md)
   - [Data Types](language/values.md)
   - [Language Constructs](language/constructs.md)
diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/contributing/cli-guideline.md
index e53d2d178..f7e24d96b 100644
--- a/doc/manual/src/contributing/cli-guideline.md
+++ b/doc/manual/src/contributing/cli-guideline.md
@@ -426,7 +426,7 @@ This leads to the following guidelines:
 ### Examples
 
 
-This is bad, because all keys must be assumed to be store implementations:
+This is bad, because all keys must be assumed to be store types:
 
 ```json
 {
diff --git a/doc/manual/src/store/index.md b/doc/manual/src/store/index.md
index 8a5305062..f1e8f1402 100644
--- a/doc/manual/src/store/index.md
+++ b/doc/manual/src/store/index.md
@@ -2,4 +2,4 @@
 
 The *Nix store* is an abstraction to store immutable file system data (such as software packages) that can have dependencies on other such data.
 
-There are multiple implementations of Nix stores with different capabilities, such as the actual filesystem (`/nix/store`) or binary caches.
+There are [multiple types of Nix stores](./types/index.md) with different capabilities, such as the default one on the [local filesystem](./types/local-store.md) (`/nix/store`) or [binary caches](./types/http-binary-cache-store.md).
diff --git a/doc/manual/src/store/types/index.md.in b/doc/manual/src/store/types/index.md.in
index bb166a1fc..b4db553a2 100644
--- a/doc/manual/src/store/types/index.md.in
+++ b/doc/manual/src/store/types/index.md.in
@@ -1,4 +1,6 @@
-Nix supports different types of stores. These are described below.
+Nix supports different types of stores:
+
+@store-types@
 
 ## Store URL format
 
@@ -39,4 +41,3 @@ store as follows:
 
 * Otherwise, use the [local store](#local-store) `/nix/store`.
 
-@stores@
diff --git a/doc/manual/utils.nix b/doc/manual/utils.nix
index 849832b2c..19ff49b64 100644
--- a/doc/manual/utils.nix
+++ b/doc/manual/utils.nix
@@ -1,5 +1,11 @@
 with builtins;
 
+let
+  lowerChars = stringToCharacters "abcdefghijklmnopqrstuvwxyz";
+  upperChars = stringToCharacters "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+  stringToCharacters = s: genList (p: substring p 1 s) (stringLength s);
+in
+
 rec {
   splitLines = s: filter (x: !isList x) (split "\n" s);
 
@@ -18,6 +24,8 @@ rec {
     in
       if replaced == string then string else replaceStringsRec from to replaced;
 
+  toLower = replaceStrings upperChars lowerChars;
+
   squash = replaceStringsRec "\n\n\n" "\n\n";
 
   trim = string:
diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc
index 401acc38e..193972272 100644
--- a/src/libcmd/common-eval-args.cc
+++ b/src/libcmd/common-eval-args.cc
@@ -141,7 +141,7 @@ MixEvalArgs::MixEvalArgs()
         .longName = "eval-store",
         .description =
           R"(
-            The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
+            The [URL of the Nix store](@docroot@/store/types/index.md#store-url-format)
             to use for evaluation, i.e. to store derivations (`.drv` files) and inputs referenced by them.
           )",
         .category = category,
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index ebf2549e4..4ecdda55c 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -4460,7 +4460,7 @@ void EvalState::createBaseEnv()
         .doc = R"(
           Logical file system location of the [Nix store](@docroot@/glossary.md#gloss-store) currently in use.
 
-          This value is determined by the `store` parameter in [Store URLs](@docroot@/command-ref/new-cli/nix3-help-stores.md):
+          This value is determined by the `store` parameter in [Store URLs](@docroot@/store/types/index.md#store-url-format):
 
           ```shell-session
           $ nix-instantiate --store 'dummy://?store=/blah' --eval --expr builtins.storeDir
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 838d2aba2..38b0d516c 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -117,10 +117,11 @@ public:
 
     Setting<std::string> storeUri{this, getEnv("NIX_REMOTE").value_or("auto"), "store",
         R"(
-          The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
+          The [URL of the Nix store](@docroot@/store/types/index.md#store-url-format)
           to use for most operations.
-          See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md)
-          for supported store types and settings.
+          See the
+          [Store Types](@docroot@/store/types/index.md)
+          section of the manual for supported store types and settings.
         )"};
 
     Setting<bool> keepFailed{this, false, "keep-failed",
@@ -759,7 +760,7 @@ public:
         Strings{"https://cache.nixos.org/"},
         "substituters",
         R"(
-          A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) to be used as substituters, separated by whitespace.
+          A list of [URLs of Nix stores](@docroot@/store/types/index.md#store-url-format) to be used as substituters, separated by whitespace.
           A substituter is an additional [store]{@docroot@/glossary.md##gloss-store} from which Nix can obtain [store objects](@docroot@/glossary.md#gloss-store-object) instead of building them.
 
           Substituters are tried based on their priority value, which each substituter can set independently.
@@ -778,7 +779,7 @@ public:
     Setting<StringSet> trustedSubstituters{
         this, {}, "trusted-substituters",
         R"(
-          A list of [Nix store URLs](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), separated by whitespace.
+          A list of [Nix store URLs](@docroot@/store/types/index.md#store-url-format), separated by whitespace.
           These are not used by default, but users of the Nix daemon can enable them by specifying [`substituters`](#conf-substituters).
 
           Unprivileged users (those set in only [`allowed-users`](#conf-allowed-users) but not [`trusted-users`](#conf-trusted-users)) can pass as `substituters` only those URLs listed in `trusted-substituters`.
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index f16949f42..3d3919882 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -20,7 +20,7 @@
 
 namespace nix {
 
-/* TODO: Separate these store impls into different files, give them better names */
+/* TODO: Separate these store types into different files, give them better names */
 RemoteStore::RemoteStore(const Params & params)
     : RemoteStoreConfig(params)
     , Store(params)
diff --git a/src/libstore/ssh-store-config.hh b/src/libstore/ssh-store-config.hh
index bf55d20cf..4ce4ffc4c 100644
--- a/src/libstore/ssh-store-config.hh
+++ b/src/libstore/ssh-store-config.hh
@@ -20,7 +20,7 @@ struct CommonSSHStoreConfig : virtual StoreConfig
 
     const Setting<std::string> remoteStore{this, "", "remote-store",
         R"(
-          [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
+          [Store URL](@docroot@/store/types/index.md#store-url-format)
           to be used on the remote machine. The default is `auto`
           (i.e. use the Nix daemon or `/nix/store` directly).
         )"};
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 8b6bf9aed..e28baf34e 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -30,7 +30,7 @@
 namespace nix {
 
 /**
- * About the class hierarchy of the store implementations:
+ * About the class hierarchy of the store types:
  *
  * Each store type `Foo` consists of two classes:
  *
@@ -962,7 +962,7 @@ OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * ev
  * - ‘ssh://[user@]<host>’: A remote Nix store accessed by running
  *   ‘nix-store --serve’ via SSH.
  *
- * You can pass parameters to the store implementation by appending
+ * You can pass parameters to the store type by appending
  * ‘?key=value&key=value&...’ to the URI.
  */
 ref<Store> openStore(const std::string & uri = settings.storeUri.get(),
diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc
index 2418e3f4c..e4bdb8cb3 100644
--- a/src/libutil/experimental-features.cc
+++ b/src/libutil/experimental-features.cc
@@ -252,7 +252,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
         .tag = Xp::ReadOnlyLocalStore,
         .name = "read-only-local-store",
         .description = R"(
-            Allow the use of the `read-only` parameter in [local store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) URIs.
+            Allow the use of the `read-only` parameter in [local store](@docroot@/store/types/local-store.md) URIs.
         )",
     },
     {
diff --git a/src/nix/nix.md b/src/nix/nix.md
index eb150f03b..749456014 100644
--- a/src/nix/nix.md
+++ b/src/nix/nix.md
@@ -235,10 +235,14 @@ operate are determined as follows:
 
 # Nix stores
 
-Most `nix` subcommands operate on a *Nix store*. These are documented
-in [`nix help-stores`](./nix3-help-stores.md).
+Most `nix` subcommands operate on a *Nix store*.
+The various store types are documented in the
+[Store Types](@docroot@/store/types/index.md)
+section of the manual.
 
-# Shebang interpreter 
+The same information is also available from the [`nix help-stores`](./nix3-help-stores.md) command.
+
+# Shebang interpreter
 
 The `nix` command can be used as a `#!` interpreter.
 Arguments to Nix can be passed on subsequent lines in the script.

From 333ea684b065318aa49aec367c995b3d8c5d65ed Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <silvan.mosberger@tweag.io>
Date: Fri, 1 Dec 2023 01:39:52 +0100
Subject: [PATCH 237/294] Add boost::regex regression test

---
 src/libexpr/tests/primops.cc | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/libexpr/tests/primops.cc b/src/libexpr/tests/primops.cc
index d820b860e..7485fa0d0 100644
--- a/src/libexpr/tests/primops.cc
+++ b/src/libexpr/tests/primops.cc
@@ -814,6 +814,14 @@ namespace nix {
         ASSERT_THAT(*v.listElems()[0], IsStringEq("FOO"));
     }
 
+    TEST_F(PrimOpTest, match5) {
+        // The regex "\\{}" is valid and matches the string "{}".
+        // Caused a regression before when trying to switch from std::regex to boost::regex.
+        // See https://github.com/NixOS/nix/pull/7762#issuecomment-1834303659
+        auto v = eval("builtins.match \"\\\\{}\" \"{}\"");
+        ASSERT_THAT(v, IsListOfSize(0));
+    }
+
     TEST_F(PrimOpTest, attrNames) {
         auto v = eval("builtins.attrNames { x = 1; y = 2; z = 3; a = 2; }");
         ASSERT_THAT(v, IsListOfSize(4));

From d5e934fb73496a2509755be5945a8bcf1730d59d Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Fri, 1 Dec 2023 01:54:48 +0100
Subject: [PATCH 238/294] add redirect to new store page

---
 doc/manual/_redirects | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/manual/_redirects b/doc/manual/_redirects
index 2038671d7..62c693c97 100644
--- a/doc/manual/_redirects
+++ b/doc/manual/_redirects
@@ -31,9 +31,9 @@
 /installation/installation /installation 301!
 
 /package-management/basic-package-mgmt /command-ref/nix-env 301!
-/package-management/channels* /command-ref/nix-channel 301!
+/package-management/channels /command-ref/nix-channel 301!
 /package-management/package-management /package-management 301!
-/package-management/s3-substituter* /command-ref/new-cli/nix3-help-stores#s3-binary-cache-store 301!
+/package-management/s3-substituter /store/types/s3-binary-cache-store 301!
 
 /protocols/protocols /protocols 301!
 

From ea2dd166235e049699cf7f70c243c2b83089f824 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <theophane.hufschmitt@tweag.io>
Date: Fri, 1 Dec 2023 15:35:21 +0100
Subject: [PATCH 239/294] Use a proper enum rather than a boolean in
 runProgramInStore

Makes the call-site much easier to understand.
---
 src/nix/develop.cc | 2 +-
 src/nix/fmt.cc     | 2 +-
 src/nix/run.cc     | 8 ++++----
 src/nix/run.hh     | 7 ++++++-
 4 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index ae9be79a3..606b044b0 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -662,7 +662,7 @@ struct CmdDevelop : Common, MixEnvironment
             }
         }
 
-        runProgramInStore(store, true, shell, args, buildEnvironment.getSystem());
+        runProgramInStore(store, UseSearchPath::Use, shell, args, buildEnvironment.getSystem());
     }
 };
 
diff --git a/src/nix/fmt.cc b/src/nix/fmt.cc
index 396c93dbb..059904150 100644
--- a/src/nix/fmt.cc
+++ b/src/nix/fmt.cc
@@ -49,7 +49,7 @@ struct CmdFmt : SourceExprCommand {
             }
         }
 
-        runProgramInStore(store, false, app.program, programArgs);
+        runProgramInStore(store, UseSearchPath::DontUse, app.program, programArgs);
     };
 };
 
diff --git a/src/nix/run.cc b/src/nix/run.cc
index d531f712d..efc0c56a1 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -25,7 +25,7 @@ std::string chrootHelperName = "__run_in_chroot";
 namespace nix {
 
 void runProgramInStore(ref<Store> store,
-    bool search,
+    UseSearchPath useSearchPath,
     const std::string & program,
     const Strings & args,
     std::optional<std::string_view> system)
@@ -59,7 +59,7 @@ void runProgramInStore(ref<Store> store,
     if (system)
         setPersonality(*system);
 
-    if (search)
+    if (useSearchPath == UseSearchPath::Use)
         execvp(program.c_str(), stringsToCharPtrs(args).data());
     else
         execv(program.c_str(), stringsToCharPtrs(args).data());
@@ -136,7 +136,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment
         Strings args;
         for (auto & arg : command) args.push_back(arg);
 
-        runProgramInStore(store, true, *command.begin(), args);
+        runProgramInStore(store, UseSearchPath::Use, *command.begin(), args);
     }
 };
 
@@ -198,7 +198,7 @@ struct CmdRun : InstallableValueCommand
         Strings allArgs{app.program};
         for (auto & i : args) allArgs.push_back(i);
 
-        runProgramInStore(store, false, app.program, allArgs);
+        runProgramInStore(store, UseSearchPath::DontUse, app.program, allArgs);
     }
 };
 
diff --git a/src/nix/run.hh b/src/nix/run.hh
index c62287e7e..a55917b06 100644
--- a/src/nix/run.hh
+++ b/src/nix/run.hh
@@ -5,8 +5,13 @@
 
 namespace nix {
 
+enum struct UseSearchPath {
+    Use,
+    DontUse
+};
+
 void runProgramInStore(ref<Store> store,
-    bool search,
+    UseSearchPath useSearchPath,
     const std::string & program,
     const Strings & args,
     std::optional<std::string_view> system = std::nullopt);

From d59bdbe4fd757d99b6625db1d3560a39a371d9e9 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 1 Dec 2023 10:20:19 -0500
Subject: [PATCH 240/294] Add two missing `#include "nar-info.hh"`

GitHub's racy CI caused this oversight to sneak through.
---
 src/libstore/tests/nar-info.cc | 1 +
 src/nix/path-info.cc           | 1 +
 2 files changed, 2 insertions(+)

diff --git a/src/libstore/tests/nar-info.cc b/src/libstore/tests/nar-info.cc
index c5b21d56b..88e6e1add 100644
--- a/src/libstore/tests/nar-info.cc
+++ b/src/libstore/tests/nar-info.cc
@@ -2,6 +2,7 @@
 #include <gtest/gtest.h>
 
 #include "path-info.hh"
+#include "nar-info.hh"
 
 #include "tests/characterization.hh"
 #include "tests/libstore.hh"
diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index 080d6bbf1..5f10cfb61 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -2,6 +2,7 @@
 #include "shared.hh"
 #include "store-api.hh"
 #include "common-args.hh"
+#include "nar-info.hh"
 
 #include <algorithm>
 #include <array>

From 91b6833686a6a6d9eac7f3f66393ec89ef1d3b57 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 25 Aug 2023 10:20:28 -0400
Subject: [PATCH 241/294] Move tests to separate directories, and document

Today, with the tests inside a `tests` intermingled with the
corresponding library's source code, we have a few problems:

- We have to be careful that wildcards don't end up with tests being
  built as part of Nix proper, or test headers being installed as part
  of Nix proper.

- Tests in libraries but not executables is not right:

  - It means each executable runs the previous unit tests again, because
    it needs the libraries.

  - It doesn't work right on Windows, which doesn't want you to load a
    DLL just for the side global variable . It could be made to work
    with the dlopen equivalent, but that's gross!

This reorg solves these problems.

There is a remaining problem which is that sibbling headers (like
`hash.hh` the test header vs `hash.hh` the main `libnixutil` header) end
up shadowing each other. This PR doesn't solve that. That is left as
future work for a future PR.

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
---
 .gitignore                                    |   6 +-
 Makefile                                      |  10 ++-
 doc/internal-api/doxygen.cfg.in               |  12 ++-
 doc/manual/src/contributing/testing.md        |  67 ++++++++++----
 flake.nix                                     |   2 +-
 mk/common-test.sh                             |   2 +-
 mk/programs.mk                                |   2 +-
 src/libexpr/tests/local.mk                    |  23 -----
 src/libstore/tests/local.mk                   |  37 --------
 src/libutil/tests/local.mk                    |  41 ---------
 tests/unit/libexpr-support/local.mk           |  23 +++++
 .../unit/libexpr-support}/tests/libexpr.hh    |   0
 .../libexpr-support/tests/value/context.cc    |  30 +++++++
 .../libexpr-support}/tests/value/context.hh   |   2 +-
 .../unit/libexpr}/derived-path.cc             |   0
 .../unit/libexpr}/error_traces.cc             |   0
 .../tests => tests/unit/libexpr}/flakeref.cc  |   0
 .../tests => tests/unit/libexpr}/json.cc      |   0
 tests/unit/libexpr/local.mk                   |  36 ++++++++
 .../tests => tests/unit/libexpr}/primops.cc   |   0
 .../unit/libexpr}/search-path.cc              |   0
 .../tests => tests/unit/libexpr}/trivial.cc   |   0
 .../unit/libexpr}/value/context.cc            |  30 -------
 .../unit/libexpr}/value/print.cc              |   0
 tests/unit/libstore-support/local.mk          |  21 +++++
 .../libstore-support/tests/derived-path.cc    |  57 ++++++++++++
 .../libstore-support}/tests/derived-path.hh   |   0
 .../unit/libstore-support}/tests/libstore.hh  |   0
 .../libstore-support/tests/outputs-spec.cc    |  24 +++++
 .../libstore-support}/tests/outputs-spec.hh   |   2 +-
 tests/unit/libstore-support/tests/path.cc     |  82 ++++++++++++++++++
 .../unit/libstore-support}/tests/path.hh      |   3 +
 .../unit/libstore-support}/tests/protocol.hh  |   2 +-
 .../unit/libstore}/common-protocol.cc         |   0
 .../data}/common-protocol/content-address.bin | Bin
 .../data}/common-protocol/drv-output.bin      | Bin
 .../optional-content-address.bin              | Bin
 .../common-protocol/optional-store-path.bin   | Bin
 .../data}/common-protocol/realisation.bin     | Bin
 .../libstore/data}/common-protocol/set.bin    | Bin
 .../data}/common-protocol/store-path.bin      | Bin
 .../libstore/data}/common-protocol/string.bin | Bin
 .../libstore/data}/common-protocol/vector.bin | Bin
 .../derivation/bad-old-version-dyn-deps.drv   |   0
 .../libstore/data}/derivation/bad-version.drv |   0
 .../data}/derivation/dynDerivationDeps.drv    |   0
 .../data}/derivation/dynDerivationDeps.json   |   0
 .../data}/derivation/output-caFixedFlat.json  |   0
 .../data}/derivation/output-caFixedNAR.json   |   0
 .../data}/derivation/output-caFixedText.json  |   0
 .../data}/derivation/output-caFloating.json   |   0
 .../data}/derivation/output-deferred.json     |   0
 .../data}/derivation/output-impure.json       |   0
 .../derivation/output-inputAddressed.json     |   0
 .../unit/libstore/data}/derivation/simple.drv |   0
 .../libstore/data}/derivation/simple.json     |   0
 .../unit/libstore/data}/nar-info/impure.json  |   0
 .../unit/libstore/data}/nar-info/pure.json    |   0
 .../unit/libstore/data}/path-info/impure.json |   0
 .../unit/libstore/data}/path-info/pure.json   |   0
 .../data}/serve-protocol/build-result-2.2.bin | Bin
 .../data}/serve-protocol/build-result-2.3.bin | Bin
 .../data}/serve-protocol/build-result-2.6.bin | Bin
 .../data}/serve-protocol/content-address.bin  | Bin
 .../data}/serve-protocol/drv-output.bin       | Bin
 .../optional-content-address.bin              | Bin
 .../serve-protocol/optional-store-path.bin    | Bin
 .../data}/serve-protocol/realisation.bin      | Bin
 .../libstore/data}/serve-protocol/set.bin     | Bin
 .../data}/serve-protocol/store-path.bin       | Bin
 .../libstore/data}/serve-protocol/string.bin  | Bin
 .../libstore/data}/serve-protocol/vector.bin  | Bin
 .../worker-protocol/build-result-1.27.bin     | Bin
 .../worker-protocol/build-result-1.28.bin     | Bin
 .../worker-protocol/build-result-1.29.bin     | Bin
 .../data}/worker-protocol/content-address.bin | Bin
 .../worker-protocol/derived-path-1.29.bin     | Bin
 .../worker-protocol/derived-path-1.30.bin     | Bin
 .../data}/worker-protocol/drv-output.bin      | Bin
 .../keyed-build-result-1.29.bin               | Bin
 .../optional-content-address.bin              | Bin
 .../worker-protocol/optional-store-path.bin   | Bin
 .../worker-protocol/optional-trusted-flag.bin | Bin
 .../data}/worker-protocol/realisation.bin     | Bin
 .../libstore/data}/worker-protocol/set.bin    | Bin
 .../data}/worker-protocol/store-path.bin      | Bin
 .../libstore/data}/worker-protocol/string.bin | Bin
 .../unkeyed-valid-path-info-1.15.bin          | Bin
 .../worker-protocol/valid-path-info-1.15.bin  | Bin
 .../worker-protocol/valid-path-info-1.16.bin  | Bin
 .../libstore/data}/worker-protocol/vector.bin | Bin
 .../unit/libstore}/derivation.cc              |   2 +-
 .../unit/libstore}/derived-path.cc            |  53 -----------
 .../unit/libstore}/downstream-placeholder.cc  |   0
 tests/unit/libstore/local.mk                  |  31 +++++++
 .../tests => tests/unit/libstore}/machines.cc |   4 +-
 .../unit/libstore}/nar-info-disk-cache.cc     |   0
 .../tests => tests/unit/libstore}/nar-info.cc |   2 +-
 .../unit/libstore}/outputs-spec.cc            |  27 +-----
 .../unit/libstore}/path-info.cc               |   2 +-
 .../tests => tests/unit/libstore}/path.cc     |  73 ----------------
 .../unit/libstore}/references.cc              |   0
 .../unit/libstore}/serve-protocol.cc          |   0
 .../libstore}/test-data/machines.bad_format   |   0
 .../unit/libstore}/test-data/machines.valid   |   0
 .../unit/libstore}/worker-protocol.cc         |   0
 tests/unit/libutil-support/local.mk           |  19 ++++
 .../tests/characterization.hh                 |   4 +-
 tests/unit/libutil-support/tests/hash.cc      |  20 +++++
 .../unit/libutil-support}/tests/hash.hh       |   0
 .../tests => tests/unit/libutil}/args.cc      |   6 +-
 .../unit/libutil}/canon-path.cc               |   0
 .../unit/libutil}/chunked-vector.cc           |   0
 .../tests => tests/unit/libutil}/closure.cc   |   0
 .../unit/libutil}/compression.cc              |   0
 .../tests => tests/unit/libutil}/config.cc    |   0
 .../unit/libutil/data}/git/check-data.sh      |   2 +-
 .../libutil/data}/git/hello-world-blob.bin    | Bin
 .../unit/libutil/data}/git/hello-world.bin    | Bin
 .../unit/libutil/data}/git/tree.bin           | Bin
 .../unit/libutil/data}/git/tree.txt           |   0
 .../tests => tests/unit/libutil}/git.cc       |   6 +-
 .../tests => tests/unit/libutil}/hash.cc      |  20 +----
 .../tests => tests/unit/libutil}/hilite.cc    |   0
 .../unit/libutil}/json-utils.cc               |   0
 tests/unit/libutil/local.mk                   |  31 +++++++
 .../tests => tests/unit/libutil}/logging.cc   |   0
 .../tests => tests/unit/libutil}/lru-cache.cc |   0
 .../tests => tests/unit/libutil}/pool.cc      |   0
 .../unit/libutil}/references.cc               |   0
 .../unit/libutil}/suggestions.cc              |   0
 .../tests => tests/unit/libutil}/tests.cc     |   0
 .../tests => tests/unit/libutil}/url.cc       |   0
 .../unit/libutil}/xml-writer.cc               |   0
 134 files changed, 464 insertions(+), 352 deletions(-)
 delete mode 100644 src/libexpr/tests/local.mk
 delete mode 100644 src/libstore/tests/local.mk
 delete mode 100644 src/libutil/tests/local.mk
 create mode 100644 tests/unit/libexpr-support/local.mk
 rename {src/libexpr => tests/unit/libexpr-support}/tests/libexpr.hh (100%)
 create mode 100644 tests/unit/libexpr-support/tests/value/context.cc
 rename {src/libexpr => tests/unit/libexpr-support}/tests/value/context.hh (95%)
 rename {src/libexpr/tests => tests/unit/libexpr}/derived-path.cc (100%)
 rename {src/libexpr/tests => tests/unit/libexpr}/error_traces.cc (100%)
 rename {src/libexpr/tests => tests/unit/libexpr}/flakeref.cc (100%)
 rename {src/libexpr/tests => tests/unit/libexpr}/json.cc (100%)
 create mode 100644 tests/unit/libexpr/local.mk
 rename {src/libexpr/tests => tests/unit/libexpr}/primops.cc (100%)
 rename {src/libexpr/tests => tests/unit/libexpr}/search-path.cc (100%)
 rename {src/libexpr/tests => tests/unit/libexpr}/trivial.cc (100%)
 rename {src/libexpr/tests => tests/unit/libexpr}/value/context.cc (83%)
 rename {src/libexpr/tests => tests/unit/libexpr}/value/print.cc (100%)
 create mode 100644 tests/unit/libstore-support/local.mk
 create mode 100644 tests/unit/libstore-support/tests/derived-path.cc
 rename {src/libstore => tests/unit/libstore-support}/tests/derived-path.hh (100%)
 rename {src/libstore => tests/unit/libstore-support}/tests/libstore.hh (100%)
 create mode 100644 tests/unit/libstore-support/tests/outputs-spec.cc
 rename {src/libstore => tests/unit/libstore-support}/tests/outputs-spec.hh (89%)
 create mode 100644 tests/unit/libstore-support/tests/path.cc
 rename {src/libstore => tests/unit/libstore-support}/tests/path.hh (82%)
 rename {src/libstore => tests/unit/libstore-support}/tests/protocol.hh (96%)
 rename {src/libstore/tests => tests/unit/libstore}/common-protocol.cc (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/content-address.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/drv-output.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/optional-content-address.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/optional-store-path.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/realisation.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/set.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/store-path.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/string.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/vector.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/bad-old-version-dyn-deps.drv (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/bad-version.drv (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/dynDerivationDeps.drv (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/dynDerivationDeps.json (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-caFixedFlat.json (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-caFixedNAR.json (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-caFixedText.json (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-caFloating.json (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-deferred.json (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-impure.json (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-inputAddressed.json (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/simple.drv (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/simple.json (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/nar-info/impure.json (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/nar-info/pure.json (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/path-info/impure.json (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/path-info/pure.json (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/build-result-2.2.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/build-result-2.3.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/build-result-2.6.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/content-address.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/drv-output.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/optional-content-address.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/optional-store-path.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/realisation.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/set.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/store-path.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/string.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/vector.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/build-result-1.27.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/build-result-1.28.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/build-result-1.29.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/content-address.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/derived-path-1.29.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/derived-path-1.30.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/drv-output.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/keyed-build-result-1.29.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/optional-content-address.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/optional-store-path.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/optional-trusted-flag.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/realisation.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/set.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/store-path.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/string.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/unkeyed-valid-path-info-1.15.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/valid-path-info-1.15.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/valid-path-info-1.16.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/vector.bin (100%)
 rename {src/libstore/tests => tests/unit/libstore}/derivation.cc (99%)
 rename {src/libstore/tests => tests/unit/libstore}/derived-path.cc (66%)
 rename {src/libstore/tests => tests/unit/libstore}/downstream-placeholder.cc (100%)
 create mode 100644 tests/unit/libstore/local.mk
 rename {src/libstore/tests => tests/unit/libstore}/machines.cc (97%)
 rename {src/libstore/tests => tests/unit/libstore}/nar-info-disk-cache.cc (100%)
 rename {src/libstore/tests => tests/unit/libstore}/nar-info.cc (98%)
 rename {src/libstore/tests => tests/unit/libstore}/outputs-spec.cc (92%)
 rename {src/libstore/tests => tests/unit/libstore}/path-info.cc (97%)
 rename {src/libstore/tests => tests/unit/libstore}/path.cc (59%)
 rename {src/libstore/tests => tests/unit/libstore}/references.cc (100%)
 rename {src/libstore/tests => tests/unit/libstore}/serve-protocol.cc (100%)
 rename {src/libstore/tests => tests/unit/libstore}/test-data/machines.bad_format (100%)
 rename {src/libstore/tests => tests/unit/libstore}/test-data/machines.valid (100%)
 rename {src/libstore/tests => tests/unit/libstore}/worker-protocol.cc (100%)
 create mode 100644 tests/unit/libutil-support/local.mk
 rename {src/libutil => tests/unit/libutil-support}/tests/characterization.hh (95%)
 create mode 100644 tests/unit/libutil-support/tests/hash.cc
 rename {src/libutil => tests/unit/libutil-support}/tests/hash.hh (100%)
 rename {src/libutil/tests => tests/unit/libutil}/args.cc (98%)
 rename {src/libutil/tests => tests/unit/libutil}/canon-path.cc (100%)
 rename {src/libutil/tests => tests/unit/libutil}/chunked-vector.cc (100%)
 rename {src/libutil/tests => tests/unit/libutil}/closure.cc (100%)
 rename {src/libutil/tests => tests/unit/libutil}/compression.cc (100%)
 rename {src/libutil/tests => tests/unit/libutil}/config.cc (100%)
 rename {unit-test-data/libutil => tests/unit/libutil/data}/git/check-data.sh (98%)
 rename {unit-test-data/libutil => tests/unit/libutil/data}/git/hello-world-blob.bin (100%)
 rename {unit-test-data/libutil => tests/unit/libutil/data}/git/hello-world.bin (100%)
 rename {unit-test-data/libutil => tests/unit/libutil/data}/git/tree.bin (100%)
 rename {unit-test-data/libutil => tests/unit/libutil/data}/git/tree.txt (100%)
 rename {src/libutil/tests => tests/unit/libutil}/git.cc (97%)
 rename {src/libutil/tests => tests/unit/libutil}/hash.cc (92%)
 rename {src/libutil/tests => tests/unit/libutil}/hilite.cc (100%)
 rename {src/libutil/tests => tests/unit/libutil}/json-utils.cc (100%)
 create mode 100644 tests/unit/libutil/local.mk
 rename {src/libutil/tests => tests/unit/libutil}/logging.cc (100%)
 rename {src/libutil/tests => tests/unit/libutil}/lru-cache.cc (100%)
 rename {src/libutil/tests => tests/unit/libutil}/pool.cc (100%)
 rename {src/libutil/tests => tests/unit/libutil}/references.cc (100%)
 rename {src/libutil/tests => tests/unit/libutil}/suggestions.cc (100%)
 rename {src/libutil/tests => tests/unit/libutil}/tests.cc (100%)
 rename {src/libutil/tests => tests/unit/libutil}/url.cc (100%)
 rename {src/libutil/tests => tests/unit/libutil}/xml-writer.cc (100%)

diff --git a/.gitignore b/.gitignore
index 767a5d6ed..d9f9d949b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -45,14 +45,14 @@ perl/Makefile.config
 /src/libexpr/parser-tab.hh
 /src/libexpr/parser-tab.output
 /src/libexpr/nix.tbl
-/src/libexpr/tests/libnixexpr-tests
+/tests/unit/libexpr/libnixexpr-tests
 
 # /src/libstore/
 *.gen.*
-/src/libstore/tests/libnixstore-tests
+/tests/unit/libstore/libnixstore-tests
 
 # /src/libutil/
-/src/libutil/tests/libnixutil-tests
+/tests/unit/libutil/libnixutil-tests
 
 /src/nix/nix
 
diff --git a/Makefile b/Makefile
index 92727bea5..eea297c89 100644
--- a/Makefile
+++ b/Makefile
@@ -25,11 +25,13 @@ makefiles = \
 endif
 
 ifeq ($(ENABLE_BUILD)_$(ENABLE_TESTS), yes_yes)
-UNIT_TEST_ENV = _NIX_TEST_UNIT_DATA=unit-test-data
 makefiles += \
-  src/libutil/tests/local.mk \
-  src/libstore/tests/local.mk \
-  src/libexpr/tests/local.mk
+  tests/unit/libutil/local.mk \
+  tests/unit/libutil-support/local.mk \
+  tests/unit/libstore/local.mk \
+  tests/unit/libstore-support/local.mk \
+  tests/unit/libexpr/local.mk \
+  tests/unit/libexpr-support/local.mk
 endif
 
 ifeq ($(ENABLE_TESTS), yes)
diff --git a/doc/internal-api/doxygen.cfg.in b/doc/internal-api/doxygen.cfg.in
index 599be2470..ad5af97e6 100644
--- a/doc/internal-api/doxygen.cfg.in
+++ b/doc/internal-api/doxygen.cfg.in
@@ -39,17 +39,21 @@ INPUT                  = \
   src/libcmd \
   src/libexpr \
   src/libexpr/flake \
-  src/libexpr/tests \
-  src/libexpr/tests/value \
+  tests/unit/libexpr \
+  tests/unit/libexpr/value \
+  tests/unit/libexpr/test \
+  tests/unit/libexpr/test/value \
   src/libexpr/value \
   src/libfetchers \
   src/libmain \
   src/libstore \
   src/libstore/build \
   src/libstore/builtins \
-  src/libstore/tests \
+  tests/unit/libstore \
+  tests/unit/libstore/test \
   src/libutil \
-  src/libutil/tests \
+  tests/unit/libutil \
+  tests/unit/libutil/test \
   src/nix \
   src/nix-env \
   src/nix-store
diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md
index 0b45b88a3..d8d162379 100644
--- a/doc/manual/src/contributing/testing.md
+++ b/doc/manual/src/contributing/testing.md
@@ -20,6 +20,7 @@ The unit tests are defined using the [googletest] and [rapidcheck] frameworks.
 
 [googletest]: https://google.github.io/googletest/
 [rapidcheck]: https://github.com/emil-e/rapidcheck
+[property testing]: https://en.wikipedia.org/wiki/Property_testing
 
 ### Source and header layout
 
@@ -28,34 +29,50 @@ The unit tests are defined using the [googletest] and [rapidcheck] frameworks.
 > ```
 > src
 > ├── libexpr
+> │   ├── local.mk
 > │   ├── value/context.hh
 > │   ├── value/context.cc
+> │   …
+> │
+> ├── tests
 > │   │
 > │   …
->     └── tests
-> │       ├── value/context.hh
-> │       ├── value/context.cc
+> │   └── unit
+> │       ├── libutil
+> │       │   ├── local.mk
+> │       │   …
+> │       │   └── data
+> │       │       ├── git/tree.txt
+> │       │       …
 > │       │
-> │       …
-> │
-> ├── unit-test-data
-> │   ├── libstore
-> │   │   ├── worker-protocol/content-address.bin
-> │   │   …
-> │   …
+> │       ├── libexpr-support
+> │       │   ├── local.mk
+> │       │   └── tests
+> │       │       ├── value/context.hh
+> │       │       ├── value/context.cc
+> │       │       …
+> │       │
+> │       ├── libexpr
+> │       …   ├── local.mk
+> │           ├── value/context.cc
+> │           …
 > …
 > ```
 
-The unit tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `src/${library_shortname}/tests` within the directory for the library (`src/${library_shortname}`).
+The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `tests/unit/${library_name_without-nix}`.
+Given a interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `tests/unit/libexpr/tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `tests/unit/libexpr-support/tests/value/context.{hh,cc}`.
 
-The data is in `unit-test-data`, with one subdir per library, with the same name as where the code goes.
-For example, `libnixstore` code is in `src/libstore`, and its test data is in `unit-test-data/libstore`.
-The path to the `unit-test-data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`.
+Data for unit tests is stored in a `data` subdir of the directory for each unit test executable.
+For example, `libnixstore` code is in `src/libstore`, and its test data is in `tests/unit/libstore/data`.
+The path to the `tests/unit/data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`.
+Note that each executable only gets the data for its tests.
 
-> **Note**
-> Due to the way googletest works, downstream unit test executables will actually include and re-run upstream library tests.
-> Therefore it is important that the same value for `_NIX_TEST_UNIT_DATA` be used with the tests for each library.
-> That is why we have the test data nested within a single `unit-test-data` directory.
+The unit test libraries are in `tests/unit/${library_name_without-nix}-lib`.
+All headers are in a `tests` subdirectory so they are included with `#include "tests/"`.
+
+The use of all these separate directories for the unit tests might seem inconvenient, as for example the tests are not "right next to" the part of the code they are testing.
+But organizing the tests this way has one big benefit:
+there is no risk of any build-system wildcards for the library accidentally picking up test code that should not built and installed as part of the library.
 
 ### Running tests
 
@@ -69,7 +86,7 @@ See [functional characterisation testing](#characterisation-testing-functional)
 Like with the functional characterisation, `_NIX_TEST_ACCEPT=1` is also used.
 For example:
 ```shell-session
-$ _NIX_TEST_ACCEPT=1 make libstore-tests-exe_RUN
+$ _NIX_TEST_ACCEPT=1 make libstore-tests_RUN
 ...
 [  SKIPPED ] WorkerProtoTest.string_read
 [  SKIPPED ] WorkerProtoTest.string_write
@@ -80,6 +97,18 @@ $ _NIX_TEST_ACCEPT=1 make libstore-tests-exe_RUN
 will regenerate the "golden master" expected result for the `libnixstore` characterisation tests.
 The characterisation tests will mark themselves "skipped" since they regenerated the expected result instead of actually testing anything.
 
+### Unit test support libraries
+
+There are headers and code which are not just used to test the library in question, but also downstream libraries.
+For example, we do [property testing] with the [rapidcheck] library.
+This requires writing `Arbitrary` "instances", which are used to describe how to generate values of a given type for the sake of running property tests.
+Because types contain other types, `Arbitrary` "instances" for some type are not just useful for testing that type, but also any other type that contains it.
+Downstream types frequently contain upstream types, so it is very important that we share arbitrary instances so that downstream libraries' property tests can also use them.
+
+It is important that these testing libraries don't contain any actual tests themselves.
+On some platforms they would be run as part of every test executable that uses them, which is redundant.
+On other platforms they wouldn't be run at all.
+
 ## Functional tests
 
 The functional tests reside under the `tests/functional` directory and are listed in `tests/functional/local.mk`.
diff --git a/flake.nix b/flake.nix
index 822b3d31e..e2e510cbc 100644
--- a/flake.nix
+++ b/flake.nix
@@ -93,7 +93,7 @@
           ./misc
           ./precompiled-headers.h
           ./src
-          ./unit-test-data
+          ./tests/unit
           ./COPYING
           ./scripts/local.mk
           functionalTestFiles
diff --git a/mk/common-test.sh b/mk/common-test.sh
index 00ccd1584..2783d293b 100644
--- a/mk/common-test.sh
+++ b/mk/common-test.sh
@@ -1,7 +1,7 @@
 # Remove overall test dir (at most one of the two should match) and
 # remove file extension.
 test_name=$(echo -n "$test" | sed \
-    -e "s|^unit-test-data/||" \
+    -e "s|^tests/unit/[^/]*/data/||" \
     -e "s|^tests/functional/||" \
     -e "s|\.sh$||" \
     )
diff --git a/mk/programs.mk b/mk/programs.mk
index a88d9d949..6235311e9 100644
--- a/mk/programs.mk
+++ b/mk/programs.mk
@@ -87,6 +87,6 @@ define build-program
   # Phony target to run this program (typically as a dependency of 'check').
   .PHONY: $(1)_RUN
   $(1)_RUN: $$($(1)_PATH)
-	$(trace-test) $$(UNIT_TEST_ENV) $$($(1)_PATH)
+	$(trace-test) $$($(1)_ENV) $$($(1)_PATH)
 
 endef
diff --git a/src/libexpr/tests/local.mk b/src/libexpr/tests/local.mk
deleted file mode 100644
index 7689a03e0..000000000
--- a/src/libexpr/tests/local.mk
+++ /dev/null
@@ -1,23 +0,0 @@
-check: libexpr-tests_RUN
-
-programs += libexpr-tests
-
-libexpr-tests_NAME := libnixexpr-tests
-
-libexpr-tests_DIR := $(d)
-
-ifeq ($(INSTALL_UNIT_TESTS), yes)
-  libexpr-tests_INSTALL_DIR := $(checkbindir)
-else
-  libexpr-tests_INSTALL_DIR :=
-endif
-
-libexpr-tests_SOURCES := \
-    $(wildcard $(d)/*.cc) \
-    $(wildcard $(d)/value/*.cc)
-
-libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests -I src/libfetchers
-
-libexpr-tests_LIBS = libstore-tests libutils-tests libexpr libutil libstore libfetchers
-
-libexpr-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock
diff --git a/src/libstore/tests/local.mk b/src/libstore/tests/local.mk
deleted file mode 100644
index e9b8b4f99..000000000
--- a/src/libstore/tests/local.mk
+++ /dev/null
@@ -1,37 +0,0 @@
-check: libstore-tests-exe_RUN
-
-programs += libstore-tests-exe
-
-libstore-tests-exe_NAME = libnixstore-tests
-
-libstore-tests-exe_DIR := $(d)
-
-ifeq ($(INSTALL_UNIT_TESTS), yes)
-  libstore-tests-exe_INSTALL_DIR := $(checkbindir)
-else
-  libstore-tests-exe_INSTALL_DIR :=
-endif
-
-libstore-tests-exe_LIBS = libstore-tests
-
-libstore-tests-exe_LDFLAGS := $(GTEST_LIBS)
-
-libraries += libstore-tests
-
-libstore-tests_NAME = libnixstore-tests
-
-libstore-tests_DIR := $(d)
-
-ifeq ($(INSTALL_UNIT_TESTS), yes)
-  libstore-tests_INSTALL_DIR := $(checklibdir)
-else
-  libstore-tests_INSTALL_DIR :=
-endif
-
-libstore-tests_SOURCES := $(wildcard $(d)/*.cc)
-
-libstore-tests_CXXFLAGS += -I src/libstore -I src/libutil
-
-libstore-tests_LIBS = libutil-tests libstore libutil
-
-libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS)
diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk
deleted file mode 100644
index 66886c45f..000000000
--- a/src/libutil/tests/local.mk
+++ /dev/null
@@ -1,41 +0,0 @@
-check: libutil-tests-exe_RUN
-
-programs += libutil-tests-exe
-
-libutil-tests-exe_NAME = libnixutil-tests
-
-libutil-tests-exe_DIR := $(d)
-
-ifeq ($(INSTALL_UNIT_TESTS), yes)
-  libutil-tests-exe_INSTALL_DIR := $(checkbindir)
-else
-  libutil-tests-exe_INSTALL_DIR :=
-endif
-
-libutil-tests-exe_LIBS = libutil-tests
-
-libutil-tests-exe_LDFLAGS := $(GTEST_LIBS)
-
-libraries += libutil-tests
-
-libutil-tests_NAME = libnixutil-tests
-
-libutil-tests_DIR := $(d)
-
-ifeq ($(INSTALL_UNIT_TESTS), yes)
-  libutil-tests_INSTALL_DIR := $(checklibdir)
-else
-  libutil-tests_INSTALL_DIR :=
-endif
-
-libutil-tests_SOURCES := $(wildcard $(d)/*.cc)
-
-libutil-tests_CXXFLAGS += -I src/libutil
-
-libutil-tests_LIBS = libutil
-
-libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS)
-
-check: unit-test-data/libutil/git/check-data.sh.test
-
-$(eval $(call run-test,unit-test-data/libutil/git/check-data.sh))
diff --git a/tests/unit/libexpr-support/local.mk b/tests/unit/libexpr-support/local.mk
new file mode 100644
index 000000000..28e87b8f2
--- /dev/null
+++ b/tests/unit/libexpr-support/local.mk
@@ -0,0 +1,23 @@
+libraries += libexpr-test-support
+
+libexpr-test-support_NAME = libnixexpr-test-support
+
+libexpr-test-support_DIR := $(d)
+
+ifeq ($(INSTALL_UNIT_TESTS), yes)
+  libexpr-test-support_INSTALL_DIR := $(checklibdir)
+else
+  libexpr-test-support_INSTALL_DIR :=
+endif
+
+libexpr-test-support_SOURCES := \
+    $(wildcard $(d)/tests/*.cc) \
+    $(wildcard $(d)/tests/value/*.cc)
+
+libexpr-test-support_CXXFLAGS += $(libexpr-tests_EXTRA_INCLUDES)
+
+libexpr-test-support_LIBS = \
+    libstore-test-support libutil-test-support \
+    libexpr libstore libutil
+
+libexpr-test-support_LDFLAGS := -lrapidcheck
diff --git a/src/libexpr/tests/libexpr.hh b/tests/unit/libexpr-support/tests/libexpr.hh
similarity index 100%
rename from src/libexpr/tests/libexpr.hh
rename to tests/unit/libexpr-support/tests/libexpr.hh
diff --git a/tests/unit/libexpr-support/tests/value/context.cc b/tests/unit/libexpr-support/tests/value/context.cc
new file mode 100644
index 000000000..8658bdaef
--- /dev/null
+++ b/tests/unit/libexpr-support/tests/value/context.cc
@@ -0,0 +1,30 @@
+#include <rapidcheck.h>
+
+#include "tests/path.hh"
+#include "tests/value/context.hh"
+
+namespace rc {
+using namespace nix;
+
+Gen<NixStringContextElem::DrvDeep> Arbitrary<NixStringContextElem::DrvDeep>::arbitrary()
+{
+    return gen::just(NixStringContextElem::DrvDeep {
+        .drvPath = *gen::arbitrary<StorePath>(),
+    });
+}
+
+Gen<NixStringContextElem> Arbitrary<NixStringContextElem>::arbitrary()
+{
+    switch (*gen::inRange<uint8_t>(0, std::variant_size_v<NixStringContextElem::Raw>)) {
+    case 0:
+        return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::Opaque>());
+    case 1:
+        return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::DrvDeep>());
+    case 2:
+        return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::Built>());
+    default:
+        assert(false);
+    }
+}
+
+}
diff --git a/src/libexpr/tests/value/context.hh b/tests/unit/libexpr-support/tests/value/context.hh
similarity index 95%
rename from src/libexpr/tests/value/context.hh
rename to tests/unit/libexpr-support/tests/value/context.hh
index c0bc97ba3..8c68c78bb 100644
--- a/src/libexpr/tests/value/context.hh
+++ b/tests/unit/libexpr-support/tests/value/context.hh
@@ -3,7 +3,7 @@
 
 #include <rapidcheck/gen/Arbitrary.h>
 
-#include <value/context.hh>
+#include "value/context.hh"
 
 namespace rc {
 using namespace nix;
diff --git a/src/libexpr/tests/derived-path.cc b/tests/unit/libexpr/derived-path.cc
similarity index 100%
rename from src/libexpr/tests/derived-path.cc
rename to tests/unit/libexpr/derived-path.cc
diff --git a/src/libexpr/tests/error_traces.cc b/tests/unit/libexpr/error_traces.cc
similarity index 100%
rename from src/libexpr/tests/error_traces.cc
rename to tests/unit/libexpr/error_traces.cc
diff --git a/src/libexpr/tests/flakeref.cc b/tests/unit/libexpr/flakeref.cc
similarity index 100%
rename from src/libexpr/tests/flakeref.cc
rename to tests/unit/libexpr/flakeref.cc
diff --git a/src/libexpr/tests/json.cc b/tests/unit/libexpr/json.cc
similarity index 100%
rename from src/libexpr/tests/json.cc
rename to tests/unit/libexpr/json.cc
diff --git a/tests/unit/libexpr/local.mk b/tests/unit/libexpr/local.mk
new file mode 100644
index 000000000..5743880d7
--- /dev/null
+++ b/tests/unit/libexpr/local.mk
@@ -0,0 +1,36 @@
+check: libexpr-tests_RUN
+
+programs += libexpr-tests
+
+libexpr-tests_NAME := libnixexpr-tests
+
+libexpr-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data
+
+libexpr-tests_DIR := $(d)
+
+ifeq ($(INSTALL_UNIT_TESTS), yes)
+  libexpr-tests_INSTALL_DIR := $(checkbindir)
+else
+  libexpr-tests_INSTALL_DIR :=
+endif
+
+libexpr-tests_SOURCES := \
+    $(wildcard $(d)/*.cc) \
+    $(wildcard $(d)/value/*.cc)
+
+libexpr-tests_EXTRA_INCLUDES = \
+    -I tests/unit/libexpr-support \
+    -I tests/unit/libstore-support \
+    -I tests/unit/libutil-support \
+    -I src/libexpr \
+    -I src/libfetchers \
+    -I src/libstore \
+    -I src/libutil
+
+libexpr-tests_CXXFLAGS += $(libexpr-tests_EXTRA_INCLUDES)
+
+libexpr-tests_LIBS = \
+    libexpr-test-support libstore-test-support libutils-test-support \
+    libexpr libfetchers libstore libutil
+
+libexpr-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock
diff --git a/src/libexpr/tests/primops.cc b/tests/unit/libexpr/primops.cc
similarity index 100%
rename from src/libexpr/tests/primops.cc
rename to tests/unit/libexpr/primops.cc
diff --git a/src/libexpr/tests/search-path.cc b/tests/unit/libexpr/search-path.cc
similarity index 100%
rename from src/libexpr/tests/search-path.cc
rename to tests/unit/libexpr/search-path.cc
diff --git a/src/libexpr/tests/trivial.cc b/tests/unit/libexpr/trivial.cc
similarity index 100%
rename from src/libexpr/tests/trivial.cc
rename to tests/unit/libexpr/trivial.cc
diff --git a/src/libexpr/tests/value/context.cc b/tests/unit/libexpr/value/context.cc
similarity index 83%
rename from src/libexpr/tests/value/context.cc
rename to tests/unit/libexpr/value/context.cc
index 92d4889ab..761286dbd 100644
--- a/src/libexpr/tests/value/context.cc
+++ b/tests/unit/libexpr/value/context.cc
@@ -117,36 +117,6 @@ TEST(NixStringContextElemTest, built_built_xp) {
         NixStringContextElem::parse("!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"),        MissingExperimentalFeature);
 }
 
-}
-
-namespace rc {
-using namespace nix;
-
-Gen<NixStringContextElem::DrvDeep> Arbitrary<NixStringContextElem::DrvDeep>::arbitrary()
-{
-    return gen::just(NixStringContextElem::DrvDeep {
-        .drvPath = *gen::arbitrary<StorePath>(),
-    });
-}
-
-Gen<NixStringContextElem> Arbitrary<NixStringContextElem>::arbitrary()
-{
-    switch (*gen::inRange<uint8_t>(0, std::variant_size_v<NixStringContextElem::Raw>)) {
-    case 0:
-        return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::Opaque>());
-    case 1:
-        return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::DrvDeep>());
-    case 2:
-        return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::Built>());
-    default:
-        assert(false);
-    }
-}
-
-}
-
-namespace nix {
-
 #ifndef COVERAGE
 
 RC_GTEST_PROP(
diff --git a/src/libexpr/tests/value/print.cc b/tests/unit/libexpr/value/print.cc
similarity index 100%
rename from src/libexpr/tests/value/print.cc
rename to tests/unit/libexpr/value/print.cc
diff --git a/tests/unit/libstore-support/local.mk b/tests/unit/libstore-support/local.mk
new file mode 100644
index 000000000..d5d657c91
--- /dev/null
+++ b/tests/unit/libstore-support/local.mk
@@ -0,0 +1,21 @@
+libraries += libstore-test-support
+
+libstore-test-support_NAME = libnixstore-test-support
+
+libstore-test-support_DIR := $(d)
+
+ifeq ($(INSTALL_UNIT_TESTS), yes)
+  libstore-test-support_INSTALL_DIR := $(checklibdir)
+else
+  libstore-test-support_INSTALL_DIR :=
+endif
+
+libstore-test-support_SOURCES := $(wildcard $(d)/tests/*.cc)
+
+libstore-test-support_CXXFLAGS += $(libstore-tests_EXTRA_INCLUDES)
+
+libstore-test-support_LIBS = \
+    libutil-test-support \
+    libstore libutil
+
+libstore-test-support_LDFLAGS := -lrapidcheck
diff --git a/tests/unit/libstore-support/tests/derived-path.cc b/tests/unit/libstore-support/tests/derived-path.cc
new file mode 100644
index 000000000..091706dba
--- /dev/null
+++ b/tests/unit/libstore-support/tests/derived-path.cc
@@ -0,0 +1,57 @@
+#include <regex>
+
+#include <rapidcheck.h>
+
+#include "tests/derived-path.hh"
+
+namespace rc {
+using namespace nix;
+
+Gen<DerivedPath::Opaque> Arbitrary<DerivedPath::Opaque>::arbitrary()
+{
+    return gen::just(DerivedPath::Opaque {
+        .path = *gen::arbitrary<StorePath>(),
+    });
+}
+
+Gen<SingleDerivedPath::Built> Arbitrary<SingleDerivedPath::Built>::arbitrary()
+{
+    return gen::just(SingleDerivedPath::Built {
+        .drvPath = make_ref<SingleDerivedPath>(*gen::arbitrary<SingleDerivedPath>()),
+        .output = (*gen::arbitrary<StorePathName>()).name,
+    });
+}
+
+Gen<DerivedPath::Built> Arbitrary<DerivedPath::Built>::arbitrary()
+{
+    return gen::just(DerivedPath::Built {
+        .drvPath = make_ref<SingleDerivedPath>(*gen::arbitrary<SingleDerivedPath>()),
+        .outputs = *gen::arbitrary<OutputsSpec>(),
+    });
+}
+
+Gen<SingleDerivedPath> Arbitrary<SingleDerivedPath>::arbitrary()
+{
+    switch (*gen::inRange<uint8_t>(0, std::variant_size_v<SingleDerivedPath::Raw>)) {
+    case 0:
+        return gen::just<SingleDerivedPath>(*gen::arbitrary<SingleDerivedPath::Opaque>());
+    case 1:
+        return gen::just<SingleDerivedPath>(*gen::arbitrary<SingleDerivedPath::Built>());
+    default:
+        assert(false);
+    }
+}
+
+Gen<DerivedPath> Arbitrary<DerivedPath>::arbitrary()
+{
+    switch (*gen::inRange<uint8_t>(0, std::variant_size_v<DerivedPath::Raw>)) {
+    case 0:
+        return gen::just<DerivedPath>(*gen::arbitrary<DerivedPath::Opaque>());
+    case 1:
+        return gen::just<DerivedPath>(*gen::arbitrary<DerivedPath::Built>());
+    default:
+        assert(false);
+    }
+}
+
+}
diff --git a/src/libstore/tests/derived-path.hh b/tests/unit/libstore-support/tests/derived-path.hh
similarity index 100%
rename from src/libstore/tests/derived-path.hh
rename to tests/unit/libstore-support/tests/derived-path.hh
diff --git a/src/libstore/tests/libstore.hh b/tests/unit/libstore-support/tests/libstore.hh
similarity index 100%
rename from src/libstore/tests/libstore.hh
rename to tests/unit/libstore-support/tests/libstore.hh
diff --git a/tests/unit/libstore-support/tests/outputs-spec.cc b/tests/unit/libstore-support/tests/outputs-spec.cc
new file mode 100644
index 000000000..e9d602203
--- /dev/null
+++ b/tests/unit/libstore-support/tests/outputs-spec.cc
@@ -0,0 +1,24 @@
+#include "tests/outputs-spec.hh"
+
+#include <rapidcheck.h>
+
+namespace rc {
+using namespace nix;
+
+Gen<OutputsSpec> Arbitrary<OutputsSpec>::arbitrary()
+{
+    switch (*gen::inRange<uint8_t>(0, std::variant_size_v<OutputsSpec::Raw>)) {
+    case 0:
+        return gen::just((OutputsSpec) OutputsSpec::All { });
+    case 1:
+        return gen::just((OutputsSpec) OutputsSpec::Names {
+            *gen::nonEmpty(gen::container<StringSet>(gen::map(
+                gen::arbitrary<StorePathName>(),
+                [](StorePathName n) { return n.name; }))),
+        });
+    default:
+        assert(false);
+    }
+}
+
+}
diff --git a/src/libstore/tests/outputs-spec.hh b/tests/unit/libstore-support/tests/outputs-spec.hh
similarity index 89%
rename from src/libstore/tests/outputs-spec.hh
rename to tests/unit/libstore-support/tests/outputs-spec.hh
index ded331b33..f5bf9042d 100644
--- a/src/libstore/tests/outputs-spec.hh
+++ b/tests/unit/libstore-support/tests/outputs-spec.hh
@@ -5,7 +5,7 @@
 
 #include <outputs-spec.hh>
 
-#include <tests/path.hh>
+#include "tests/path.hh"
 
 namespace rc {
 using namespace nix;
diff --git a/tests/unit/libstore-support/tests/path.cc b/tests/unit/libstore-support/tests/path.cc
new file mode 100644
index 000000000..e5f169e94
--- /dev/null
+++ b/tests/unit/libstore-support/tests/path.cc
@@ -0,0 +1,82 @@
+#include <regex>
+
+#include <rapidcheck.h>
+
+#include "path-regex.hh"
+#include "store-api.hh"
+
+#include "tests/hash.hh"
+#include "tests/path.hh"
+
+namespace nix {
+
+void showValue(const StorePath & p, std::ostream & os)
+{
+    os << p.to_string();
+}
+
+}
+
+namespace rc {
+using namespace nix;
+
+Gen<StorePathName> Arbitrary<StorePathName>::arbitrary()
+{
+    auto len = *gen::inRange<size_t>(
+        1,
+        StorePath::MaxPathLen - StorePath::HashLen);
+
+    std::string pre;
+    pre.reserve(len);
+
+    for (size_t c = 0; c < len; ++c) {
+        switch (auto i = *gen::inRange<uint8_t>(0, 10 + 2 * 26 + 6)) {
+            case 0 ... 9:
+                pre += '0' + i;
+            case 10 ... 35:
+                pre += 'A' + (i - 10);
+                break;
+            case 36 ... 61:
+                pre += 'a' + (i - 36);
+                break;
+            case 62:
+                pre += '+';
+                break;
+            case 63:
+                pre += '-';
+                break;
+            case 64:
+                // names aren't permitted to start with a period,
+                // so just fall through to the next case here
+                if (c != 0) {
+                    pre += '.';
+                    break;
+                }
+            case 65:
+                pre += '_';
+                break;
+            case 66:
+                pre += '?';
+                break;
+            case 67:
+                pre += '=';
+                break;
+            default:
+                assert(false);
+        }
+    }
+
+    return gen::just(StorePathName {
+        .name = std::move(pre),
+    });
+}
+
+Gen<StorePath> Arbitrary<StorePath>::arbitrary()
+{
+    return gen::just(StorePath {
+        *gen::arbitrary<Hash>(),
+        (*gen::arbitrary<StorePathName>()).name,
+    });
+}
+
+} // namespace rc
diff --git a/src/libstore/tests/path.hh b/tests/unit/libstore-support/tests/path.hh
similarity index 82%
rename from src/libstore/tests/path.hh
rename to tests/unit/libstore-support/tests/path.hh
index 21cb62310..4751b3373 100644
--- a/src/libstore/tests/path.hh
+++ b/tests/unit/libstore-support/tests/path.hh
@@ -11,6 +11,9 @@ struct StorePathName {
     std::string name;
 };
 
+// For rapidcheck
+void showValue(const StorePath & p, std::ostream & os);
+
 }
 
 namespace rc {
diff --git a/src/libstore/tests/protocol.hh b/tests/unit/libstore-support/tests/protocol.hh
similarity index 96%
rename from src/libstore/tests/protocol.hh
rename to tests/unit/libstore-support/tests/protocol.hh
index 466032a79..3c9e52c11 100644
--- a/src/libstore/tests/protocol.hh
+++ b/tests/unit/libstore-support/tests/protocol.hh
@@ -12,7 +12,7 @@ namespace nix {
 template<class Proto, const char * protocolDir>
 class ProtoTest : public CharacterizationTest, public LibStoreTest
 {
-    Path unitTestData = getUnitTestData() + "/libstore/" + protocolDir;
+    Path unitTestData = getUnitTestData() + "/" + protocolDir;
 
     Path goldenMaster(std::string_view testStem) const override {
         return unitTestData + "/" + testStem + ".bin";
diff --git a/src/libstore/tests/common-protocol.cc b/tests/unit/libstore/common-protocol.cc
similarity index 100%
rename from src/libstore/tests/common-protocol.cc
rename to tests/unit/libstore/common-protocol.cc
diff --git a/unit-test-data/libstore/common-protocol/content-address.bin b/tests/unit/libstore/data/common-protocol/content-address.bin
similarity index 100%
rename from unit-test-data/libstore/common-protocol/content-address.bin
rename to tests/unit/libstore/data/common-protocol/content-address.bin
diff --git a/unit-test-data/libstore/common-protocol/drv-output.bin b/tests/unit/libstore/data/common-protocol/drv-output.bin
similarity index 100%
rename from unit-test-data/libstore/common-protocol/drv-output.bin
rename to tests/unit/libstore/data/common-protocol/drv-output.bin
diff --git a/unit-test-data/libstore/common-protocol/optional-content-address.bin b/tests/unit/libstore/data/common-protocol/optional-content-address.bin
similarity index 100%
rename from unit-test-data/libstore/common-protocol/optional-content-address.bin
rename to tests/unit/libstore/data/common-protocol/optional-content-address.bin
diff --git a/unit-test-data/libstore/common-protocol/optional-store-path.bin b/tests/unit/libstore/data/common-protocol/optional-store-path.bin
similarity index 100%
rename from unit-test-data/libstore/common-protocol/optional-store-path.bin
rename to tests/unit/libstore/data/common-protocol/optional-store-path.bin
diff --git a/unit-test-data/libstore/common-protocol/realisation.bin b/tests/unit/libstore/data/common-protocol/realisation.bin
similarity index 100%
rename from unit-test-data/libstore/common-protocol/realisation.bin
rename to tests/unit/libstore/data/common-protocol/realisation.bin
diff --git a/unit-test-data/libstore/common-protocol/set.bin b/tests/unit/libstore/data/common-protocol/set.bin
similarity index 100%
rename from unit-test-data/libstore/common-protocol/set.bin
rename to tests/unit/libstore/data/common-protocol/set.bin
diff --git a/unit-test-data/libstore/common-protocol/store-path.bin b/tests/unit/libstore/data/common-protocol/store-path.bin
similarity index 100%
rename from unit-test-data/libstore/common-protocol/store-path.bin
rename to tests/unit/libstore/data/common-protocol/store-path.bin
diff --git a/unit-test-data/libstore/common-protocol/string.bin b/tests/unit/libstore/data/common-protocol/string.bin
similarity index 100%
rename from unit-test-data/libstore/common-protocol/string.bin
rename to tests/unit/libstore/data/common-protocol/string.bin
diff --git a/unit-test-data/libstore/common-protocol/vector.bin b/tests/unit/libstore/data/common-protocol/vector.bin
similarity index 100%
rename from unit-test-data/libstore/common-protocol/vector.bin
rename to tests/unit/libstore/data/common-protocol/vector.bin
diff --git a/unit-test-data/libstore/derivation/bad-old-version-dyn-deps.drv b/tests/unit/libstore/data/derivation/bad-old-version-dyn-deps.drv
similarity index 100%
rename from unit-test-data/libstore/derivation/bad-old-version-dyn-deps.drv
rename to tests/unit/libstore/data/derivation/bad-old-version-dyn-deps.drv
diff --git a/unit-test-data/libstore/derivation/bad-version.drv b/tests/unit/libstore/data/derivation/bad-version.drv
similarity index 100%
rename from unit-test-data/libstore/derivation/bad-version.drv
rename to tests/unit/libstore/data/derivation/bad-version.drv
diff --git a/unit-test-data/libstore/derivation/dynDerivationDeps.drv b/tests/unit/libstore/data/derivation/dynDerivationDeps.drv
similarity index 100%
rename from unit-test-data/libstore/derivation/dynDerivationDeps.drv
rename to tests/unit/libstore/data/derivation/dynDerivationDeps.drv
diff --git a/unit-test-data/libstore/derivation/dynDerivationDeps.json b/tests/unit/libstore/data/derivation/dynDerivationDeps.json
similarity index 100%
rename from unit-test-data/libstore/derivation/dynDerivationDeps.json
rename to tests/unit/libstore/data/derivation/dynDerivationDeps.json
diff --git a/unit-test-data/libstore/derivation/output-caFixedFlat.json b/tests/unit/libstore/data/derivation/output-caFixedFlat.json
similarity index 100%
rename from unit-test-data/libstore/derivation/output-caFixedFlat.json
rename to tests/unit/libstore/data/derivation/output-caFixedFlat.json
diff --git a/unit-test-data/libstore/derivation/output-caFixedNAR.json b/tests/unit/libstore/data/derivation/output-caFixedNAR.json
similarity index 100%
rename from unit-test-data/libstore/derivation/output-caFixedNAR.json
rename to tests/unit/libstore/data/derivation/output-caFixedNAR.json
diff --git a/unit-test-data/libstore/derivation/output-caFixedText.json b/tests/unit/libstore/data/derivation/output-caFixedText.json
similarity index 100%
rename from unit-test-data/libstore/derivation/output-caFixedText.json
rename to tests/unit/libstore/data/derivation/output-caFixedText.json
diff --git a/unit-test-data/libstore/derivation/output-caFloating.json b/tests/unit/libstore/data/derivation/output-caFloating.json
similarity index 100%
rename from unit-test-data/libstore/derivation/output-caFloating.json
rename to tests/unit/libstore/data/derivation/output-caFloating.json
diff --git a/unit-test-data/libstore/derivation/output-deferred.json b/tests/unit/libstore/data/derivation/output-deferred.json
similarity index 100%
rename from unit-test-data/libstore/derivation/output-deferred.json
rename to tests/unit/libstore/data/derivation/output-deferred.json
diff --git a/unit-test-data/libstore/derivation/output-impure.json b/tests/unit/libstore/data/derivation/output-impure.json
similarity index 100%
rename from unit-test-data/libstore/derivation/output-impure.json
rename to tests/unit/libstore/data/derivation/output-impure.json
diff --git a/unit-test-data/libstore/derivation/output-inputAddressed.json b/tests/unit/libstore/data/derivation/output-inputAddressed.json
similarity index 100%
rename from unit-test-data/libstore/derivation/output-inputAddressed.json
rename to tests/unit/libstore/data/derivation/output-inputAddressed.json
diff --git a/unit-test-data/libstore/derivation/simple.drv b/tests/unit/libstore/data/derivation/simple.drv
similarity index 100%
rename from unit-test-data/libstore/derivation/simple.drv
rename to tests/unit/libstore/data/derivation/simple.drv
diff --git a/unit-test-data/libstore/derivation/simple.json b/tests/unit/libstore/data/derivation/simple.json
similarity index 100%
rename from unit-test-data/libstore/derivation/simple.json
rename to tests/unit/libstore/data/derivation/simple.json
diff --git a/unit-test-data/libstore/nar-info/impure.json b/tests/unit/libstore/data/nar-info/impure.json
similarity index 100%
rename from unit-test-data/libstore/nar-info/impure.json
rename to tests/unit/libstore/data/nar-info/impure.json
diff --git a/unit-test-data/libstore/nar-info/pure.json b/tests/unit/libstore/data/nar-info/pure.json
similarity index 100%
rename from unit-test-data/libstore/nar-info/pure.json
rename to tests/unit/libstore/data/nar-info/pure.json
diff --git a/unit-test-data/libstore/path-info/impure.json b/tests/unit/libstore/data/path-info/impure.json
similarity index 100%
rename from unit-test-data/libstore/path-info/impure.json
rename to tests/unit/libstore/data/path-info/impure.json
diff --git a/unit-test-data/libstore/path-info/pure.json b/tests/unit/libstore/data/path-info/pure.json
similarity index 100%
rename from unit-test-data/libstore/path-info/pure.json
rename to tests/unit/libstore/data/path-info/pure.json
diff --git a/unit-test-data/libstore/serve-protocol/build-result-2.2.bin b/tests/unit/libstore/data/serve-protocol/build-result-2.2.bin
similarity index 100%
rename from unit-test-data/libstore/serve-protocol/build-result-2.2.bin
rename to tests/unit/libstore/data/serve-protocol/build-result-2.2.bin
diff --git a/unit-test-data/libstore/serve-protocol/build-result-2.3.bin b/tests/unit/libstore/data/serve-protocol/build-result-2.3.bin
similarity index 100%
rename from unit-test-data/libstore/serve-protocol/build-result-2.3.bin
rename to tests/unit/libstore/data/serve-protocol/build-result-2.3.bin
diff --git a/unit-test-data/libstore/serve-protocol/build-result-2.6.bin b/tests/unit/libstore/data/serve-protocol/build-result-2.6.bin
similarity index 100%
rename from unit-test-data/libstore/serve-protocol/build-result-2.6.bin
rename to tests/unit/libstore/data/serve-protocol/build-result-2.6.bin
diff --git a/unit-test-data/libstore/serve-protocol/content-address.bin b/tests/unit/libstore/data/serve-protocol/content-address.bin
similarity index 100%
rename from unit-test-data/libstore/serve-protocol/content-address.bin
rename to tests/unit/libstore/data/serve-protocol/content-address.bin
diff --git a/unit-test-data/libstore/serve-protocol/drv-output.bin b/tests/unit/libstore/data/serve-protocol/drv-output.bin
similarity index 100%
rename from unit-test-data/libstore/serve-protocol/drv-output.bin
rename to tests/unit/libstore/data/serve-protocol/drv-output.bin
diff --git a/unit-test-data/libstore/serve-protocol/optional-content-address.bin b/tests/unit/libstore/data/serve-protocol/optional-content-address.bin
similarity index 100%
rename from unit-test-data/libstore/serve-protocol/optional-content-address.bin
rename to tests/unit/libstore/data/serve-protocol/optional-content-address.bin
diff --git a/unit-test-data/libstore/serve-protocol/optional-store-path.bin b/tests/unit/libstore/data/serve-protocol/optional-store-path.bin
similarity index 100%
rename from unit-test-data/libstore/serve-protocol/optional-store-path.bin
rename to tests/unit/libstore/data/serve-protocol/optional-store-path.bin
diff --git a/unit-test-data/libstore/serve-protocol/realisation.bin b/tests/unit/libstore/data/serve-protocol/realisation.bin
similarity index 100%
rename from unit-test-data/libstore/serve-protocol/realisation.bin
rename to tests/unit/libstore/data/serve-protocol/realisation.bin
diff --git a/unit-test-data/libstore/serve-protocol/set.bin b/tests/unit/libstore/data/serve-protocol/set.bin
similarity index 100%
rename from unit-test-data/libstore/serve-protocol/set.bin
rename to tests/unit/libstore/data/serve-protocol/set.bin
diff --git a/unit-test-data/libstore/serve-protocol/store-path.bin b/tests/unit/libstore/data/serve-protocol/store-path.bin
similarity index 100%
rename from unit-test-data/libstore/serve-protocol/store-path.bin
rename to tests/unit/libstore/data/serve-protocol/store-path.bin
diff --git a/unit-test-data/libstore/serve-protocol/string.bin b/tests/unit/libstore/data/serve-protocol/string.bin
similarity index 100%
rename from unit-test-data/libstore/serve-protocol/string.bin
rename to tests/unit/libstore/data/serve-protocol/string.bin
diff --git a/unit-test-data/libstore/serve-protocol/vector.bin b/tests/unit/libstore/data/serve-protocol/vector.bin
similarity index 100%
rename from unit-test-data/libstore/serve-protocol/vector.bin
rename to tests/unit/libstore/data/serve-protocol/vector.bin
diff --git a/unit-test-data/libstore/worker-protocol/build-result-1.27.bin b/tests/unit/libstore/data/worker-protocol/build-result-1.27.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/build-result-1.27.bin
rename to tests/unit/libstore/data/worker-protocol/build-result-1.27.bin
diff --git a/unit-test-data/libstore/worker-protocol/build-result-1.28.bin b/tests/unit/libstore/data/worker-protocol/build-result-1.28.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/build-result-1.28.bin
rename to tests/unit/libstore/data/worker-protocol/build-result-1.28.bin
diff --git a/unit-test-data/libstore/worker-protocol/build-result-1.29.bin b/tests/unit/libstore/data/worker-protocol/build-result-1.29.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/build-result-1.29.bin
rename to tests/unit/libstore/data/worker-protocol/build-result-1.29.bin
diff --git a/unit-test-data/libstore/worker-protocol/content-address.bin b/tests/unit/libstore/data/worker-protocol/content-address.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/content-address.bin
rename to tests/unit/libstore/data/worker-protocol/content-address.bin
diff --git a/unit-test-data/libstore/worker-protocol/derived-path-1.29.bin b/tests/unit/libstore/data/worker-protocol/derived-path-1.29.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/derived-path-1.29.bin
rename to tests/unit/libstore/data/worker-protocol/derived-path-1.29.bin
diff --git a/unit-test-data/libstore/worker-protocol/derived-path-1.30.bin b/tests/unit/libstore/data/worker-protocol/derived-path-1.30.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/derived-path-1.30.bin
rename to tests/unit/libstore/data/worker-protocol/derived-path-1.30.bin
diff --git a/unit-test-data/libstore/worker-protocol/drv-output.bin b/tests/unit/libstore/data/worker-protocol/drv-output.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/drv-output.bin
rename to tests/unit/libstore/data/worker-protocol/drv-output.bin
diff --git a/unit-test-data/libstore/worker-protocol/keyed-build-result-1.29.bin b/tests/unit/libstore/data/worker-protocol/keyed-build-result-1.29.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/keyed-build-result-1.29.bin
rename to tests/unit/libstore/data/worker-protocol/keyed-build-result-1.29.bin
diff --git a/unit-test-data/libstore/worker-protocol/optional-content-address.bin b/tests/unit/libstore/data/worker-protocol/optional-content-address.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/optional-content-address.bin
rename to tests/unit/libstore/data/worker-protocol/optional-content-address.bin
diff --git a/unit-test-data/libstore/worker-protocol/optional-store-path.bin b/tests/unit/libstore/data/worker-protocol/optional-store-path.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/optional-store-path.bin
rename to tests/unit/libstore/data/worker-protocol/optional-store-path.bin
diff --git a/unit-test-data/libstore/worker-protocol/optional-trusted-flag.bin b/tests/unit/libstore/data/worker-protocol/optional-trusted-flag.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/optional-trusted-flag.bin
rename to tests/unit/libstore/data/worker-protocol/optional-trusted-flag.bin
diff --git a/unit-test-data/libstore/worker-protocol/realisation.bin b/tests/unit/libstore/data/worker-protocol/realisation.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/realisation.bin
rename to tests/unit/libstore/data/worker-protocol/realisation.bin
diff --git a/unit-test-data/libstore/worker-protocol/set.bin b/tests/unit/libstore/data/worker-protocol/set.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/set.bin
rename to tests/unit/libstore/data/worker-protocol/set.bin
diff --git a/unit-test-data/libstore/worker-protocol/store-path.bin b/tests/unit/libstore/data/worker-protocol/store-path.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/store-path.bin
rename to tests/unit/libstore/data/worker-protocol/store-path.bin
diff --git a/unit-test-data/libstore/worker-protocol/string.bin b/tests/unit/libstore/data/worker-protocol/string.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/string.bin
rename to tests/unit/libstore/data/worker-protocol/string.bin
diff --git a/unit-test-data/libstore/worker-protocol/unkeyed-valid-path-info-1.15.bin b/tests/unit/libstore/data/worker-protocol/unkeyed-valid-path-info-1.15.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/unkeyed-valid-path-info-1.15.bin
rename to tests/unit/libstore/data/worker-protocol/unkeyed-valid-path-info-1.15.bin
diff --git a/unit-test-data/libstore/worker-protocol/valid-path-info-1.15.bin b/tests/unit/libstore/data/worker-protocol/valid-path-info-1.15.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/valid-path-info-1.15.bin
rename to tests/unit/libstore/data/worker-protocol/valid-path-info-1.15.bin
diff --git a/unit-test-data/libstore/worker-protocol/valid-path-info-1.16.bin b/tests/unit/libstore/data/worker-protocol/valid-path-info-1.16.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/valid-path-info-1.16.bin
rename to tests/unit/libstore/data/worker-protocol/valid-path-info-1.16.bin
diff --git a/unit-test-data/libstore/worker-protocol/vector.bin b/tests/unit/libstore/data/worker-protocol/vector.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/vector.bin
rename to tests/unit/libstore/data/worker-protocol/vector.bin
diff --git a/src/libstore/tests/derivation.cc b/tests/unit/libstore/derivation.cc
similarity index 99%
rename from src/libstore/tests/derivation.cc
rename to tests/unit/libstore/derivation.cc
index 7becfa5ab..a7f4488fa 100644
--- a/src/libstore/tests/derivation.cc
+++ b/tests/unit/libstore/derivation.cc
@@ -13,7 +13,7 @@ using nlohmann::json;
 
 class DerivationTest : public CharacterizationTest, public LibStoreTest
 {
-    Path unitTestData = getUnitTestData() + "/libstore/derivation";
+    Path unitTestData = getUnitTestData() + "/derivation";
 
 public:
     Path goldenMaster(std::string_view testStem) const override {
diff --git a/src/libstore/tests/derived-path.cc b/tests/unit/libstore/derived-path.cc
similarity index 66%
rename from src/libstore/tests/derived-path.cc
rename to tests/unit/libstore/derived-path.cc
index 3fa3c0801..c62d79a78 100644
--- a/src/libstore/tests/derived-path.cc
+++ b/tests/unit/libstore/derived-path.cc
@@ -1,64 +1,11 @@
 #include <regex>
 
-#include <nlohmann/json.hpp>
 #include <gtest/gtest.h>
 #include <rapidcheck/gtest.h>
 
 #include "tests/derived-path.hh"
 #include "tests/libstore.hh"
 
-namespace rc {
-using namespace nix;
-
-Gen<DerivedPath::Opaque> Arbitrary<DerivedPath::Opaque>::arbitrary()
-{
-    return gen::just(DerivedPath::Opaque {
-        .path = *gen::arbitrary<StorePath>(),
-    });
-}
-
-Gen<SingleDerivedPath::Built> Arbitrary<SingleDerivedPath::Built>::arbitrary()
-{
-    return gen::just(SingleDerivedPath::Built {
-        .drvPath = make_ref<SingleDerivedPath>(*gen::arbitrary<SingleDerivedPath>()),
-        .output = (*gen::arbitrary<StorePathName>()).name,
-    });
-}
-
-Gen<DerivedPath::Built> Arbitrary<DerivedPath::Built>::arbitrary()
-{
-    return gen::just(DerivedPath::Built {
-        .drvPath = make_ref<SingleDerivedPath>(*gen::arbitrary<SingleDerivedPath>()),
-        .outputs = *gen::arbitrary<OutputsSpec>(),
-    });
-}
-
-Gen<SingleDerivedPath> Arbitrary<SingleDerivedPath>::arbitrary()
-{
-    switch (*gen::inRange<uint8_t>(0, std::variant_size_v<SingleDerivedPath::Raw>)) {
-    case 0:
-        return gen::just<SingleDerivedPath>(*gen::arbitrary<SingleDerivedPath::Opaque>());
-    case 1:
-        return gen::just<SingleDerivedPath>(*gen::arbitrary<SingleDerivedPath::Built>());
-    default:
-        assert(false);
-    }
-}
-
-Gen<DerivedPath> Arbitrary<DerivedPath>::arbitrary()
-{
-    switch (*gen::inRange<uint8_t>(0, std::variant_size_v<DerivedPath::Raw>)) {
-    case 0:
-        return gen::just<DerivedPath>(*gen::arbitrary<DerivedPath::Opaque>());
-    case 1:
-        return gen::just<DerivedPath>(*gen::arbitrary<DerivedPath::Built>());
-    default:
-        assert(false);
-    }
-}
-
-}
-
 namespace nix {
 
 class DerivedPathTest : public LibStoreTest
diff --git a/src/libstore/tests/downstream-placeholder.cc b/tests/unit/libstore/downstream-placeholder.cc
similarity index 100%
rename from src/libstore/tests/downstream-placeholder.cc
rename to tests/unit/libstore/downstream-placeholder.cc
diff --git a/tests/unit/libstore/local.mk b/tests/unit/libstore/local.mk
new file mode 100644
index 000000000..63f6d011f
--- /dev/null
+++ b/tests/unit/libstore/local.mk
@@ -0,0 +1,31 @@
+check: libstore-tests_RUN
+
+programs += libstore-tests
+
+libstore-tests_NAME = libnixstore-tests
+
+libstore-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data
+
+libstore-tests_DIR := $(d)
+
+ifeq ($(INSTALL_UNIT_TESTS), yes)
+  libstore-tests_INSTALL_DIR := $(checkbindir)
+else
+  libstore-tests_INSTALL_DIR :=
+endif
+
+libstore-tests_SOURCES := $(wildcard $(d)/*.cc)
+
+libstore-tests_EXTRA_INCLUDES = \
+    -I tests/unit/libstore-support \
+    -I tests/unit/libutil-support \
+    -I src/libstore \
+    -I src/libutil
+
+libstore-tests_CXXFLAGS += $(libstore-tests_EXTRA_INCLUDES)
+
+libstore-tests_LIBS = \
+    libstore-test-support libutil-test-support \
+    libstore libutil
+
+libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS)
diff --git a/src/libstore/tests/machines.cc b/tests/unit/libstore/machines.cc
similarity index 97%
rename from src/libstore/tests/machines.cc
rename to tests/unit/libstore/machines.cc
index fede328ea..5b66e5a5b 100644
--- a/src/libstore/tests/machines.cc
+++ b/tests/unit/libstore/machines.cc
@@ -139,7 +139,7 @@ TEST(machines, getMachinesWithIncorrectFormat) {
 }
 
 TEST(machines, getMachinesWithCorrectFileReference) {
-    auto path = absPath("src/libstore/tests/test-data/machines.valid");
+    auto path = absPath("tests/unit/libstore/test-data/machines.valid");
     ASSERT_TRUE(pathExists(path));
 
     settings.builders = std::string("@") + path;
@@ -166,6 +166,6 @@ TEST(machines, getMachinesWithIncorrectFileReference) {
 }
 
 TEST(machines, getMachinesWithCorrectFileReferenceToIncorrectFile) {
-    settings.builders = std::string("@") + absPath("src/libstore/tests/test-data/machines.bad_format");
+    settings.builders = std::string("@") + absPath("tests/unit/libstore/test-data/machines.bad_format");
     EXPECT_THROW(getMachines(), FormatError);
 }
diff --git a/src/libstore/tests/nar-info-disk-cache.cc b/tests/unit/libstore/nar-info-disk-cache.cc
similarity index 100%
rename from src/libstore/tests/nar-info-disk-cache.cc
rename to tests/unit/libstore/nar-info-disk-cache.cc
diff --git a/src/libstore/tests/nar-info.cc b/tests/unit/libstore/nar-info.cc
similarity index 98%
rename from src/libstore/tests/nar-info.cc
rename to tests/unit/libstore/nar-info.cc
index 88e6e1add..4f124e89e 100644
--- a/src/libstore/tests/nar-info.cc
+++ b/tests/unit/libstore/nar-info.cc
@@ -13,7 +13,7 @@ using nlohmann::json;
 
 class NarInfoTest : public CharacterizationTest, public LibStoreTest
 {
-    Path unitTestData = getUnitTestData() + "/libstore/nar-info";
+    Path unitTestData = getUnitTestData() + "/nar-info";
 
     Path goldenMaster(PathView testStem) const override {
         return unitTestData + "/" + testStem + ".json";
diff --git a/src/libstore/tests/outputs-spec.cc b/tests/unit/libstore/outputs-spec.cc
similarity index 92%
rename from src/libstore/tests/outputs-spec.cc
rename to tests/unit/libstore/outputs-spec.cc
index 952945185..456196be1 100644
--- a/src/libstore/tests/outputs-spec.cc
+++ b/tests/unit/libstore/outputs-spec.cc
@@ -1,4 +1,4 @@
-#include "outputs-spec.hh"
+#include "tests/outputs-spec.hh"
 
 #include <nlohmann/json.hpp>
 #include <gtest/gtest.h>
@@ -199,31 +199,6 @@ TEST_JSON(ExtendedOutputsSpec, names, R"(["a","b"])", (ExtendedOutputsSpec::Expl
 
 #undef TEST_JSON
 
-}
-
-namespace rc {
-using namespace nix;
-
-Gen<OutputsSpec> Arbitrary<OutputsSpec>::arbitrary()
-{
-    switch (*gen::inRange<uint8_t>(0, std::variant_size_v<OutputsSpec::Raw>)) {
-    case 0:
-        return gen::just((OutputsSpec) OutputsSpec::All { });
-    case 1:
-        return gen::just((OutputsSpec) OutputsSpec::Names {
-            *gen::nonEmpty(gen::container<StringSet>(gen::map(
-                gen::arbitrary<StorePathName>(),
-                [](StorePathName n) { return n.name; }))),
-        });
-    default:
-        assert(false);
-    }
-}
-
-}
-
-namespace nix {
-
 #ifndef COVERAGE
 
 RC_GTEST_PROP(
diff --git a/src/libstore/tests/path-info.cc b/tests/unit/libstore/path-info.cc
similarity index 97%
rename from src/libstore/tests/path-info.cc
rename to tests/unit/libstore/path-info.cc
index 49bf623bd..18f00ca19 100644
--- a/src/libstore/tests/path-info.cc
+++ b/tests/unit/libstore/path-info.cc
@@ -12,7 +12,7 @@ using nlohmann::json;
 
 class PathInfoTest : public CharacterizationTest, public LibStoreTest
 {
-    Path unitTestData = getUnitTestData() + "/libstore/path-info";
+    Path unitTestData = getUnitTestData() + "/path-info";
 
     Path goldenMaster(PathView testStem) const override {
         return unitTestData + "/" + testStem + ".json";
diff --git a/src/libstore/tests/path.cc b/tests/unit/libstore/path.cc
similarity index 59%
rename from src/libstore/tests/path.cc
rename to tests/unit/libstore/path.cc
index 5a84d646c..30631b5fd 100644
--- a/src/libstore/tests/path.cc
+++ b/tests/unit/libstore/path.cc
@@ -66,79 +66,6 @@ TEST_DO_PARSE(equals_sign, "foo=foo")
 
 #undef TEST_DO_PARSE
 
-// For rapidcheck
-void showValue(const StorePath & p, std::ostream & os) {
-    os << p.to_string();
-}
-
-}
-
-namespace rc {
-using namespace nix;
-
-Gen<StorePathName> Arbitrary<StorePathName>::arbitrary()
-{
-    auto len = *gen::inRange<size_t>(
-        1,
-        StorePath::MaxPathLen - std::string_view { HASH_PART }.size());
-
-    std::string pre;
-    pre.reserve(len);
-
-    for (size_t c = 0; c < len; ++c) {
-        switch (auto i = *gen::inRange<uint8_t>(0, 10 + 2 * 26 + 6)) {
-            case 0 ... 9:
-                pre += '0' + i;
-            case 10 ... 35:
-                pre += 'A' + (i - 10);
-                break;
-            case 36 ... 61:
-                pre += 'a' + (i - 36);
-                break;
-            case 62:
-                pre += '+';
-                break;
-            case 63:
-                pre += '-';
-                break;
-            case 64:
-                // names aren't permitted to start with a period,
-                // so just fall through to the next case here
-                if (c != 0) {
-                    pre += '.';
-                    break;
-                }
-            case 65:
-                pre += '_';
-                break;
-            case 66:
-                pre += '?';
-                break;
-            case 67:
-                pre += '=';
-                break;
-            default:
-                assert(false);
-        }
-    }
-
-    return gen::just(StorePathName {
-        .name = std::move(pre),
-    });
-}
-
-Gen<StorePath> Arbitrary<StorePath>::arbitrary()
-{
-    return gen::just(StorePath {
-        *gen::arbitrary<Hash>(),
-        (*gen::arbitrary<StorePathName>()).name,
-    });
-}
-
-} // namespace rc
-
-namespace nix {
-
 #ifndef COVERAGE
 
 RC_GTEST_FIXTURE_PROP(
diff --git a/src/libstore/tests/references.cc b/tests/unit/libstore/references.cc
similarity index 100%
rename from src/libstore/tests/references.cc
rename to tests/unit/libstore/references.cc
diff --git a/src/libstore/tests/serve-protocol.cc b/tests/unit/libstore/serve-protocol.cc
similarity index 100%
rename from src/libstore/tests/serve-protocol.cc
rename to tests/unit/libstore/serve-protocol.cc
diff --git a/src/libstore/tests/test-data/machines.bad_format b/tests/unit/libstore/test-data/machines.bad_format
similarity index 100%
rename from src/libstore/tests/test-data/machines.bad_format
rename to tests/unit/libstore/test-data/machines.bad_format
diff --git a/src/libstore/tests/test-data/machines.valid b/tests/unit/libstore/test-data/machines.valid
similarity index 100%
rename from src/libstore/tests/test-data/machines.valid
rename to tests/unit/libstore/test-data/machines.valid
diff --git a/src/libstore/tests/worker-protocol.cc b/tests/unit/libstore/worker-protocol.cc
similarity index 100%
rename from src/libstore/tests/worker-protocol.cc
rename to tests/unit/libstore/worker-protocol.cc
diff --git a/tests/unit/libutil-support/local.mk b/tests/unit/libutil-support/local.mk
new file mode 100644
index 000000000..43a1551e5
--- /dev/null
+++ b/tests/unit/libutil-support/local.mk
@@ -0,0 +1,19 @@
+libraries += libutil-test-support
+
+libutil-test-support_NAME = libnixutil-test-support
+
+libutil-test-support_DIR := $(d)
+
+ifeq ($(INSTALL_UNIT_TESTS), yes)
+  libutil-test-support_INSTALL_DIR := $(checklibdir)
+else
+  libutil-test-support_INSTALL_DIR :=
+endif
+
+libutil-test-support_SOURCES := $(wildcard $(d)/tests/*.cc)
+
+libutil-test-support_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES)
+
+libutil-test-support_LIBS = libutil
+
+libutil-test-support_LDFLAGS := -lrapidcheck
diff --git a/src/libutil/tests/characterization.hh b/tests/unit/libutil-support/tests/characterization.hh
similarity index 95%
rename from src/libutil/tests/characterization.hh
rename to tests/unit/libutil-support/tests/characterization.hh
index 6eb513d68..9d6c850f0 100644
--- a/src/libutil/tests/characterization.hh
+++ b/tests/unit/libutil-support/tests/characterization.hh
@@ -9,8 +9,8 @@
 namespace nix {
 
 /**
- * The path to the `unit-test-data` directory. See the contributing
- * guide in the manual for further details.
+ * The path to the unit test data directory. See the contributing guide
+ * in the manual for further details.
  */
 static Path getUnitTestData() {
     return getEnv("_NIX_TEST_UNIT_DATA").value();
diff --git a/tests/unit/libutil-support/tests/hash.cc b/tests/unit/libutil-support/tests/hash.cc
new file mode 100644
index 000000000..577e9890e
--- /dev/null
+++ b/tests/unit/libutil-support/tests/hash.cc
@@ -0,0 +1,20 @@
+#include <regex>
+
+#include <rapidcheck.h>
+
+#include "hash.hh"
+
+#include "tests/hash.hh"
+
+namespace rc {
+using namespace nix;
+
+Gen<Hash> Arbitrary<Hash>::arbitrary()
+{
+    Hash hash(htSHA1);
+    for (size_t i = 0; i < hash.hashSize; ++i)
+        hash.hash[i] = *gen::arbitrary<uint8_t>();
+    return gen::just(hash);
+}
+
+}
diff --git a/src/libutil/tests/hash.hh b/tests/unit/libutil-support/tests/hash.hh
similarity index 100%
rename from src/libutil/tests/hash.hh
rename to tests/unit/libutil-support/tests/hash.hh
diff --git a/src/libutil/tests/args.cc b/tests/unit/libutil/args.cc
similarity index 98%
rename from src/libutil/tests/args.cc
rename to tests/unit/libutil/args.cc
index bea74a8c8..950224430 100644
--- a/src/libutil/tests/args.cc
+++ b/tests/unit/libutil/args.cc
@@ -1,5 +1,5 @@
-#include "../args.hh"
-#include "libutil/fs-sink.hh"
+#include "args.hh"
+#include "fs-sink.hh"
 #include <list>
 
 #include <gtest/gtest.h>
@@ -165,4 +165,4 @@ RC_GTEST_PROP(
 
 #endif
 
-}
\ No newline at end of file
+}
diff --git a/src/libutil/tests/canon-path.cc b/tests/unit/libutil/canon-path.cc
similarity index 100%
rename from src/libutil/tests/canon-path.cc
rename to tests/unit/libutil/canon-path.cc
diff --git a/src/libutil/tests/chunked-vector.cc b/tests/unit/libutil/chunked-vector.cc
similarity index 100%
rename from src/libutil/tests/chunked-vector.cc
rename to tests/unit/libutil/chunked-vector.cc
diff --git a/src/libutil/tests/closure.cc b/tests/unit/libutil/closure.cc
similarity index 100%
rename from src/libutil/tests/closure.cc
rename to tests/unit/libutil/closure.cc
diff --git a/src/libutil/tests/compression.cc b/tests/unit/libutil/compression.cc
similarity index 100%
rename from src/libutil/tests/compression.cc
rename to tests/unit/libutil/compression.cc
diff --git a/src/libutil/tests/config.cc b/tests/unit/libutil/config.cc
similarity index 100%
rename from src/libutil/tests/config.cc
rename to tests/unit/libutil/config.cc
diff --git a/unit-test-data/libutil/git/check-data.sh b/tests/unit/libutil/data/git/check-data.sh
similarity index 98%
rename from unit-test-data/libutil/git/check-data.sh
rename to tests/unit/libutil/data/git/check-data.sh
index 68b705c95..b3f59c4f1 100644
--- a/unit-test-data/libutil/git/check-data.sh
+++ b/tests/unit/libutil/data/git/check-data.sh
@@ -2,7 +2,7 @@
 
 set -eu -o pipefail
 
-export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/git-hashing/unit-test-data
+export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/git-hashing/check-data
 mkdir -p $TEST_ROOT
 
 repo="$TEST_ROOT/scratch"
diff --git a/unit-test-data/libutil/git/hello-world-blob.bin b/tests/unit/libutil/data/git/hello-world-blob.bin
similarity index 100%
rename from unit-test-data/libutil/git/hello-world-blob.bin
rename to tests/unit/libutil/data/git/hello-world-blob.bin
diff --git a/unit-test-data/libutil/git/hello-world.bin b/tests/unit/libutil/data/git/hello-world.bin
similarity index 100%
rename from unit-test-data/libutil/git/hello-world.bin
rename to tests/unit/libutil/data/git/hello-world.bin
diff --git a/unit-test-data/libutil/git/tree.bin b/tests/unit/libutil/data/git/tree.bin
similarity index 100%
rename from unit-test-data/libutil/git/tree.bin
rename to tests/unit/libutil/data/git/tree.bin
diff --git a/unit-test-data/libutil/git/tree.txt b/tests/unit/libutil/data/git/tree.txt
similarity index 100%
rename from unit-test-data/libutil/git/tree.txt
rename to tests/unit/libutil/data/git/tree.txt
diff --git a/src/libutil/tests/git.cc b/tests/unit/libutil/git.cc
similarity index 97%
rename from src/libutil/tests/git.cc
rename to tests/unit/libutil/git.cc
index 2842ea4d0..551a2d105 100644
--- a/src/libutil/tests/git.cc
+++ b/tests/unit/libutil/git.cc
@@ -11,7 +11,7 @@ using namespace git;
 
 class GitTest : public CharacterizationTest
 {
-    Path unitTestData = getUnitTestData() + "/libutil/git";
+    Path unitTestData = getUnitTestData() + "/git";
 
 public:
 
@@ -86,8 +86,8 @@ TEST_F(GitTest, blob_write) {
 
 /**
  * This data is for "shallow" tree tests. However, we use "real" hashes
- * so that we can check our test data in the corresponding functional
- * test (`git-hashing/unit-test-data`).
+ * so that we can check our test data in a small shell script test test
+ * (`tests/unit/libutil/data/git/check-data.sh`).
  */
 const static Tree tree = {
     {
diff --git a/src/libutil/tests/hash.cc b/tests/unit/libutil/hash.cc
similarity index 92%
rename from src/libutil/tests/hash.cc
rename to tests/unit/libutil/hash.cc
index 9a5ebbb30..92291afce 100644
--- a/src/libutil/tests/hash.cc
+++ b/tests/unit/libutil/hash.cc
@@ -1,12 +1,8 @@
 #include <regex>
 
-#include <nlohmann/json.hpp>
 #include <gtest/gtest.h>
-#include <rapidcheck/gtest.h>
 
-#include <hash.hh>
-
-#include "tests/hash.hh"
+#include "hash.hh"
 
 namespace nix {
 
@@ -68,7 +64,6 @@ namespace nix {
                 "7ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd"
                 "454d4423643ce80e2a9ac94fa54ca49f");
     }
-
     TEST(hashString, testKnownSHA512Hashes2) {
         // values taken from: https://tools.ietf.org/html/rfc4634
         auto s = "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu";
@@ -95,16 +90,3 @@ namespace nix {
         ASSERT_EQ(parseHashFormatOpt("sha0042"), std::nullopt);
     }
 }
-
-namespace rc {
-using namespace nix;
-
-Gen<Hash> Arbitrary<Hash>::arbitrary()
-{
-    Hash hash(htSHA1);
-    for (size_t i = 0; i < hash.hashSize; ++i)
-        hash.hash[i] = *gen::arbitrary<uint8_t>();
-    return gen::just(hash);
-}
-
-}
diff --git a/src/libutil/tests/hilite.cc b/tests/unit/libutil/hilite.cc
similarity index 100%
rename from src/libutil/tests/hilite.cc
rename to tests/unit/libutil/hilite.cc
diff --git a/src/libutil/tests/json-utils.cc b/tests/unit/libutil/json-utils.cc
similarity index 100%
rename from src/libutil/tests/json-utils.cc
rename to tests/unit/libutil/json-utils.cc
diff --git a/tests/unit/libutil/local.mk b/tests/unit/libutil/local.mk
new file mode 100644
index 000000000..930efb90b
--- /dev/null
+++ b/tests/unit/libutil/local.mk
@@ -0,0 +1,31 @@
+check: libutil-tests_RUN
+
+programs += libutil-tests
+
+libutil-tests_NAME = libnixutil-tests
+
+libutil-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data
+
+libutil-tests_DIR := $(d)
+
+ifeq ($(INSTALL_UNIT_TESTS), yes)
+  libutil-tests_INSTALL_DIR := $(checkbindir)
+else
+  libutil-tests_INSTALL_DIR :=
+endif
+
+libutil-tests_SOURCES := $(wildcard $(d)/*.cc)
+
+libutil-tests_EXTRA_INCLUDES = \
+    -I tests/unit/libutil-support \
+    -I src/libutil
+
+libutil-tests_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES)
+
+libutil-tests_LIBS = libutil-test-support libutil
+
+libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS)
+
+check: $(d)/data/git/check-data.sh.test
+
+$(eval $(call run-test,$(d)/data/git/check-data.sh))
diff --git a/src/libutil/tests/logging.cc b/tests/unit/libutil/logging.cc
similarity index 100%
rename from src/libutil/tests/logging.cc
rename to tests/unit/libutil/logging.cc
diff --git a/src/libutil/tests/lru-cache.cc b/tests/unit/libutil/lru-cache.cc
similarity index 100%
rename from src/libutil/tests/lru-cache.cc
rename to tests/unit/libutil/lru-cache.cc
diff --git a/src/libutil/tests/pool.cc b/tests/unit/libutil/pool.cc
similarity index 100%
rename from src/libutil/tests/pool.cc
rename to tests/unit/libutil/pool.cc
diff --git a/src/libutil/tests/references.cc b/tests/unit/libutil/references.cc
similarity index 100%
rename from src/libutil/tests/references.cc
rename to tests/unit/libutil/references.cc
diff --git a/src/libutil/tests/suggestions.cc b/tests/unit/libutil/suggestions.cc
similarity index 100%
rename from src/libutil/tests/suggestions.cc
rename to tests/unit/libutil/suggestions.cc
diff --git a/src/libutil/tests/tests.cc b/tests/unit/libutil/tests.cc
similarity index 100%
rename from src/libutil/tests/tests.cc
rename to tests/unit/libutil/tests.cc
diff --git a/src/libutil/tests/url.cc b/tests/unit/libutil/url.cc
similarity index 100%
rename from src/libutil/tests/url.cc
rename to tests/unit/libutil/url.cc
diff --git a/src/libutil/tests/xml-writer.cc b/tests/unit/libutil/xml-writer.cc
similarity index 100%
rename from src/libutil/tests/xml-writer.cc
rename to tests/unit/libutil/xml-writer.cc

From 7355a48b1a4ce2e393598c2a72ef520cba9d172d Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 1 Dec 2023 15:55:18 -0500
Subject: [PATCH 242/294] flake.lock: Update Nixpkgs to fix static build
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The problem was since switching to use libgit2, we had a package in our
closure (`http-parser`) that was always trying to build as a shared
object.

Underlying Nixpkgs PR (a 23.05 backport)
https://github.com/NixOS/nixpkgs/pull/271202

Flake lock file updates:

• Updated input 'nixpkgs':
    'github:NixOS/nixpkgs/9ba29e2346bc542e9909d1021e8fd7d4b3f64db0' (2023-11-13)
  → 'github:NixOS/nixpkgs/36c4ac09e9bebcec1fa7b7539cddb0c9e837409c' (2023-11-30)
---
 flake.lock | 8 ++++----
 flake.nix  | 8 +++++++-
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/flake.lock b/flake.lock
index f120d3b5f..3cb9e72c9 100644
--- a/flake.lock
+++ b/flake.lock
@@ -50,16 +50,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1700748986,
-        "narHash": "sha256-/nqLrNU297h3PCw4QyDpZKZEUHmialJdZW2ceYFobds=",
+        "lastModified": 1701355166,
+        "narHash": "sha256-4V7XMI0Gd+y0zsi++cEHd99u3GNL0xSTGRmiWKzGnUQ=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "9ba29e2346bc542e9909d1021e8fd7d4b3f64db0",
+        "rev": "36c4ac09e9bebcec1fa7b7539cddb0c9e837409c",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-23.05-small",
+        "ref": "staging-23.05",
         "repo": "nixpkgs",
         "type": "github"
       }
diff --git a/flake.nix b/flake.nix
index e2e510cbc..dbd45f053 100644
--- a/flake.nix
+++ b/flake.nix
@@ -1,7 +1,13 @@
 {
   description = "The purely functional package manager";
 
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small";
+  # TODO Go back to nixos-23.05-small once
+  # https://github.com/NixOS/nixpkgs/pull/271202 is merged.
+  #
+  # Also, do not grab arbitrary further staging commits. This PR was
+  # carefully made to be based on release-23.05 and just contain
+  # rebuild-causing changes to packages that Nix actually uses.
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/staging-23.05";
   inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
   inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
   inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; };

From 59c4c82aebb814d864548c3ad2e9128ab6e902bf Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Sat, 2 Dec 2023 00:56:23 +0100
Subject: [PATCH 243/294] fix links in stores overview

---
 doc/manual/generate-manpage.nix        | 3 ++-
 doc/manual/src/store/types/index.md.in | 8 ++++----
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix
index 365422af7..ae31b2a1f 100644
--- a/doc/manual/generate-manpage.nix
+++ b/doc/manual/generate-manpage.nix
@@ -103,7 +103,8 @@ let
             ${allStores}
           '';
           index = replaceStrings
-            [ "@store-types@" ] [ storesOverview ]
+            [ "@store-types@" "./local-store.md" "./local-daemon-store.md" ]
+            [ storesOverview "#local-store" "#local-daemon-store" ]
             details.doc;
           storesOverview =
             let
diff --git a/doc/manual/src/store/types/index.md.in b/doc/manual/src/store/types/index.md.in
index b4db553a2..a35161ce8 100644
--- a/doc/manual/src/store/types/index.md.in
+++ b/doc/manual/src/store/types/index.md.in
@@ -29,15 +29,15 @@ supported settings for each store type are documented below.
 The special store URL `auto` causes Nix to automatically select a
 store as follows:
 
-* Use the [local store](#local-store) `/nix/store` if `/nix/var/nix`
+* Use the [local store](./local-store.md) `/nix/store` if `/nix/var/nix`
   is writable by the current user.
 
 * Otherwise, if `/nix/var/nix/daemon-socket/socket` exists, [connect
-  to the Nix daemon listening on that socket](#local-daemon-store).
+  to the Nix daemon listening on that socket](./local-daemon-store.md).
 
-* Otherwise, on Linux only, use the [local chroot store](#local-store)
+* Otherwise, on Linux only, use the [local chroot store](./local-store.md)
   `~/.local/share/nix/root`, which will be created automatically if it
   does not exist.
 
-* Otherwise, use the [local store](#local-store) `/nix/store`.
+* Otherwise, use the [local store](./local-store.md) `/nix/store`.
 

From 24b781773f3d24b62b1c36154c36fc98417cbcdb Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Sat, 2 Dec 2023 03:02:58 +0100
Subject: [PATCH 244/294] fix random docs errors

remove link to the contributing guide from user documentation.
it doesn't help here, and the target at first glance shows redundant
information.
---
 src/libexpr/primops.cc  | 2 +-
 src/libstore/globals.hh | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 63c90795a..c2499bdae 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -4400,7 +4400,7 @@ void EvalState::createBaseEnv()
     addConstant("__currentSystem", v, {
         .type = nString,
         .doc = R"(
-          The value of the [`system` configuration option](@docroot@/command-ref/conf-file.md#conf-pure-eval).
+          The value of the [`system` configuration option](@docroot@/command-ref/conf-file.md#conf-system).
 
           It can be used to set the `system` attribute for [`builtins.derivation`](@docroot@/language/derivations.md) such that the resulting derivation can be built on the same system that evaluates the Nix expression:
 
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 38b0d516c..8da9e371f 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -201,7 +201,7 @@ public:
           Nix will only build a given [derivation](@docroot@/language/derivations.md) locally when its `system` attribute equals any of the values specified here or in [`extra-platforms`](#conf-extra-platforms).
 
           The default value is set when Nix itself is compiled for the system it will run on.
-          The following system types are widely used, as [Nix is actively supported on these platforms](@docroot@/contributing/hacking.md#platforms):
+          The following system types are widely used, as Nix is actively supported on these platforms:
 
           - `x86_64-linux`
           - `x86_64-darwin`
@@ -761,7 +761,7 @@ public:
         "substituters",
         R"(
           A list of [URLs of Nix stores](@docroot@/store/types/index.md#store-url-format) to be used as substituters, separated by whitespace.
-          A substituter is an additional [store]{@docroot@/glossary.md##gloss-store} from which Nix can obtain [store objects](@docroot@/glossary.md#gloss-store-object) instead of building them.
+          A substituter is an additional [store](@docroot@/glossary.md#gloss-store) from which Nix can obtain [store objects](@docroot@/glossary.md#gloss-store-object) instead of building them.
 
           Substituters are tried based on their priority value, which each substituter can set independently.
           Lower value means higher priority.

From 2e5abc0fd0d5d45e125e1d981958149624268090 Mon Sep 17 00:00:00 2001
From: wh0 <wh0@users.noreply.github.com>
Date: Sun, 3 Dec 2023 17:18:58 -0800
Subject: [PATCH 245/294] tests: avoid a chroot store without sandbox support

---
 tests/functional/build-remote-trustless-should-fail-0.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/functional/build-remote-trustless-should-fail-0.sh b/tests/functional/build-remote-trustless-should-fail-0.sh
index fad1def59..3d4a4b097 100644
--- a/tests/functional/build-remote-trustless-should-fail-0.sh
+++ b/tests/functional/build-remote-trustless-should-fail-0.sh
@@ -4,6 +4,7 @@ enableFeatures "daemon-trust-override"
 
 restartDaemon
 
+requireSandboxSupport
 [[ $busybox =~ busybox ]] || skipTest "no busybox"
 
 unset NIX_STORE_DIR

From 3c310bde2e492c2dd8bdccdfd80076231905a429 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Tue, 14 Nov 2023 11:40:56 +0100
Subject: [PATCH 246/294] reword description for the `fetch-tree` experimental
 feature

without knowing a lot of context, it's not clear who "we" are in that
text. I'm also strongly opposed to adding procedural notes into
a reference manual; it just won't age well.

this change leaves a factual description of the experimental feature and
its purpose.
---
 src/libutil/experimental-features.cc | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc
index e4bdb8cb3..9b46fc5b0 100644
--- a/src/libutil/experimental-features.cc
+++ b/src/libutil/experimental-features.cc
@@ -80,12 +80,11 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
         .description = R"(
             Enable the use of the [`fetchTree`](@docroot@/language/builtins.md#builtins-fetchTree) built-in function in the Nix language.
 
-            `fetchTree` exposes a large suite of fetching functionality in a more systematic way.
+            `fetchTree` exposes a generic interface for fetching remote file system trees from different types of remote sources.
             The [`flakes`](#xp-feature-flakes) feature flag always enables `fetch-tree`.
+            This built-in was previously guarded by the `flakes` experimental feature because of that overlap.
 
-            This built-in was previously guarded by the `flakes` experimental feature because of that overlap,
-            but since the plan is to work on stabilizing this first (due 2024 Q1), we are putting it underneath a separate feature.
-            Once we've made the changes we want to make, enabling just this feature will serve as a "release candidate" --- allowing users to try out the functionality we want to stabilize and not any other functionality we don't yet want to, in isolation.
+            Enabling just this feature serves as a "release candidate", allowing users to try it out in isolation.
         )",
     },
     {

From 5fe2accb754249df6cb8f840330abfcf3bd26695 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Tue, 14 Nov 2023 11:44:34 +0100
Subject: [PATCH 247/294] fix up release note

---
 doc/manual/src/release-notes/rl-2.19.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/src/release-notes/rl-2.19.md b/doc/manual/src/release-notes/rl-2.19.md
index 4eecaf929..ba6eb9c64 100644
--- a/doc/manual/src/release-notes/rl-2.19.md
+++ b/doc/manual/src/release-notes/rl-2.19.md
@@ -18,7 +18,7 @@
 - `nix-shell` shebang lines now support single-quoted arguments.
 
 - `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree).
-  As described in the documentation for that feature, this is because we anticipate polishing it and then stabilizing it before the rest of flakes.
+  This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/contributing/experimental-features.md#xp-fetch-tree).
 
 - The interface for creating and updating lock files has been overhauled:
 

From 823512c1e705d1fce8dfb8cde65228364c9a8045 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 4 Dec 2023 22:09:10 +0000
Subject: [PATCH 248/294] Bump zeebe-io/backport-action from 2.1.1 to 2.2.0

Bumps [zeebe-io/backport-action](https://github.com/zeebe-io/backport-action) from 2.1.1 to 2.2.0.
- [Release notes](https://github.com/zeebe-io/backport-action/releases)
- [Commits](https://github.com/zeebe-io/backport-action/compare/v2.1.1...v2.2.0)

---
updated-dependencies:
- dependency-name: zeebe-io/backport-action
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/backport.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml
index 975c90b91..85ddcfad3 100644
--- a/.github/workflows/backport.yml
+++ b/.github/workflows/backport.yml
@@ -21,7 +21,7 @@ jobs:
           fetch-depth: 0
       - name: Create backport PRs
         # should be kept in sync with `version`
-        uses: zeebe-io/backport-action@v2.1.1
+        uses: zeebe-io/backport-action@v2.2.0
         with:
           # Config README: https://github.com/zeebe-io/backport-action#backport-action
           github_token: ${{ secrets.GITHUB_TOKEN }}

From c446e5294dbc12729e7bc55ee10b40dbaeeaacf0 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 4 Dec 2023 22:09:14 +0000
Subject: [PATCH 249/294] Bump cachix/install-nix-action from 23 to 24

Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 23 to 24.
- [Release notes](https://github.com/cachix/install-nix-action/releases)
- [Commits](https://github.com/cachix/install-nix-action/compare/v23...v24)

---
updated-dependencies:
- dependency-name: cachix/install-nix-action
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/ci.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index afe4dc2e3..34a23b5f2 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -20,7 +20,7 @@ jobs:
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v23
+    - uses: cachix/install-nix-action@v24
       with:
         # The sandbox would otherwise be disabled by default on Darwin
         extra_nix_config: "sandbox = true"
@@ -62,7 +62,7 @@ jobs:
       with:
         fetch-depth: 0
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v23
+    - uses: cachix/install-nix-action@v24
       with:
         install_url: https://releases.nixos.org/nix/nix-2.13.3/install
     - uses: cachix/cachix-action@v12
@@ -84,7 +84,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v23
+    - uses: cachix/install-nix-action@v24
       with:
         install_url: '${{needs.installer.outputs.installerURL}}'
         install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve"
@@ -114,7 +114,7 @@ jobs:
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v23
+    - uses: cachix/install-nix-action@v24
       with:
         install_url: https://releases.nixos.org/nix/nix-2.13.3/install
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV

From e6a3cbfceb66e06184b625a3913a786f68e71a1f Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 4 Dec 2023 22:09:18 +0000
Subject: [PATCH 250/294] Bump cachix/cachix-action from 12 to 13

Bumps [cachix/cachix-action](https://github.com/cachix/cachix-action) from 12 to 13.
- [Release notes](https://github.com/cachix/cachix-action/releases)
- [Commits](https://github.com/cachix/cachix-action/compare/v12...v13)

---
updated-dependencies:
- dependency-name: cachix/cachix-action
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/ci.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index afe4dc2e3..033832c9a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -25,7 +25,7 @@ jobs:
         # The sandbox would otherwise be disabled by default on Darwin
         extra_nix_config: "sandbox = true"
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/cachix-action@v12
+    - uses: cachix/cachix-action@v13
       if: needs.check_secrets.outputs.cachix == 'true'
       with:
         name: '${{ env.CACHIX_NAME }}'
@@ -65,7 +65,7 @@ jobs:
     - uses: cachix/install-nix-action@v23
       with:
         install_url: https://releases.nixos.org/nix/nix-2.13.3/install
-    - uses: cachix/cachix-action@v12
+    - uses: cachix/cachix-action@v13
       with:
         name: '${{ env.CACHIX_NAME }}'
         signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
@@ -119,7 +119,7 @@ jobs:
         install_url: https://releases.nixos.org/nix/nix-2.13.3/install
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
     - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV
-    - uses: cachix/cachix-action@v12
+    - uses: cachix/cachix-action@v13
       if: needs.check_secrets.outputs.cachix == 'true'
       with:
         name: '${{ env.CACHIX_NAME }}'

From e488a43f457f3ef9dba92184428bbe5381fe2634 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 4 Dec 2023 22:09:25 +0000
Subject: [PATCH 251/294] Bump actions/labeler from 4 to 5

Bumps [actions/labeler](https://github.com/actions/labeler) from 4 to 5.
- [Release notes](https://github.com/actions/labeler/releases)
- [Commits](https://github.com/actions/labeler/compare/v4...v5)

---
updated-dependencies:
- dependency-name: actions/labeler
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/labels.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml
index d83cb4f18..34aa4e6bd 100644
--- a/.github/workflows/labels.yml
+++ b/.github/workflows/labels.yml
@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-latest
     if: github.repository_owner == 'NixOS'
     steps:
-    - uses: actions/labeler@v4
+    - uses: actions/labeler@v5
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         sync-labels: false

From 345f79d01676680f2d4ef8803790896a190c855b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 5 Dec 2023 15:14:28 +0100
Subject: [PATCH 252/294] Check that we can't follow symlinks outside of the
 allowed paths

---
 tests/functional/restricted.sh | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh
index b8deceacc..cb83c34b1 100644
--- a/tests/functional/restricted.sh
+++ b/tests/functional/restricted.sh
@@ -39,6 +39,15 @@ nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I $TEST_ROOT -
 
 [[ $(nix eval --raw --impure --restrict-eval -I . --expr 'builtins.readFile "${import ./simple.nix}/hello"') == 'Hello World!' ]]
 
+# Check that we can't follow a symlink outside of the allowed paths.
+mkdir -p $TEST_ROOT/tunnel.d
+ln -sfn .. $TEST_ROOT/tunnel.d/tunnel
+echo foo > $TEST_ROOT/bar
+
+expectStderr 1 nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readFile <foo/tunnel/bar>" -I $TEST_ROOT/tunnel.d | grepQuiet "forbidden in restricted mode"
+
+expectStderr 1 nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readDir <foo/tunnel>" -I $TEST_ROOT/tunnel.d | grepQuiet "forbidden in restricted mode"
+
 # Check whether we can leak symlink information through directory traversal.
 traverseDir="$(pwd)/restricted-traverse-me"
 ln -sfn "$(pwd)/restricted-secret" "$(pwd)/restricted-innocent"

From 733333e87db391e4f832de65f0f49f60e50c45a4 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 1 Dec 2023 17:38:34 -0500
Subject: [PATCH 253/294] Including `config.h` also needs `$(buildprefix)`

Per the instruction in the manual, we want to run configure in a
different directory so that we can configure + build for multiple
platforms. That means `config.h` will be in the build directory. This is
just like `Makefile.config`, which already is used with
`$(buildprefix)`.
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index eea297c89..41f14ac92 100644
--- a/Makefile
+++ b/Makefile
@@ -64,4 +64,4 @@ $(eval $(call include-sub-makefile, doc/manual/local.mk))
 $(eval $(call include-sub-makefile, doc/internal-api/local.mk))
 endif
 
-GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++2a -I src
+GLOBAL_CXXFLAGS += -g -Wall -include $(buildprefix)config.h -std=c++2a -I src

From 83c067c0fa0cc5a2dca440e5c986afe40b163802 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 5 Dec 2023 23:02:59 +0100
Subject: [PATCH 254/294] PosixSourceAccessor: Don't follow any symlinks

All path components must not be symlinks now (so the user needs to
call `resolveSymlinks()` when needed).
---
 src/libexpr/parser.y                 | 11 +++++-----
 src/libexpr/primops.cc               | 30 ++++++++++++++--------------
 src/libutil/posix-source-accessor.cc | 27 +++++++++++++++++++++----
 src/libutil/posix-source-accessor.hh |  5 +++++
 src/nix-env/nix-env.cc               |  6 +++---
 src/nix-env/user-env.cc              |  2 +-
 tests/functional/restricted.sh       |  7 +++++--
 7 files changed, 58 insertions(+), 30 deletions(-)

diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 58fc580fc..16ad8af2e 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -692,16 +692,17 @@ SourcePath resolveExprPath(SourcePath path)
 
     /* If `path' is a symlink, follow it.  This is so that relative
        path references work. */
-    while (true) {
+    while (!path.path.isRoot()) {
         // Basic cycle/depth limit to avoid infinite loops.
         if (++followCount >= maxFollow)
             throw Error("too many symbolic links encountered while traversing the path '%s'", path);
-        if (path.lstat().type != InputAccessor::tSymlink) break;
-        path = {path.accessor, CanonPath(path.readLink(), path.path.parent().value_or(CanonPath::root))};
+        auto p = path.parent().resolveSymlinks() + path.baseName();
+        if (p.lstat().type != InputAccessor::tSymlink) break;
+        path = {path.accessor, CanonPath(p.readLink(), path.path.parent().value_or(CanonPath::root))};
     }
 
     /* If `path' refers to a directory, append `/default.nix'. */
-    if (path.lstat().type == InputAccessor::tDirectory)
+    if (path.resolveSymlinks().lstat().type == InputAccessor::tDirectory)
         return path + "default.nix";
 
     return path;
@@ -716,7 +717,7 @@ Expr * EvalState::parseExprFromFile(const SourcePath & path)
 
 Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr<StaticEnv> & staticEnv)
 {
-    auto buffer = path.readFile();
+    auto buffer = path.resolveSymlinks().readFile();
     // readFile hopefully have left some extra space for terminators
     buffer.append("\0\0", 2);
     return parse(buffer.data(), buffer.size(), Pos::Origin(path), path.parent(), staticEnv);
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index c442de986..f2d51f8f5 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -110,7 +110,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
     return res;
 }
 
-static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v)
+static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, bool resolveSymlinks = true)
 {
     NixStringContext context;
 
@@ -120,9 +120,9 @@ static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v)
         if (!context.empty() && path.accessor == state.rootFS) {
             auto rewrites = state.realiseContext(context);
             auto realPath = state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context);
-            return {path.accessor, CanonPath(realPath)};
-        } else
-            return path;
+            path = {path.accessor, CanonPath(realPath)};
+        }
+        return resolveSymlinks ? path.resolveSymlinks() : path;
     } catch (Error & e) {
         e.addTrace(state.positions[pos], "while realising the context of path '%s'", path);
         throw;
@@ -162,7 +162,7 @@ static void mkOutputString(
    argument. */
 static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * vScope, Value & v)
 {
-    auto path = realisePath(state, pos, vPath);
+    auto path = realisePath(state, pos, vPath, false);
     auto path2 = path.path.abs();
 
     // FIXME
@@ -1525,16 +1525,16 @@ static RegisterPrimOp primop_storePath({
 
 static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    auto & arg = *args[0];
-
-    auto path = realisePath(state, pos, arg);
-
-    /* SourcePath doesn't know about trailing slash. */
-    auto mustBeDir = arg.type() == nString
-        && (arg.string_view().ends_with("/")
-            || arg.string_view().ends_with("/."));
-
     try {
+        auto & arg = *args[0];
+
+        auto path = realisePath(state, pos, arg);
+
+        /* SourcePath doesn't know about trailing slash. */
+        auto mustBeDir = arg.type() == nString
+            && (arg.string_view().ends_with("/")
+                || arg.string_view().ends_with("/."));
+
         auto st = path.maybeLstat();
         auto exists = st && (!mustBeDir || st->type == SourceAccessor::tDirectory);
         v.mkBool(exists);
@@ -1771,7 +1771,7 @@ static std::string_view fileTypeToString(InputAccessor::Type type)
 
 static void prim_readFileType(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    auto path = realisePath(state, pos, *args[0]);
+    auto path = realisePath(state, pos, *args[0], false);
     /* Retrieve the directory entry type and stringize it. */
     v.mkString(fileTypeToString(path.lstat().type));
 }
diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc
index dc96f84e5..0601e6387 100644
--- a/src/libutil/posix-source-accessor.cc
+++ b/src/libutil/posix-source-accessor.cc
@@ -8,9 +8,9 @@ void PosixSourceAccessor::readFile(
     Sink & sink,
     std::function<void(uint64_t)> sizeCallback)
 {
-    // FIXME: add O_NOFOLLOW since symlinks should be resolved by the
-    // caller?
-    AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
+    assertNoSymlinks(path);
+
+    AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC | O_NOFOLLOW);
     if (!fd)
         throw SysError("opening file '%1%'", path);
 
@@ -42,14 +42,16 @@ void PosixSourceAccessor::readFile(
 
 bool PosixSourceAccessor::pathExists(const CanonPath & path)
 {
+    if (auto parent = path.parent()) assertNoSymlinks(*parent);
     return nix::pathExists(path.abs());
 }
 
 std::optional<SourceAccessor::Stat> PosixSourceAccessor::maybeLstat(const CanonPath & path)
 {
+    if (auto parent = path.parent()) assertNoSymlinks(*parent);
     struct stat st;
     if (::lstat(path.c_str(), &st)) {
-        if (errno == ENOENT) return std::nullopt;
+        if (errno == ENOENT || errno == ENOTDIR) return std::nullopt;
         throw SysError("getting status of '%s'", showPath(path));
     }
     mtime = std::max(mtime, st.st_mtime);
@@ -66,6 +68,7 @@ std::optional<SourceAccessor::Stat> PosixSourceAccessor::maybeLstat(const CanonP
 
 SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & path)
 {
+    assertNoSymlinks(path);
     DirEntries res;
     for (auto & entry : nix::readDirectory(path.abs())) {
         std::optional<Type> type;
@@ -81,6 +84,7 @@ SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath &
 
 std::string PosixSourceAccessor::readLink(const CanonPath & path)
 {
+    if (auto parent = path.parent()) assertNoSymlinks(*parent);
     return nix::readLink(path.abs());
 }
 
@@ -89,4 +93,19 @@ std::optional<CanonPath> PosixSourceAccessor::getPhysicalPath(const CanonPath &
     return path;
 }
 
+void PosixSourceAccessor::assertNoSymlinks(CanonPath path)
+{
+    // FIXME: cache this since it potentially causes a lot of lstat calls.
+    while (!path.isRoot()) {
+        struct stat st;
+        if (::lstat(path.c_str(), &st)) {
+            if (errno != ENOENT)
+                throw SysError("getting status of '%s'", showPath(path));
+        }
+        if (S_ISLNK(st.st_mode))
+            throw Error("path '%s' is a symlink", showPath(path));
+        path.pop();
+    }
+}
+
 }
diff --git a/src/libutil/posix-source-accessor.hh b/src/libutil/posix-source-accessor.hh
index a45d96bf8..7189a40e5 100644
--- a/src/libutil/posix-source-accessor.hh
+++ b/src/libutil/posix-source-accessor.hh
@@ -29,6 +29,11 @@ struct PosixSourceAccessor : virtual SourceAccessor
     std::string readLink(const CanonPath & path) override;
 
     std::optional<CanonPath> getPhysicalPath(const CanonPath & path) override;
+
+    /**
+     * Throw an error if `path` or any of its ancestors are symlinks.
+     */
+    void assertNoSymlinks(CanonPath path);
 };
 
 }
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 86126c7ad..e2bbd9775 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -97,7 +97,7 @@ static bool isNixExpr(const SourcePath & path, struct InputAccessor::Stat & st)
 {
     return
         st.type == InputAccessor::tRegular
-        || (st.type == InputAccessor::tDirectory && (path + "default.nix").pathExists());
+        || (st.type == InputAccessor::tDirectory && (path + "default.nix").resolveSymlinks().pathExists());
 }
 
 
@@ -116,11 +116,11 @@ static void getAllExprs(EvalState & state,
            are implemented using profiles). */
         if (i == "manifest.nix") continue;
 
-        SourcePath path2 = path + i;
+        auto path2 = (path + i).resolveSymlinks();
 
         InputAccessor::Stat st;
         try {
-            st = path2.resolveSymlinks().lstat();
+            st = path2.lstat();
         } catch (Error &) {
             continue; // ignore dangling symlinks in ~/.nix-defexpr
         }
diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc
index 250224e7d..34f6bd005 100644
--- a/src/nix-env/user-env.cc
+++ b/src/nix-env/user-env.cc
@@ -21,7 +21,7 @@ DrvInfos queryInstalled(EvalState & state, const Path & userEnv)
     auto manifestFile = userEnv + "/manifest.nix";
     if (pathExists(manifestFile)) {
         Value v;
-        state.evalFile(state.rootPath(CanonPath(manifestFile)), v);
+        state.evalFile(state.rootPath(CanonPath(manifestFile)).resolveSymlinks(), v);
         Bindings & bindings(*state.allocBindings(0));
         getDerivations(state, v, "", bindings, elems, false);
     }
diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh
index cb83c34b1..2d6ab964b 100644
--- a/tests/functional/restricted.sh
+++ b/tests/functional/restricted.sh
@@ -40,13 +40,16 @@ nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I $TEST_ROOT -
 [[ $(nix eval --raw --impure --restrict-eval -I . --expr 'builtins.readFile "${import ./simple.nix}/hello"') == 'Hello World!' ]]
 
 # Check that we can't follow a symlink outside of the allowed paths.
-mkdir -p $TEST_ROOT/tunnel.d
+mkdir -p $TEST_ROOT/tunnel.d $TEST_ROOT/foo2
 ln -sfn .. $TEST_ROOT/tunnel.d/tunnel
 echo foo > $TEST_ROOT/bar
 
 expectStderr 1 nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readFile <foo/tunnel/bar>" -I $TEST_ROOT/tunnel.d | grepQuiet "forbidden in restricted mode"
 
-expectStderr 1 nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readDir <foo/tunnel>" -I $TEST_ROOT/tunnel.d | grepQuiet "forbidden in restricted mode"
+expectStderr 1 nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readDir <foo/tunnel/foo2>" -I $TEST_ROOT/tunnel.d | grepQuiet "forbidden in restricted mode"
+
+# Reading the parents of allowed paths should show only the ancestors of the allowed paths.
+[[ $(nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readDir <foo/tunnel>" -I $TEST_ROOT/tunnel.d) == '{ "tunnel.d" = "directory"; }' ]]
 
 # Check whether we can leak symlink information through directory traversal.
 traverseDir="$(pwd)/restricted-traverse-me"

From b23273f6a29c725646b3523b1c35a0ae4a84ef61 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 5 Dec 2023 18:10:37 -0500
Subject: [PATCH 255/294] Add missing `-pthread` for test support libraries

This is good in general (see how the other libraries also have long had
it, since 49fe9592a47e7819179c2de4fd6068e897e944c7) but in particular
needed to fix the NetBSD build.
---
 tests/unit/libexpr-support/local.mk  | 2 +-
 tests/unit/libstore-support/local.mk | 2 +-
 tests/unit/libutil-support/local.mk  | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/unit/libexpr-support/local.mk b/tests/unit/libexpr-support/local.mk
index 28e87b8f2..12a76206a 100644
--- a/tests/unit/libexpr-support/local.mk
+++ b/tests/unit/libexpr-support/local.mk
@@ -20,4 +20,4 @@ libexpr-test-support_LIBS = \
     libstore-test-support libutil-test-support \
     libexpr libstore libutil
 
-libexpr-test-support_LDFLAGS := -lrapidcheck
+libexpr-test-support_LDFLAGS := -pthread -lrapidcheck
diff --git a/tests/unit/libstore-support/local.mk b/tests/unit/libstore-support/local.mk
index d5d657c91..ff075c96a 100644
--- a/tests/unit/libstore-support/local.mk
+++ b/tests/unit/libstore-support/local.mk
@@ -18,4 +18,4 @@ libstore-test-support_LIBS = \
     libutil-test-support \
     libstore libutil
 
-libstore-test-support_LDFLAGS := -lrapidcheck
+libstore-test-support_LDFLAGS := -pthread -lrapidcheck
diff --git a/tests/unit/libutil-support/local.mk b/tests/unit/libutil-support/local.mk
index 43a1551e5..2ee2cdb6c 100644
--- a/tests/unit/libutil-support/local.mk
+++ b/tests/unit/libutil-support/local.mk
@@ -16,4 +16,4 @@ libutil-test-support_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES)
 
 libutil-test-support_LIBS = libutil
 
-libutil-test-support_LDFLAGS := -lrapidcheck
+libutil-test-support_LDFLAGS := -pthread -lrapidcheck

From 504e4fc4576dc6a4cd5c083a3bf7b80dfb0ca220 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 6 Dec 2023 13:45:59 +0100
Subject: [PATCH 256/294] CanonPath: Support std::hash

---
 src/libfetchers/git-utils.cc |  2 +-
 src/libutil/canon-path.hh    | 11 +++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index 19eae0e1d..5f2a7a8bc 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -554,7 +554,7 @@ struct GitInputAccessor : InputAccessor
         return toHash(*git_tree_entry_id(entry));
     }
 
-    std::map<CanonPath, TreeEntry> lookupCache;
+    std::unordered_map<CanonPath, TreeEntry> lookupCache;
 
     /* Recursively look up 'path' relative to the root. */
     git_tree_entry * lookup(const CanonPath & path)
diff --git a/src/libutil/canon-path.hh b/src/libutil/canon-path.hh
index 6d0519f4f..6aff4ec0d 100644
--- a/src/libutil/canon-path.hh
+++ b/src/libutil/canon-path.hh
@@ -205,8 +205,19 @@ public:
      * `CanonPath(this.makeRelative(x), this) == path`.
      */
     std::string makeRelative(const CanonPath & path) const;
+
+    friend class std::hash<CanonPath>;
 };
 
 std::ostream & operator << (std::ostream & stream, const CanonPath & path);
 
 }
+
+template<>
+struct std::hash<nix::CanonPath>
+{
+    std::size_t operator ()(const nix::CanonPath & s) const noexcept
+    {
+        return std::hash<std::string>{}(s.path);
+    }
+};

From 57246c4c3802920e6167fd540dae2a0abca97f15 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 6 Dec 2023 13:55:07 +0100
Subject: [PATCH 257/294] PosixSourceAccessor: Cache lstat() calls

Since we're doing a lot of them in assertNoSymlinks().
---
 src/libutil/posix-source-accessor.cc | 56 +++++++++++++++++++---------
 src/libutil/posix-source-accessor.hh |  4 ++
 2 files changed, 42 insertions(+), 18 deletions(-)

diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc
index 0601e6387..15ff76e59 100644
--- a/src/libutil/posix-source-accessor.cc
+++ b/src/libutil/posix-source-accessor.cc
@@ -1,5 +1,8 @@
 #include "posix-source-accessor.hh"
 #include "signals.hh"
+#include "sync.hh"
+
+#include <unordered_map>
 
 namespace nix {
 
@@ -46,23 +49,45 @@ bool PosixSourceAccessor::pathExists(const CanonPath & path)
     return nix::pathExists(path.abs());
 }
 
+std::optional<struct stat> PosixSourceAccessor::cachedLstat(const CanonPath & path)
+{
+    static Sync<std::unordered_map<CanonPath, std::optional<struct stat>>> _cache;
+
+    {
+        auto cache(_cache.lock());
+        auto i = cache->find(path);
+        if (i != cache->end()) return i->second;
+    }
+
+    std::optional<struct stat> st{std::in_place};
+    if (::lstat(path.c_str(), &*st)) {
+        if (errno == ENOENT || errno == ENOTDIR)
+            st.reset();
+        else
+            throw SysError("getting status of '%s'", showPath(path));
+    }
+
+    auto cache(_cache.lock());
+    if (cache->size() >= 16384) cache->clear();
+    cache->emplace(path, st);
+
+    return st;
+}
+
 std::optional<SourceAccessor::Stat> PosixSourceAccessor::maybeLstat(const CanonPath & path)
 {
     if (auto parent = path.parent()) assertNoSymlinks(*parent);
-    struct stat st;
-    if (::lstat(path.c_str(), &st)) {
-        if (errno == ENOENT || errno == ENOTDIR) return std::nullopt;
-        throw SysError("getting status of '%s'", showPath(path));
-    }
-    mtime = std::max(mtime, st.st_mtime);
+    auto st = cachedLstat(path);
+    if (!st) return std::nullopt;
+    mtime = std::max(mtime, st->st_mtime);
     return Stat {
         .type =
-            S_ISREG(st.st_mode) ? tRegular :
-            S_ISDIR(st.st_mode) ? tDirectory :
-            S_ISLNK(st.st_mode) ? tSymlink :
+            S_ISREG(st->st_mode) ? tRegular :
+            S_ISDIR(st->st_mode) ? tDirectory :
+            S_ISLNK(st->st_mode) ? tSymlink :
             tMisc,
-        .fileSize = S_ISREG(st.st_mode) ? std::optional<uint64_t>(st.st_size) : std::nullopt,
-        .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR,
+        .fileSize = S_ISREG(st->st_mode) ? std::optional<uint64_t>(st->st_size) : std::nullopt,
+        .isExecutable = S_ISREG(st->st_mode) && st->st_mode & S_IXUSR,
     };
 }
 
@@ -95,14 +120,9 @@ std::optional<CanonPath> PosixSourceAccessor::getPhysicalPath(const CanonPath &
 
 void PosixSourceAccessor::assertNoSymlinks(CanonPath path)
 {
-    // FIXME: cache this since it potentially causes a lot of lstat calls.
     while (!path.isRoot()) {
-        struct stat st;
-        if (::lstat(path.c_str(), &st)) {
-            if (errno != ENOENT)
-                throw SysError("getting status of '%s'", showPath(path));
-        }
-        if (S_ISLNK(st.st_mode))
+        auto st = cachedLstat(path);
+        if (st && S_ISLNK(st->st_mode))
             throw Error("path '%s' is a symlink", showPath(path));
         path.pop();
     }
diff --git a/src/libutil/posix-source-accessor.hh b/src/libutil/posix-source-accessor.hh
index 7189a40e5..b2bd39805 100644
--- a/src/libutil/posix-source-accessor.hh
+++ b/src/libutil/posix-source-accessor.hh
@@ -30,10 +30,14 @@ struct PosixSourceAccessor : virtual SourceAccessor
 
     std::optional<CanonPath> getPhysicalPath(const CanonPath & path) override;
 
+private:
+
     /**
      * Throw an error if `path` or any of its ancestors are symlinks.
      */
     void assertNoSymlinks(CanonPath path);
+
+    std::optional<struct stat> cachedLstat(const CanonPath & path);
 };
 
 }

From 53ab5d87c2eef72202bd76eb43e072636bbc72e8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 6 Dec 2023 14:05:32 +0100
Subject: [PATCH 258/294] Use expectStderr

---
 tests/functional/restricted.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh
index 2d6ab964b..3de26eb36 100644
--- a/tests/functional/restricted.sh
+++ b/tests/functional/restricted.sh
@@ -14,7 +14,7 @@ nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I sr
 (! nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel')
 nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel' -I src=../../src
 
-(! nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile <foo/simple.nix>')
+expectStderr 1 nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile <foo/simple.nix>' | grepQuiet "forbidden in restricted mode"
 nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile <foo/simple.nix>' -I src=.
 
 p=$(nix eval --raw --expr "builtins.fetchurl file://$(pwd)/restricted.sh" --impure --restrict-eval --allowed-uris "file://$(pwd)")

From ee8540ae9055791cfec4cbf8cb6335368b867acc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <7226587+thufschmitt@users.noreply.github.com>
Date: Wed, 6 Dec 2023 14:07:08 +0100
Subject: [PATCH 259/294] Fix the labeler.yml config file

labeler 5.0 changed the configuration file in a non-backwards-compatible way (https://github.com/actions/labeler/tree/main#breaking-changes-in-v5), so update our config file to match that (because all the CIs are red otherwise :grimacing: ).
---
 .github/labeler.yml | 31 +++++++++++++++++++------------
 1 file changed, 19 insertions(+), 12 deletions(-)

diff --git a/.github/labeler.yml b/.github/labeler.yml
index 7544f07a6..b1b18c488 100644
--- a/.github/labeler.yml
+++ b/.github/labeler.yml
@@ -1,23 +1,30 @@
 "documentation":
-  - doc/manual/*
-  - src/nix/**/*.md
+  - changed-files:
+    - any-glob-to-any-file: "doc/manual/*"
+    - any-glob-to-any-file: "src/nix/**/*.md"
 
 "store":
-  - src/libstore/store-api.*
-  - src/libstore/*-store.*
+  - changed-files:
+    - any-glob-to-any-file: "src/libstore/store-api.*"
+    - any-glob-to-any-file: "src/libstore/*-store.*"
 
 "fetching":
-  - src/libfetchers/**/*
+  - changed-files:
+    - any-glob-to-any-file: "src/libfetchers/**/*"
 
 "repl":
-  - src/libcmd/repl.*
-  - src/nix/repl.*
+  - changed-files:
+    - any-glob-to-any-file: "src/libcmd/repl.*"
+    - any-glob-to-any-file: "src/nix/repl.*"
 
 "new-cli":
-  - src/nix/**/*
+  - changed-files:
+    - any-glob-to-any-file: "src/nix/**/*"
 
 "with-tests":
-  # Unit tests
-  - src/*/tests/**/*
-  # Functional and integration tests
-  - tests/functional/**/*
+  - changed-files:
+    # Unit tests
+    - any-glob-to-any-file: "src/*/tests/**/*"
+    # Functional and integration tests
+    - any-glob-to-any-file: "tests/functional/**/*"
+    

From 2bd83225004012af97d2d5977dc1de952f60aa8d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 6 Dec 2023 14:08:40 +0100
Subject: [PATCH 260/294] Update src/libfetchers/filtering-input-accessor.hh

Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
---
 src/libfetchers/filtering-input-accessor.hh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libfetchers/filtering-input-accessor.hh b/src/libfetchers/filtering-input-accessor.hh
index 209d26974..e1b83c929 100644
--- a/src/libfetchers/filtering-input-accessor.hh
+++ b/src/libfetchers/filtering-input-accessor.hh
@@ -13,8 +13,8 @@ typedef std::function<RestrictedPathError(const CanonPath & path)> MakeNotAllowe
 
 /**
  * An abstract wrapping `InputAccessor` that performs access
- * control. Subclasses should override `checkAccess()` to implement an
- * access control policy.
+ * control. Subclasses should override `isAllowed()` to implement an
+ * access control policy. The error message is customized at construction.
  */
 struct FilteringInputAccessor : InputAccessor
 {

From 7fff625e39fa6b11c4c61eeacadc70a0253bdab6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <7226587+thufschmitt@users.noreply.github.com>
Date: Wed, 6 Dec 2023 14:13:45 +0100
Subject: [PATCH 261/294] =?UTF-8?q?Improve=20the=20error=20message=20for?=
 =?UTF-8?q?=20=E2=80=9Cmulticommands=E2=80=9D=20commands=20(#9510)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Factor out the default `MultiCommand` behavior

All the `MultiCommand`s had (nearly) the same behavior when called
without a subcommand.
Factor out this behavior into the `NixMultiCommand` class.

* Display the list of available subcommands when none is specified

Whenever a user runs a command that excepts a subcommand, add the list
of available subcommands to the error message.

* Print the multi-command lists as Markdown lists

This takes more screen real estate, but is also much more readable than
a comma-separated list
---
 src/libcmd/command.cc  | 14 ++++++++++++++
 src/libcmd/command.hh  |  6 +++++-
 src/libutil/args.cc    |  5 +++--
 src/libutil/args.hh    |  9 ++++++---
 src/nix/config.cc      | 11 ++---------
 src/nix/derivation.cc  | 11 ++---------
 src/nix/flake.cc       |  8 ++++----
 src/nix/hash.cc        | 11 +++--------
 src/nix/main.cc        |  2 +-
 src/nix/nar.cc         |  9 +--------
 src/nix/profile.cc     | 11 +++--------
 src/nix/realisation.cc | 11 ++---------
 src/nix/registry.cc    | 14 ++++----------
 src/nix/sigs.cc        | 11 +++--------
 src/nix/store.cc       | 11 ++---------
 15 files changed, 55 insertions(+), 89 deletions(-)

diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc
index de9f546fc..369fa6004 100644
--- a/src/libcmd/command.cc
+++ b/src/libcmd/command.cc
@@ -1,4 +1,5 @@
 #include "command.hh"
+#include "markdown.hh"
 #include "store-api.hh"
 #include "local-fs-store.hh"
 #include "derivations.hh"
@@ -34,6 +35,19 @@ nlohmann::json NixMultiCommand::toJSON()
     return MultiCommand::toJSON();
 }
 
+void NixMultiCommand::run()
+{
+    if (!command) {
+        std::set<std::string> subCommandTextLines;
+        for (auto & [name, _] : commands)
+            subCommandTextLines.insert(fmt("- `%s`", name));
+        std::string markdownError = fmt("`nix %s` requires a sub-command. Available sub-commands:\n\n%s\n",
+                commandName, concatStringsSep("\n", subCommandTextLines));
+        throw UsageError(renderMarkdownToTerminal(markdownError));
+    }
+    command->second->run();
+}
+
 StoreCommand::StoreCommand()
 {
 }
diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh
index 120c832ac..4a72627ed 100644
--- a/src/libcmd/command.hh
+++ b/src/libcmd/command.hh
@@ -26,9 +26,13 @@ static constexpr Command::Category catNixInstallation = 102;
 
 static constexpr auto installablesCategory = "Options that change the interpretation of [installables](@docroot@/command-ref/new-cli/nix.md#installables)";
 
-struct NixMultiCommand : virtual MultiCommand, virtual Command
+struct NixMultiCommand : MultiCommand, virtual Command
 {
     nlohmann::json toJSON() override;
+
+    using MultiCommand::MultiCommand;
+
+    virtual void run() override;
 };
 
 // For the overloaded run methods
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 4480a03f5..c4b2975ee 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -483,7 +483,7 @@ bool Args::processArgs(const Strings & args, bool finish)
         if (!anyCompleted)
             exp.handler.fun(ss);
 
-        /* Move the list element to the processedArgs. This is almost the same as 
+        /* Move the list element to the processedArgs. This is almost the same as
            `processedArgs.push_back(expectedArgs.front()); expectedArgs.pop_front()`,
            except that it will only adjust the next and prev pointers of the list
            elements, meaning the actual contents don't move in memory. This is
@@ -622,8 +622,9 @@ std::optional<ExperimentalFeature> Command::experimentalFeature ()
     return { Xp::NixCommand };
 }
 
-MultiCommand::MultiCommand(const Commands & commands_)
+MultiCommand::MultiCommand(std::string_view commandName, const Commands & commands_)
     : commands(commands_)
+    , commandName(commandName)
 {
     expectArgs({
         .label = "subcommand",
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 7af82b178..72278dccc 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -223,11 +223,11 @@ protected:
     std::list<ExpectedArg> expectedArgs;
     /**
      * List of processed positional argument forms.
-     * 
+     *
      * All items removed from `expectedArgs` are added here. After all
      * arguments were processed, this list should be exactly the same as
      * `expectedArgs` was before.
-     * 
+     *
      * This list is used to extend the lifetime of the argument forms.
      * If this is not done, some closures that reference the command
      * itself will segfault.
@@ -356,13 +356,16 @@ public:
      */
     std::optional<std::pair<std::string, ref<Command>>> command;
 
-    MultiCommand(const Commands & commands);
+    MultiCommand(std::string_view commandName, const Commands & commands);
 
     bool processFlag(Strings::iterator & pos, Strings::iterator end) override;
 
     bool processArgs(const Strings & args, bool finish) override;
 
     nlohmann::json toJSON() override;
+
+protected:
+    std::string commandName = "";
 };
 
 Strings argvToStrings(int argc, char * * argv);
diff --git a/src/nix/config.cc b/src/nix/config.cc
index 5b280d11d..52706afcf 100644
--- a/src/nix/config.cc
+++ b/src/nix/config.cc
@@ -7,9 +7,9 @@
 
 using namespace nix;
 
-struct CmdConfig : virtual NixMultiCommand
+struct CmdConfig : NixMultiCommand
 {
-    CmdConfig() : MultiCommand(RegisterCommand::getCommandsFor({"config"}))
+    CmdConfig() : NixMultiCommand("config", RegisterCommand::getCommandsFor({"config"}))
     { }
 
     std::string description() override
@@ -18,13 +18,6 @@ struct CmdConfig : virtual NixMultiCommand
     }
 
     Category category() override { return catUtility; }
-
-    void run() override
-    {
-        if (!command)
-            throw UsageError("'nix config' requires a sub-command.");
-        command->second->run();
-    }
 };
 
 struct CmdConfigShow : Command, MixJSON
diff --git a/src/nix/derivation.cc b/src/nix/derivation.cc
index cd3975a4f..59a78d378 100644
--- a/src/nix/derivation.cc
+++ b/src/nix/derivation.cc
@@ -2,9 +2,9 @@
 
 using namespace nix;
 
-struct CmdDerivation : virtual NixMultiCommand
+struct CmdDerivation : NixMultiCommand
 {
-    CmdDerivation() : MultiCommand(RegisterCommand::getCommandsFor({"derivation"}))
+    CmdDerivation() : NixMultiCommand("derivation", RegisterCommand::getCommandsFor({"derivation"}))
     { }
 
     std::string description() override
@@ -13,13 +13,6 @@ struct CmdDerivation : virtual NixMultiCommand
     }
 
     Category category() override { return catUtility; }
-
-    void run() override
-    {
-        if (!command)
-            throw UsageError("'nix derivation' requires a sub-command.");
-        command->second->run();
-    }
 };
 
 static auto rCmdDerivation = registerCommand<CmdDerivation>("derivation");
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index e0c67fdfa..2b6e56283 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -1399,7 +1399,9 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON
 struct CmdFlake : NixMultiCommand
 {
     CmdFlake()
-        : MultiCommand({
+        : NixMultiCommand(
+            "flake",
+            {
                 {"update", []() { return make_ref<CmdFlakeUpdate>(); }},
                 {"lock", []() { return make_ref<CmdFlakeLock>(); }},
                 {"metadata", []() { return make_ref<CmdFlakeMetadata>(); }},
@@ -1429,10 +1431,8 @@ struct CmdFlake : NixMultiCommand
 
     void run() override
     {
-        if (!command)
-            throw UsageError("'nix flake' requires a sub-command.");
         experimentalFeatureSettings.require(Xp::Flakes);
-        command->second->run();
+        NixMultiCommand::run();
     }
 };
 
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index d6595dcca..ededf6ef2 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -130,7 +130,9 @@ struct CmdToBase : Command
 struct CmdHash : NixMultiCommand
 {
     CmdHash()
-        : MultiCommand({
+        : NixMultiCommand(
+            "hash",
+            {
                 {"file", []() { return make_ref<CmdHashBase>(FileIngestionMethod::Flat);; }},
                 {"path", []() { return make_ref<CmdHashBase>(FileIngestionMethod::Recursive); }},
                 {"to-base16", []() { return make_ref<CmdToBase>(HashFormat::Base16); }},
@@ -146,13 +148,6 @@ struct CmdHash : NixMultiCommand
     }
 
     Category category() override { return catUtility; }
-
-    void run() override
-    {
-        if (!command)
-            throw UsageError("'nix hash' requires a sub-command.");
-        command->second->run();
-    }
 };
 
 static auto rCmdHash = registerCommand<CmdHash>("hash");
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 3d44e4a9d..109d2cc04 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -67,7 +67,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs
     bool helpRequested = false;
     bool showVersion = false;
 
-    NixArgs() : MultiCommand(RegisterCommand::getCommandsFor({})), MixCommonArgs("nix")
+    NixArgs() : MultiCommand("", RegisterCommand::getCommandsFor({})), MixCommonArgs("nix")
     {
         categories.clear();
         categories[catHelp] = "Help commands";
diff --git a/src/nix/nar.cc b/src/nix/nar.cc
index 9815410cf..8ad4f92a7 100644
--- a/src/nix/nar.cc
+++ b/src/nix/nar.cc
@@ -4,7 +4,7 @@ using namespace nix;
 
 struct CmdNar : NixMultiCommand
 {
-    CmdNar() : MultiCommand(RegisterCommand::getCommandsFor({"nar"}))
+    CmdNar() : NixMultiCommand("nar", RegisterCommand::getCommandsFor({"nar"}))
     { }
 
     std::string description() override
@@ -20,13 +20,6 @@ struct CmdNar : NixMultiCommand
     }
 
     Category category() override { return catUtility; }
-
-    void run() override
-    {
-        if (!command)
-            throw UsageError("'nix nar' requires a sub-command.");
-        command->second->run();
-    }
 };
 
 static auto rCmdNar = registerCommand<CmdNar>("nar");
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index 476ddcd60..147b4680b 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -825,7 +825,9 @@ struct CmdProfileWipeHistory : virtual StoreCommand, MixDefaultProfile, MixDryRu
 struct CmdProfile : NixMultiCommand
 {
     CmdProfile()
-        : MultiCommand({
+        : NixMultiCommand(
+            "profile",
+            {
               {"install", []() { return make_ref<CmdProfileInstall>(); }},
               {"remove", []() { return make_ref<CmdProfileRemove>(); }},
               {"upgrade", []() { return make_ref<CmdProfileUpgrade>(); }},
@@ -848,13 +850,6 @@ struct CmdProfile : NixMultiCommand
           #include "profile.md"
           ;
     }
-
-    void run() override
-    {
-        if (!command)
-            throw UsageError("'nix profile' requires a sub-command.");
-        command->second->run();
-    }
 };
 
 static auto rCmdProfile = registerCommand<CmdProfile>("profile");
diff --git a/src/nix/realisation.cc b/src/nix/realisation.cc
index e19e93219..e1f231222 100644
--- a/src/nix/realisation.cc
+++ b/src/nix/realisation.cc
@@ -5,9 +5,9 @@
 
 using namespace nix;
 
-struct CmdRealisation : virtual NixMultiCommand
+struct CmdRealisation : NixMultiCommand
 {
-    CmdRealisation() : MultiCommand(RegisterCommand::getCommandsFor({"realisation"}))
+    CmdRealisation() : NixMultiCommand("realisation", RegisterCommand::getCommandsFor({"realisation"}))
     { }
 
     std::string description() override
@@ -16,13 +16,6 @@ struct CmdRealisation : virtual NixMultiCommand
     }
 
     Category category() override { return catUtility; }
-
-    void run() override
-    {
-        if (!command)
-            throw UsageError("'nix realisation' requires a sub-command.");
-        command->second->run();
-    }
 };
 
 static auto rCmdRealisation = registerCommand<CmdRealisation>("realisation");
diff --git a/src/nix/registry.cc b/src/nix/registry.cc
index f509ccae8..0346ec1e0 100644
--- a/src/nix/registry.cc
+++ b/src/nix/registry.cc
@@ -196,10 +196,12 @@ struct CmdRegistryPin : RegistryCommand, EvalCommand
     }
 };
 
-struct CmdRegistry : virtual NixMultiCommand
+struct CmdRegistry : NixMultiCommand
 {
     CmdRegistry()
-        : MultiCommand({
+        : NixMultiCommand(
+            "registry",
+            {
                 {"list", []() { return make_ref<CmdRegistryList>(); }},
                 {"add", []() { return make_ref<CmdRegistryAdd>(); }},
                 {"remove", []() { return make_ref<CmdRegistryRemove>(); }},
@@ -221,14 +223,6 @@ struct CmdRegistry : virtual NixMultiCommand
     }
 
     Category category() override { return catSecondary; }
-
-    void run() override
-    {
-        experimentalFeatureSettings.require(Xp::Flakes);
-        if (!command)
-            throw UsageError("'nix registry' requires a sub-command.");
-        command->second->run();
-    }
 };
 
 static auto rCmdRegistry = registerCommand<CmdRegistry>("registry");
diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc
index 39555c9ea..a57a407e6 100644
--- a/src/nix/sigs.cc
+++ b/src/nix/sigs.cc
@@ -205,7 +205,9 @@ struct CmdKeyConvertSecretToPublic : Command
 struct CmdKey : NixMultiCommand
 {
     CmdKey()
-        : MultiCommand({
+        : NixMultiCommand(
+            "key",
+            {
                 {"generate-secret", []() { return make_ref<CmdKeyGenerateSecret>(); }},
                 {"convert-secret-to-public", []() { return make_ref<CmdKeyConvertSecretToPublic>(); }},
             })
@@ -218,13 +220,6 @@ struct CmdKey : NixMultiCommand
     }
 
     Category category() override { return catUtility; }
-
-    void run() override
-    {
-        if (!command)
-            throw UsageError("'nix key' requires a sub-command.");
-        command->second->run();
-    }
 };
 
 static auto rCmdKey = registerCommand<CmdKey>("key");
diff --git a/src/nix/store.cc b/src/nix/store.cc
index 2879e03b3..79b41e096 100644
--- a/src/nix/store.cc
+++ b/src/nix/store.cc
@@ -2,9 +2,9 @@
 
 using namespace nix;
 
-struct CmdStore : virtual NixMultiCommand
+struct CmdStore : NixMultiCommand
 {
-    CmdStore() : MultiCommand(RegisterCommand::getCommandsFor({"store"}))
+    CmdStore() : NixMultiCommand("store", RegisterCommand::getCommandsFor({"store"}))
     { }
 
     std::string description() override
@@ -13,13 +13,6 @@ struct CmdStore : virtual NixMultiCommand
     }
 
     Category category() override { return catUtility; }
-
-    void run() override
-    {
-        if (!command)
-            throw UsageError("'nix store' requires a sub-command.");
-        command->second->run();
-    }
 };
 
 static auto rCmdStore = registerCommand<CmdStore>("store");

From e7abf60a0c8db19927e4fb195789b698c84e8d5a Mon Sep 17 00:00:00 2001
From: Peter Kolloch <info@eigenvalue.net>
Date: Sat, 25 Nov 2023 17:33:44 +0100
Subject: [PATCH 262/294] hash.cc/hash.h: Minor C++ improvements

---
 src/libutil/hash.hh | 14 +++++++-------
 src/nix/hash.cc     |  2 +-
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh
index 6ade6555c..0e5c91b79 100644
--- a/src/libutil/hash.hh
+++ b/src/libutil/hash.hh
@@ -52,7 +52,7 @@ struct Hash
     /**
      * Create a zero-filled hash object.
      */
-    Hash(HashType type);
+    explicit Hash(HashType type);
 
     /**
      * Parse the hash from a string representation in the format
@@ -103,7 +103,7 @@ public:
     /**
      * Returns the length of a base-16 representation of this hash.
      */
-    size_t base16Len() const
+    [[nodiscard]] size_t base16Len() const
     {
         return hashSize * 2;
     }
@@ -111,7 +111,7 @@ public:
     /**
      * Returns the length of a base-32 representation of this hash.
      */
-    size_t base32Len() const
+    [[nodiscard]] size_t base32Len() const
     {
         return (hashSize * 8 - 1) / 5 + 1;
     }
@@ -119,7 +119,7 @@ public:
     /**
      * Returns the length of a base-64 representation of this hash.
      */
-    size_t base64Len() const
+    [[nodiscard]] size_t base64Len() const
     {
         return ((4 * hashSize / 3) + 3) & ~3;
     }
@@ -129,14 +129,14 @@ public:
      * or base-64. By default, this is prefixed by the hash type
      * (e.g. "sha256:").
      */
-    std::string to_string(HashFormat hashFormat, bool includeType) const;
+    [[nodiscard]] std::string to_string(HashFormat hashFormat, bool includeType) const;
 
-    std::string gitRev() const
+    [[nodiscard]] std::string gitRev() const
     {
         return to_string(HashFormat::Base16, false);
     }
 
-    std::string gitShortRev() const
+    [[nodiscard]] std::string gitShortRev() const
     {
         return std::string(to_string(HashFormat::Base16, false), 0, 7);
     }
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index ededf6ef2..cac65006b 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -17,7 +17,7 @@ struct CmdHashBase : Command
     std::vector<std::string> paths;
     std::optional<std::string> modulus;
 
-    CmdHashBase(FileIngestionMethod mode) : mode(mode)
+    explicit CmdHashBase(FileIngestionMethod mode) : mode(mode)
     {
         addFlag({
             .longName = "sri",

From 156ea78d7402368e3816855800eb6e0ed33a1ecc Mon Sep 17 00:00:00 2001
From: Peter Kolloch <info@eigenvalue.net>
Date: Sat, 25 Nov 2023 17:34:16 +0100
Subject: [PATCH 263/294] CmdHashBase: doc comment

---
 src/nix/hash.cc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index cac65006b..dfef44221 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -8,6 +8,11 @@
 
 using namespace nix;
 
+/**
+ * Base for `nix hash file` (deprecated), `nix hash path` and `nix-hash` (legacy).
+ *
+ * Deprecation Issue: https://github.com/NixOS/nix/issues/8876
+ */
 struct CmdHashBase : Command
 {
     FileIngestionMethod mode;

From 6bbd900d4f9983f74dcd9a0f85ab899331f661c7 Mon Sep 17 00:00:00 2001
From: Peter Kolloch <info@eigenvalue.net>
Date: Sat, 25 Nov 2023 17:35:24 +0100
Subject: [PATCH 264/294] nix hash convert: added

This deviated from the proposal! See comments on the issue.

https://github.com/NixOS/nix/issues/8876
---
 src/nix/hash.cc          | 63 ++++++++++++++++++++++++++++++++++++++++
 tests/functional/hash.sh | 19 +++++++++++-
 2 files changed, 81 insertions(+), 1 deletion(-)

diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index dfef44221..2b32ac03c 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -132,12 +132,75 @@ struct CmdToBase : Command
     }
 };
 
+/**
+ * `nix hash convert`
+ */
+struct CmdHashConvert : Command
+{
+    std::optional<HashFormat> from;
+    HashFormat to;
+    std::optional<HashType> type;
+    std::vector<std::string> hashStrings;
+
+    CmdHashConvert(): to(HashFormat::SRI) {
+        addFlag({
+            .longName = "from",
+            // TODO: List format choices. Maybe introduce a constant?
+            .description = "The format of the input hash.",
+            .labels = {"hash format"},
+            .handler = {[this](std::string str) {
+                from = parseHashFormat(str);
+            }},
+        });
+        addFlag({
+            .longName = "to",
+            // TODO: List format choices. Maybe introduce a constant?
+            .description = "The format of the output hash.",
+            .labels = {"hash format"},
+            .handler = {[this](std::string str) {
+                to = parseHashFormat(str);
+            }},
+        });
+        addFlag({
+            .longName = "type",
+            .description = "Specify the type if it can't be auto-detected.",
+            .labels = {"hash type"},
+            .handler = {[this](std::string str) {
+                type = parseHashType(str);
+            }},
+        });
+        expectArgs({
+           .label = "hashes",
+           .handler = {&hashStrings},
+        });
+    }
+
+    std::string description() override
+    {
+        return "convert between different hash formats, e.g. base16 and sri.";
+    }
+
+    Category category() override { return catUtility; }
+
+    void run() override {
+        for (const auto& s: hashStrings) {
+            Hash h = Hash::parseAny(s, type);
+            if (from && h.to_string(*from, from == HashFormat::SRI) != s) {
+                auto from_as_string = printHashFormat(*from);
+                throw BadHash("input hash '%s' does not have the expected format '--from %s'", s, from_as_string);
+            }
+            logger->cout(h.to_string(to, to == HashFormat::SRI));
+        }
+    }
+};
+
 struct CmdHash : NixMultiCommand
 {
     CmdHash()
         : NixMultiCommand(
             "hash",
             {
+                {"convert", []() { return make_ref<CmdHashConvert>();}},
                 {"file", []() { return make_ref<CmdHashBase>(FileIngestionMethod::Flat);; }},
                 {"path", []() { return make_ref<CmdHashBase>(FileIngestionMethod::Recursive); }},
                 {"to-base16", []() { return make_ref<CmdToBase>(HashFormat::Base16); }},
diff --git a/tests/functional/hash.sh b/tests/functional/hash.sh
index 34c1bb38a..d66b27a26 100644
--- a/tests/functional/hash.sh
+++ b/tests/functional/hash.sh
@@ -81,24 +81,41 @@ rm $TEST_ROOT/hash-path/hello
 ln -s x $TEST_ROOT/hash-path/hello
 try2 md5 "f78b733a68f5edbdf9413899339eaa4a"
 
-# Conversion.
+# Conversion with `nix hash` `nix-hash` and `nix hash convert`
 try3() {
+    # $1 = hash type
+    # $2 = expected hash in base16
+    # $3 = expected hash in base32
+    # $4 = expected hash in base64
+    h64=$(nix hash convert --type "$1" --to base64 "$2")
+    [ "$h64" = "$4" ]
     h64=$(nix-hash --type "$1" --to-base64 "$2")
     [ "$h64" = "$4" ]
+    # Deprecated experiment
     h64=$(nix hash to-base64 --type "$1" "$2")
     [ "$h64" = "$4" ]
+
+    sri=$(nix hash convert --type "$1" --to sri "$2")
+    [ "$sri" = "$1-$4" ]
     sri=$(nix-hash --type "$1" --to-sri "$2")
     [ "$sri" = "$1-$4" ]
     sri=$(nix hash to-sri --type "$1" "$2")
     [ "$sri" = "$1-$4" ]
+    h32=$(nix hash convert --type "$1" --to base32 "$2")
+    [ "$h32" = "$3" ]
     h32=$(nix-hash --type "$1" --to-base32 "$2")
     [ "$h32" = "$3" ]
     h32=$(nix hash to-base32 --type "$1" "$2")
     [ "$h32" = "$3" ]
     h16=$(nix-hash --type "$1" --to-base16 "$h32")
     [ "$h16" = "$2" ]
+
+    h16=$(nix hash convert --type "$1" --to base16 "$h64")
+    [ "$h16" = "$2" ]
     h16=$(nix hash to-base16 --type "$1" "$h64")
     [ "$h16" = "$2" ]
+    h16=$(nix hash convert --to base16 "$sri")
+    [ "$h16" = "$2" ]
     h16=$(nix hash to-base16 "$sri")
     [ "$h16" = "$2" ]
 }

From 0c2d5f7673ae0196b660c39b59941755103c23d0 Mon Sep 17 00:00:00 2001
From: Peter Kolloch <info@eigenvalue.net>
Date: Tue, 28 Nov 2023 11:42:52 +0100
Subject: [PATCH 265/294] nix hash convert: s/--type/--algo/ + more functional
 tests

https://github.com/NixOS/nix/issues/8876
---
 src/libutil/hash.hh      |  1 -
 src/nix/hash.cc          |  8 ++---
 tests/functional/hash.sh | 72 +++++++++++++++++++++++++++++++++++++---
 3 files changed, 71 insertions(+), 10 deletions(-)

diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh
index 0e5c91b79..820154e7a 100644
--- a/src/libutil/hash.hh
+++ b/src/libutil/hash.hh
@@ -40,7 +40,6 @@ enum struct HashFormat : int {
     SRI
 };
 
-
 struct Hash
 {
     constexpr static size_t maxHashSize = 64;
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index 2b32ac03c..62f96ef1d 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -162,9 +162,9 @@ struct CmdHashConvert : Command
             }},
         });
         addFlag({
-            .longName = "type",
-            .description = "Specify the type if it can't be auto-detected.",
-            .labels = {"hash type"},
+            .longName = "algo",
+            .description = "Specify the algorithm if it can't be auto-detected.",
+            .labels = {"hash algorithm"},
             .handler = {[this](std::string str) {
                 type = parseHashType(str);
             }},
@@ -177,7 +177,7 @@ struct CmdHashConvert : Command
 
     std::string description() override
     {
-        return "convert between different hash formats, e.g. base16 and sri.";
+        return "convert between different hash formats, e.g. base16, nix32, base64 and sri.";
     }
 
     Category category() override { return catUtility; }
diff --git a/tests/functional/hash.sh b/tests/functional/hash.sh
index d66b27a26..031e33adf 100644
--- a/tests/functional/hash.sh
+++ b/tests/functional/hash.sh
@@ -83,11 +83,11 @@ try2 md5 "f78b733a68f5edbdf9413899339eaa4a"
 
 # Conversion with `nix hash` `nix-hash` and `nix hash convert`
 try3() {
-    # $1 = hash type
+    # $1 = hash algo
     # $2 = expected hash in base16
     # $3 = expected hash in base32
     # $4 = expected hash in base64
-    h64=$(nix hash convert --type "$1" --to base64 "$2")
+    h64=$(nix hash convert --algo "$1" --to base64 "$2")
     [ "$h64" = "$4" ]
     h64=$(nix-hash --type "$1" --to-base64 "$2")
     [ "$h64" = "$4" ]
@@ -95,13 +95,13 @@ try3() {
     h64=$(nix hash to-base64 --type "$1" "$2")
     [ "$h64" = "$4" ]
 
-    sri=$(nix hash convert --type "$1" --to sri "$2")
+    sri=$(nix hash convert --algo "$1" --to sri "$2")
     [ "$sri" = "$1-$4" ]
     sri=$(nix-hash --type "$1" --to-sri "$2")
     [ "$sri" = "$1-$4" ]
     sri=$(nix hash to-sri --type "$1" "$2")
     [ "$sri" = "$1-$4" ]
-    h32=$(nix hash convert --type "$1" --to base32 "$2")
+    h32=$(nix hash convert --algo "$1" --to base32 "$2")
     [ "$h32" = "$3" ]
     h32=$(nix-hash --type "$1" --to-base32 "$2")
     [ "$h32" = "$3" ]
@@ -110,7 +110,7 @@ try3() {
     h16=$(nix-hash --type "$1" --to-base16 "$h32")
     [ "$h16" = "$2" ]
 
-    h16=$(nix hash convert --type "$1" --to base16 "$h64")
+    h16=$(nix hash convert --algo "$1" --to base16 "$h64")
     [ "$h16" = "$2" ]
     h16=$(nix hash to-base16 --type "$1" "$h64")
     [ "$h16" = "$2" ]
@@ -118,7 +118,69 @@ try3() {
     [ "$h16" = "$2" ]
     h16=$(nix hash to-base16 "$sri")
     [ "$h16" = "$2" ]
+
+    #
+    # Converting from SRI
+    #
+
+    # Input hash algo auto-detected from SRI and output defaults to SRI as well.
+    sri=$(nix hash convert "$1-$4")
+    [ "$sri" = "$1-$4" ]
+
+    sri=$(nix hash convert --from sri "$1-$4")
+    [ "$sri" = "$1-$4" ]
+
+    sri=$(nix hash convert --to sri "$1-$4")
+    [ "$sri" = "$1-$4" ]
+
+    sri=$(nix hash convert --from sri --to sri "$1-$4")
+    [ "$sri" = "$1-$4" ]
+
+    sri=$(nix hash convert --to base64 "$1-$4")
+    [ "$sri" = "$4" ]
+
+    #
+    # Auto-detecting the input from algo and length.
+    #
+
+    sri=$(nix hash convert --algo "$1" "$2")
+    [ "$sri" = "$1-$4" ]
+    sri=$(nix hash convert --algo "$1" "$3")
+    [ "$sri" = "$1-$4" ]
+    sri=$(nix hash convert --algo "$1" "$4")
+    [ "$sri" = "$1-$4" ]
+
+    sri=$(nix hash convert --algo "$1" "$2")
+    [ "$sri" = "$1-$4" ]
+    sri=$(nix hash convert --algo "$1" "$3")
+    [ "$sri" = "$1-$4" ]
+    sri=$(nix hash convert --algo "$1" "$4")
+    [ "$sri" = "$1-$4" ]
+
+    #
+    # Asserting input format succeeds.
+    #
+
+    sri=$(nix hash convert --algo "$1" --from base16 "$2")
+    [ "$sri" = "$1-$4" ]
+    sri=$(nix hash convert --algo "$1" --from base32 "$3")
+    [ "$sri" = "$1-$4" ]
+    sri=$(nix hash convert --algo "$1" --from base64 "$4")
+    [ "$sri" = "$1-$4" ]
+
+    #
+    # Asserting input format fails.
+    #
+
+    fail=$(nix hash convert --algo "$1" --from base32 "$2" 2>&1 || echo "exit: $?")
+    [[ "$fail" == "error: input hash"*"exit: 1" ]]
+    fail=$(nix hash convert --algo "$1" --from base16 "$3" 2>&1 || echo "exit: $?")
+    [[ "$fail" == "error: input hash"*"exit: 1" ]]
+    fail=$(nix hash convert --algo "$1" --from base32 "$4" 2>&1 || echo "exit: $?")
+    [[ "$fail" == "error: input hash"*"exit: 1" ]]
+
 }
+
 try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0" "gA1Zz808BekAy04hS+SPa4hqCN8="
 try3 sha256 "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0="
 try3 sha512 "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445" "12k9jiq29iyqm03swfsgiw5mlqs173qazm3n7daz43infy12pyrcdf30fkk3qwv4yl2ick8yipc2mqnlh48xsvvxl60lbx8vp38yji0" "IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ=="

From 5334c9c792a208db4d3824e88019a626ded1b65d Mon Sep 17 00:00:00 2001
From: Peter Kolloch <info@eigenvalue.net>
Date: Tue, 28 Nov 2023 14:20:27 +0100
Subject: [PATCH 266/294] HashType: Rename to HashAlgorithm

To be consistent with CLI, nix API
and many other references.

As part of this, we also converted it to a scoped enum.

https://github.com/NixOS/nix/issues/8876
---
 perl/lib/Nix/Store.xs                       |  12 +-
 src/libexpr/flake/flake.cc                  |   2 +-
 src/libexpr/primops.cc                      |  36 ++---
 src/libexpr/primops/fetchMercurial.cc       |   4 +-
 src/libexpr/primops/fetchTree.cc            |   8 +-
 src/libfetchers/fetchers.cc                 |   6 +-
 src/libfetchers/git-utils.cc                |   4 +-
 src/libfetchers/git.cc                      |   6 +-
 src/libfetchers/github.cc                   |  10 +-
 src/libfetchers/indirect.cc                 |   4 +-
 src/libfetchers/input-accessor.cc           |   4 +-
 src/libfetchers/mercurial.cc                |  10 +-
 src/libfetchers/tarball.cc                  |   6 +-
 src/libstore/binary-cache-store.cc          |  24 +--
 src/libstore/binary-cache-store.hh          |  16 +-
 src/libstore/build/local-derivation-goal.cc |  42 +++---
 src/libstore/build/worker.cc                |   4 +-
 src/libstore/builtins/fetchurl.cc           |   4 +-
 src/libstore/content-address.cc             |  28 ++--
 src/libstore/content-address.hh             |   4 +-
 src/libstore/daemon.cc                      |  16 +-
 src/libstore/derivations.cc                 |  64 ++++----
 src/libstore/derivations.hh                 |   8 +-
 src/libstore/downstream-placeholder.cc      |   4 +-
 src/libstore/export-import.cc               |   6 +-
 src/libstore/gc.cc                          |   2 +-
 src/libstore/legacy-ssh-store.cc            |  14 +-
 src/libstore/local-store.cc                 |  30 ++--
 src/libstore/local-store.hh                 |   6 +-
 src/libstore/make-content-addressed.cc      |   4 +-
 src/libstore/nar-info.cc                    |   4 +-
 src/libstore/optimise-store.cc              |   4 +-
 src/libstore/path-references.cc             |   2 +-
 src/libstore/path.cc                        |   2 +-
 src/libstore/remote-store.cc                |  28 ++--
 src/libstore/remote-store.hh                |  14 +-
 src/libstore/store-api.cc                   |  46 +++---
 src/libstore/store-api.hh                   |  22 +--
 src/libstore/store-dir-config.hh            |   2 +-
 src/libstore/worker-protocol.cc             |   2 +-
 src/libutil/args.cc                         |  14 +-
 src/libutil/args.hh                         |   6 +-
 src/libutil/git.cc                          |   8 +-
 src/libutil/git.hh                          |   6 +-
 src/libutil/hash.cc                         | 158 ++++++++++----------
 src/libutil/hash.hh                         |  38 ++---
 src/libutil/references.cc                   |   4 +-
 src/libutil/references.hh                   |   2 +-
 src/libutil/source-accessor.cc              |   8 +-
 src/libutil/source-accessor.hh              |   6 +-
 src/nix-store/nix-store.cc                  |  12 +-
 src/nix/add-to-store.cc                     |   4 +-
 src/nix/hash.cc                             |  24 +--
 src/nix/prefetch.cc                         |  34 ++---
 src/nix/profile.cc                          |   2 +-
 src/nix/verify.cc                           |   2 +-
 tests/unit/libstore/common-protocol.cc      |   8 +-
 tests/unit/libstore/derivation.cc           |   4 +-
 tests/unit/libstore/nar-info.cc             |   2 +-
 tests/unit/libstore/path-info.cc            |   2 +-
 tests/unit/libstore/serve-protocol.cc       |   8 +-
 tests/unit/libstore/worker-protocol.cc      |  10 +-
 tests/unit/libutil/git.cc                   |   8 +-
 tests/unit/libutil/hash.cc                  |  16 +-
 64 files changed, 450 insertions(+), 450 deletions(-)

diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index 40257ed74..50148141b 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -205,7 +205,7 @@ void importPaths(int fd, int dontCheckSigs)
 SV * hashPath(char * algo, int base32, char * path)
     PPCODE:
         try {
-            Hash h = hashPath(parseHashType(algo), path).first;
+            Hash h = hashPath(parseHashAlgo(algo), path).first;
             auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
@@ -216,7 +216,7 @@ SV * hashPath(char * algo, int base32, char * path)
 SV * hashFile(char * algo, int base32, char * path)
     PPCODE:
         try {
-            Hash h = hashFile(parseHashType(algo), path);
+            Hash h = hashFile(parseHashAlgo(algo), path);
             auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
@@ -227,7 +227,7 @@ SV * hashFile(char * algo, int base32, char * path)
 SV * hashString(char * algo, int base32, char * s)
     PPCODE:
         try {
-            Hash h = hashString(parseHashType(algo), s);
+            Hash h = hashString(parseHashAlgo(algo), s);
             auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
@@ -238,7 +238,7 @@ SV * hashString(char * algo, int base32, char * s)
 SV * convertHash(char * algo, char * s, int toBase32)
     PPCODE:
         try {
-            auto h = Hash::parseAny(s, parseHashType(algo));
+            auto h = Hash::parseAny(s, parseHashAlgo(algo));
             auto s = h.to_string(toBase32 ? HashFormat::Base32 : HashFormat::Base16, false);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
@@ -281,7 +281,7 @@ SV * addToStore(char * srcPath, int recursive, char * algo)
     PPCODE:
         try {
             auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
-            auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, method, parseHashType(algo));
+            auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, method, parseHashAlgo(algo));
             XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -291,7 +291,7 @@ SV * addToStore(char * srcPath, int recursive, char * algo)
 SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name)
     PPCODE:
         try {
-            auto h = Hash::parseAny(hash, parseHashType(algo));
+            auto h = Hash::parseAny(hash, parseHashAlgo(algo));
             auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
             auto path = store()->makeFixedOutputPath(name, FixedOutputInfo {
                 .method = method,
diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index 8f8fc64f0..fee58792b 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -904,7 +904,7 @@ Fingerprint LockedFlake::getFingerprint() const
     // FIXME: as an optimization, if the flake contains a lock file
     // and we haven't changed it, then it's sufficient to use
     // flake.sourceInfo.storePath for the fingerprint.
-    return hashString(htSHA256,
+    return hashString(HashAlgorithm::SHA256,
         fmt("%s;%s;%d;%d;%s",
             flake.storePath.to_string(),
             flake.lockedRef.subdir,
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index c2499bdae..7831f3803 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1317,7 +1317,7 @@ drvName, Bindings * attrs, Value & v)
                 .errPos = state.positions[noPos]
             }));
 
-        auto h = newHashAllowEmpty(*outputHash, parseHashTypeOpt(outputHashAlgo));
+        auto h = newHashAllowEmpty(*outputHash, parseHashAlgoOpt(outputHashAlgo));
 
         auto method = ingestionMethod.value_or(FileIngestionMethod::Flat);
 
@@ -1339,7 +1339,7 @@ drvName, Bindings * attrs, Value & v)
                 .errPos = state.positions[noPos]
             });
 
-        auto ht = parseHashTypeOpt(outputHashAlgo).value_or(htSHA256);
+        auto ht = parseHashAlgoOpt(outputHashAlgo).value_or(HashAlgorithm::SHA256);
         auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive);
 
         for (auto & i : outputs) {
@@ -1348,13 +1348,13 @@ drvName, Bindings * attrs, Value & v)
                 drv.outputs.insert_or_assign(i,
                     DerivationOutput::Impure {
                         .method = method,
-                        .hashType = ht,
+                        .hashAlgo = ht,
                     });
             else
                 drv.outputs.insert_or_assign(i,
                     DerivationOutput::CAFloating {
                         .method = method,
-                        .hashType = ht,
+                        .hashAlgo = ht,
                     });
         }
     }
@@ -1754,17 +1754,17 @@ static RegisterPrimOp primop_findFile(PrimOp {
 /* Return the cryptographic hash of a file in base-16. */
 static void prim_hashFile(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    auto type = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashFile");
-    std::optional<HashType> ht = parseHashType(type);
-    if (!ht)
+    auto algo = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashFile");
+    std::optional<HashAlgorithm> ha = parseHashAlgo(algo);
+    if (!ha)
         state.debugThrowLastTrace(Error({
-            .msg = hintfmt("unknown hash type '%1%'", type),
+            .msg = hintfmt("unknown hash algo '%1%'", algo),
             .errPos = state.positions[pos]
         }));
 
     auto path = realisePath(state, pos, *args[1]);
 
-    v.mkString(hashString(*ht, path.readFile()).to_string(HashFormat::Base16, false));
+    v.mkString(hashString(*ha, path.readFile()).to_string(HashFormat::Base16, false));
 }
 
 static RegisterPrimOp primop_hashFile({
@@ -2341,7 +2341,7 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value
         else if (n == "recursive")
             method = FileIngestionMethod { state.forceBool(*attr.value, attr.pos, "while evaluating the `recursive` attribute passed to builtins.path") };
         else if (n == "sha256")
-            expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), htSHA256);
+            expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), HashAlgorithm::SHA256);
         else
             state.debugThrowLastTrace(EvalError({
                 .msg = hintfmt("unsupported argument '%1%' to 'addPath'", state.symbols[attr.name]),
@@ -3766,18 +3766,18 @@ static RegisterPrimOp primop_stringLength({
 /* Return the cryptographic hash of a string in base-16. */
 static void prim_hashString(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    auto type = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashString");
-    std::optional<HashType> ht = parseHashType(type);
-    if (!ht)
+    auto algo = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashString");
+    std::optional<HashAlgorithm> ha = parseHashAlgo(algo);
+    if (!ha)
         state.debugThrowLastTrace(Error({
-            .msg = hintfmt("unknown hash type '%1%'", type),
+            .msg = hintfmt("unknown hash algo '%1%'", algo),
             .errPos = state.positions[pos]
         }));
 
     NixStringContext context; // discarded
     auto s = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString");
 
-    v.mkString(hashString(*ht, s).to_string(HashFormat::Base16, false));
+    v.mkString(hashString(*ha, s).to_string(HashFormat::Base16, false));
 }
 
 static RegisterPrimOp primop_hashString({
@@ -3800,15 +3800,15 @@ static void prim_convertHash(EvalState & state, const PosIdx pos, Value * * args
     auto hash = state.forceStringNoCtx(*iteratorHash->value, pos, "while evaluating the attribute 'hash'");
 
     Bindings::iterator iteratorHashAlgo = inputAttrs->find(state.symbols.create("hashAlgo"));
-    std::optional<HashType> ht = std::nullopt;
+    std::optional<HashAlgorithm> ha = std::nullopt;
     if (iteratorHashAlgo != inputAttrs->end()) {
-        ht = parseHashType(state.forceStringNoCtx(*iteratorHashAlgo->value, pos, "while evaluating the attribute 'hashAlgo'"));
+        ha = parseHashAlgo(state.forceStringNoCtx(*iteratorHashAlgo->value, pos, "while evaluating the attribute 'hashAlgo'"));
     }
 
     Bindings::iterator iteratorToHashFormat = getAttr(state, state.symbols.create("toHashFormat"), args[0]->attrs, "while locating the attribute 'toHashFormat'");
     HashFormat hf = parseHashFormat(state.forceStringNoCtx(*iteratorToHashFormat->value, pos, "while evaluating the attribute 'toHashFormat'"));
 
-    v.mkString(Hash::parseAny(hash, ht).to_string(hf, hf == HashFormat::SRI));
+    v.mkString(Hash::parseAny(hash, ha).to_string(hf, hf == HashFormat::SRI));
 }
 
 static RegisterPrimOp primop_convertHash({
diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index e76ce455d..58fe6f173 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -31,7 +31,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a
                 // be both a revision or a branch/tag name.
                 auto value = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `rev` attribute passed to builtins.fetchMercurial");
                 if (std::regex_match(value.begin(), value.end(), revRegex))
-                    rev = Hash::parseAny(value, htSHA1);
+                    rev = Hash::parseAny(value, HashAlgorithm::SHA1);
                 else
                     ref = value;
             }
@@ -79,7 +79,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a
         attrs2.alloc("branch").mkString(*input2.getRef());
     // Backward compatibility: set 'rev' to
     // 0000000000000000000000000000000000000000 for a dirty tree.
-    auto rev2 = input2.getRev().value_or(Hash(htSHA1));
+    auto rev2 = input2.getRev().value_or(Hash(HashAlgorithm::SHA1));
     attrs2.alloc("rev").mkString(rev2.gitRev());
     attrs2.alloc("shortRev").mkString(rev2.gitRev().substr(0, 12));
     if (auto revCount = input2.getRevCount())
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 383ec7c58..ef80c634f 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -46,7 +46,7 @@ void emitTreeAttrs(
             attrs.alloc("shortRev").mkString(rev->gitShortRev());
         } else if (emptyRevFallback) {
             // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev
-            auto emptyHash = Hash(htSHA1);
+            auto emptyHash = Hash(HashAlgorithm::SHA1);
             attrs.alloc("rev").mkString(emptyHash.gitRev());
             attrs.alloc("shortRev").mkString(emptyHash.gitShortRev());
         }
@@ -246,7 +246,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
             if (n == "url")
                 url = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the url we should fetch");
             else if (n == "sha256")
-                expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the sha256 of the content we should fetch"), htSHA256);
+                expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the sha256 of the content we should fetch"), HashAlgorithm::SHA256);
             else if (n == "name")
                 name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the name of the content we should fetch");
             else
@@ -276,7 +276,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
         state.debugThrowLastTrace(EvalError("in pure evaluation mode, '%s' requires a 'sha256' argument", who));
 
     // early exit if pinned and already in the store
-    if (expectedHash && expectedHash->type == htSHA256) {
+    if (expectedHash && expectedHash->algo == HashAlgorithm::SHA256) {
         auto expectedPath = state.store->makeFixedOutputPath(
             name,
             FixedOutputInfo {
@@ -301,7 +301,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
     if (expectedHash) {
         auto hash = unpack
             ? state.store->queryPathInfo(storePath)->narHash
-            : hashFile(htSHA256, state.store->toRealPath(storePath));
+            : hashFile(HashAlgorithm::SHA256, state.store->toRealPath(storePath));
         if (hash != *expectedHash)
             state.debugThrowLastTrace(EvalError((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n  specified: %s\n  got:       %s",
                 *url, expectedHash->to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true)));
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index 60208619e..573341a3d 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -289,8 +289,8 @@ std::string Input::getType() const
 std::optional<Hash> Input::getNarHash() const
 {
     if (auto s = maybeGetStrAttr(attrs, "narHash")) {
-        auto hash = s->empty() ? Hash(htSHA256) : Hash::parseSRI(*s);
-        if (hash.type != htSHA256)
+        auto hash = s->empty() ? Hash(HashAlgorithm::SHA256) : Hash::parseSRI(*s);
+        if (hash.algo != HashAlgorithm::SHA256)
             throw UsageError("narHash must use SHA-256");
         return hash;
     }
@@ -314,7 +314,7 @@ std::optional<Hash> Input::getRev() const
         } catch (BadHash &e) {
             // Default to sha1 for backwards compatibility with existing
             // usages (e.g. `builtins.fetchTree` calls or flake inputs).
-            hash = Hash::parseAny(*s, htSHA1);
+            hash = Hash::parseAny(*s, HashAlgorithm::SHA1);
         }
     }
 
diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index 19eae0e1d..9356e5817 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -91,7 +91,7 @@ Hash toHash(const git_oid & oid)
     #ifdef GIT_EXPERIMENTAL_SHA256
     assert(oid.type == GIT_OID_SHA1);
     #endif
-    Hash hash(htSHA1);
+    Hash hash(HashAlgorithm::SHA1);
     memcpy(hash.hash, oid.id, hash.hashSize);
     return hash;
 }
@@ -439,7 +439,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         std::string re = R"(Good "git" signature for \* with .* key SHA256:[)";
         for (const fetchers::PublicKey & k : publicKeys){
             // Calculate sha256 fingerprint from public key and escape the regex symbol '+' to match the key literally
-            auto fingerprint = trim(hashString(htSHA256, base64Decode(k.key)).to_string(nix::HashFormat::Base64, false), "=");
+            auto fingerprint = trim(hashString(HashAlgorithm::SHA256, base64Decode(k.key)).to_string(nix::HashFormat::Base64, false), "=");
             auto escaped_fingerprint = std::regex_replace(fingerprint, std::regex("\\+"), "\\+" );
             re += "(" + escaped_fingerprint + ")";
         }
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 8cd74057c..a89acc1c0 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -52,7 +52,7 @@ bool touchCacheFile(const Path & path, time_t touch_time)
 Path getCachePath(std::string_view key)
 {
     return getCacheDir() + "/nix/gitv3/" +
-        hashString(htSHA256, key).to_string(HashFormat::Base32, false);
+        hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Base32, false);
 }
 
 // Returns the name of the HEAD branch.
@@ -369,7 +369,7 @@ struct GitInputScheme : InputScheme
     {
         auto checkHashType = [&](const std::optional<Hash> & hash)
         {
-            if (hash.has_value() && !(hash->type == htSHA1 || hash->type == htSHA256))
+            if (hash.has_value() && !(hash->algo == HashAlgorithm::SHA1 || hash->algo == HashAlgorithm::SHA256))
                 throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(HashFormat::Base16, true));
         };
 
@@ -559,7 +559,7 @@ struct GitInputScheme : InputScheme
                         repoInfo.url
                         );
             } else
-                input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), htSHA1).gitRev());
+                input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), HashAlgorithm::SHA1).gitRev());
 
             // cache dir lock is removed at scope end; we will only use read-only operations on specific revisions in the remainder
         }
diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 661ad4884..70acb9354 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -42,7 +42,7 @@ struct GitArchiveInputScheme : InputScheme
         auto size = path.size();
         if (size == 3) {
             if (std::regex_match(path[2], revRegex))
-                rev = Hash::parseAny(path[2], htSHA1);
+                rev = Hash::parseAny(path[2], HashAlgorithm::SHA1);
             else if (std::regex_match(path[2], refRegex))
                 ref = path[2];
             else
@@ -68,7 +68,7 @@ struct GitArchiveInputScheme : InputScheme
             if (name == "rev") {
                 if (rev)
                     throw BadURL("URL '%s' contains multiple commit hashes", url.url);
-                rev = Hash::parseAny(value, htSHA1);
+                rev = Hash::parseAny(value, HashAlgorithm::SHA1);
             }
             else if (name == "ref") {
                 if (!std::regex_match(value, refRegex))
@@ -284,7 +284,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
             readFile(
                 store->toRealPath(
                     downloadFile(store, url, "source", false, headers).storePath)));
-        auto rev = Hash::parseAny(std::string { json["sha"] }, htSHA1);
+        auto rev = Hash::parseAny(std::string { json["sha"] }, HashAlgorithm::SHA1);
         debug("HEAD revision for '%s' is %s", url, rev.gitRev());
         return rev;
     }
@@ -356,7 +356,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
             readFile(
                 store->toRealPath(
                     downloadFile(store, url, "source", false, headers).storePath)));
-        auto rev = Hash::parseAny(std::string(json[0]["id"]), htSHA1);
+        auto rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1);
         debug("HEAD revision for '%s' is %s", url, rev.gitRev());
         return rev;
     }
@@ -448,7 +448,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
         if(!id)
             throw BadURL("in '%d', couldn't find ref '%d'", input.to_string(), ref);
 
-        auto rev = Hash::parseAny(*id, htSHA1);
+        auto rev = Hash::parseAny(*id, HashAlgorithm::SHA1);
         debug("HEAD revision for '%s' is %s", fmt("%s/%s", base_url, ref), rev.gitRev());
         return rev;
     }
diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc
index 8e30284c6..002c0c292 100644
--- a/src/libfetchers/indirect.cc
+++ b/src/libfetchers/indirect.cc
@@ -20,7 +20,7 @@ struct IndirectInputScheme : InputScheme
         if (path.size() == 1) {
         } else if (path.size() == 2) {
             if (std::regex_match(path[1], revRegex))
-                rev = Hash::parseAny(path[1], htSHA1);
+                rev = Hash::parseAny(path[1], HashAlgorithm::SHA1);
             else if (std::regex_match(path[1], refRegex))
                 ref = path[1];
             else
@@ -31,7 +31,7 @@ struct IndirectInputScheme : InputScheme
             ref = path[1];
             if (!std::regex_match(path[2], revRegex))
                 throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]);
-            rev = Hash::parseAny(path[2], htSHA1);
+            rev = Hash::parseAny(path[2], HashAlgorithm::SHA1);
         } else
             throw BadURL("GitHub URL '%s' is invalid", url.url);
 
diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc
index 85dc4609f..eabef55d8 100644
--- a/src/libfetchers/input-accessor.cc
+++ b/src/libfetchers/input-accessor.cc
@@ -44,8 +44,8 @@ StorePath InputAccessor::fetchToStore(
 
     auto storePath =
         settings.readOnlyMode
-        ? store->computeStorePathFromDump(*source, name, method, htSHA256).first
-        : store->addToStoreFromDump(*source, name, method, htSHA256, repair);
+        ? store->computeStorePathFromDump(*source, name, method, HashAlgorithm::SHA256).first
+        : store->addToStoreFromDump(*source, name, method, HashAlgorithm::SHA256, repair);
 
     if (cacheKey)
         fetchers::getCache()->add(store, *cacheKey, {}, storePath, true);
diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc
index aa991a75d..713f24bbb 100644
--- a/src/libfetchers/mercurial.cc
+++ b/src/libfetchers/mercurial.cc
@@ -210,7 +210,7 @@ struct MercurialInputScheme : InputScheme
                     return files.count(file);
                 };
 
-                auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter);
+                auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, HashAlgorithm::SHA256, filter);
 
                 return {std::move(storePath), input};
             }
@@ -220,7 +220,7 @@ struct MercurialInputScheme : InputScheme
 
         auto checkHashType = [&](const std::optional<Hash> & hash)
         {
-            if (hash.has_value() && hash->type != htSHA1)
+            if (hash.has_value() && hash->algo != HashAlgorithm::SHA1)
                 throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", hash->to_string(HashFormat::Base16, true));
         };
 
@@ -260,14 +260,14 @@ struct MercurialInputScheme : InputScheme
         });
 
         if (auto res = getCache()->lookup(store, unlockedAttrs)) {
-            auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1);
+            auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), HashAlgorithm::SHA1);
             if (!input.getRev() || input.getRev() == rev2) {
                 input.attrs.insert_or_assign("rev", rev2.gitRev());
                 return makeResult(res->first, std::move(res->second));
             }
         }
 
-        Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(HashFormat::Base32, false));
+        Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(HashAlgorithm::SHA256, actualUrl).to_string(HashFormat::Base32, false));
 
         /* If this is a commit hash that we already have, we don't
            have to pull again. */
@@ -301,7 +301,7 @@ struct MercurialInputScheme : InputScheme
             runHg({ "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" }));
         assert(tokens.size() == 3);
 
-        input.attrs.insert_or_assign("rev", Hash::parseAny(tokens[0], htSHA1).gitRev());
+        input.attrs.insert_or_assign("rev", Hash::parseAny(tokens[0], HashAlgorithm::SHA1).gitRev());
         auto revCount = std::stoull(tokens[1]);
         input.attrs.insert_or_assign("ref", tokens[2]);
 
diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index 0062878a9..086366180 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -73,7 +73,7 @@ DownloadFileResult downloadFile(
     } else {
         StringSink sink;
         dumpString(res.data, sink);
-        auto hash = hashString(htSHA256, res.data);
+        auto hash = hashString(HashAlgorithm::SHA256, res.data);
         ValidPathInfo info {
             *store,
             name,
@@ -82,7 +82,7 @@ DownloadFileResult downloadFile(
                 .hash = hash,
                 .references = {},
             },
-            hashString(htSHA256, sink.s),
+            hashString(HashAlgorithm::SHA256, sink.s),
         };
         info.narSize = sink.s.size();
         auto source = StringSource { sink.s };
@@ -156,7 +156,7 @@ DownloadTarballResult downloadTarball(
             throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
         auto topDir = tmpDir + "/" + members.begin()->name;
         lastModified = lstat(topDir).st_mtime;
-        unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, NoRepair);
+        unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, HashAlgorithm::SHA256, defaultPathFilter, NoRepair);
     }
 
     Attrs infoAttrs({
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index ae483c95e..f287d72a8 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -143,9 +143,9 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     /* Read the NAR simultaneously into a CompressionSink+FileSink (to
        write the compressed NAR to disk), into a HashSink (to get the
        NAR hash), and into a NarAccessor (to get the NAR listing). */
-    HashSink fileHashSink { htSHA256 };
+    HashSink fileHashSink { HashAlgorithm::SHA256 };
     std::shared_ptr<SourceAccessor> narAccessor;
-    HashSink narHashSink { htSHA256 };
+    HashSink narHashSink { HashAlgorithm::SHA256 };
     {
     FdSink fileSink(fdTemp.get());
     TeeSink teeSinkCompressed { fileSink, fileHashSink };
@@ -301,9 +301,9 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
 }
 
 StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view name,
-    FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references)
+                                               FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references)
 {
-    if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256)
+    if (method != FileIngestionMethod::Recursive || hashAlgo != HashAlgorithm::SHA256)
         unsupported("addToStoreFromDump");
     return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) {
         ValidPathInfo info {
@@ -399,13 +399,13 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
 }
 
 StorePath BinaryCacheStore::addToStore(
-    std::string_view name,
-    const Path & srcPath,
-    FileIngestionMethod method,
-    HashType hashAlgo,
-    PathFilter & filter,
-    RepairFlag repair,
-    const StorePathSet & references)
+        std::string_view name,
+        const Path & srcPath,
+        FileIngestionMethod method,
+        HashAlgorithm hashAlgo,
+        PathFilter & filter,
+        RepairFlag repair,
+        const StorePathSet & references)
 {
     /* FIXME: Make BinaryCacheStore::addToStoreCommon support
        non-recursive+sha256 so we can just use the default
@@ -448,7 +448,7 @@ StorePath BinaryCacheStore::addTextToStore(
     const StorePathSet & references,
     RepairFlag repair)
 {
-    auto textHash = hashString(htSHA256, s);
+    auto textHash = hashString(HashAlgorithm::SHA256, s);
     auto path = makeTextPath(name, TextInfo { { textHash }, references });
 
     if (!repair && isValidPath(path))
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index cea2a571f..395e1b479 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -124,16 +124,16 @@ public:
         RepairFlag repair, CheckSigsFlag checkSigs) override;
 
     StorePath addToStoreFromDump(Source & dump, std::string_view name,
-        FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override;
+                                 FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) override;
 
     StorePath addToStore(
-        std::string_view name,
-        const Path & srcPath,
-        FileIngestionMethod method,
-        HashType hashAlgo,
-        PathFilter & filter,
-        RepairFlag repair,
-        const StorePathSet & references) override;
+            std::string_view name,
+            const Path & srcPath,
+            FileIngestionMethod method,
+            HashAlgorithm hashAlgo,
+            PathFilter & filter,
+            RepairFlag repair,
+            const StorePathSet & references) override;
 
     StorePath addTextToStore(
         std::string_view name,
diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 198402ff7..4c3dc1f5c 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -1066,7 +1066,7 @@ void LocalDerivationGoal::initTmpDir() {
             if (passAsFile.find(i.first) == passAsFile.end()) {
                 env[i.first] = i.second;
             } else {
-                auto hash = hashString(htSHA256, i.first);
+                auto hash = hashString(HashAlgorithm::SHA256, i.first);
                 std::string fn = ".attr-" + hash.to_string(HashFormat::Base32, false);
                 Path p = tmpDir + "/" + fn;
                 writeFile(p, rewriteStrings(i.second, inputRewrites));
@@ -1290,13 +1290,13 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In
     { throw Error("queryPathFromHashPart"); }
 
     StorePath addToStore(
-        std::string_view name,
-        const Path & srcPath,
-        FileIngestionMethod method,
-        HashType hashAlgo,
-        PathFilter & filter,
-        RepairFlag repair,
-        const StorePathSet & references) override
+            std::string_view name,
+            const Path & srcPath,
+            FileIngestionMethod method,
+            HashAlgorithm hashAlgo,
+            PathFilter & filter,
+            RepairFlag repair,
+            const StorePathSet & references) override
     { throw Error("addToStore"); }
 
     void addToStore(const ValidPathInfo & info, Source & narSource,
@@ -1318,12 +1318,12 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In
     }
 
     StorePath addToStoreFromDump(
-        Source & dump,
-        std::string_view name,
-        FileIngestionMethod method,
-        HashType hashAlgo,
-        RepairFlag repair,
-        const StorePathSet & references) override
+            Source & dump,
+            std::string_view name,
+            FileIngestionMethod method,
+            HashAlgorithm hashAlgo,
+            RepairFlag repair,
+            const StorePathSet & references) override
     {
         auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair, references);
         goal.addDependency(path);
@@ -2466,7 +2466,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
             rewriteOutput(outputRewrites);
             /* FIXME optimize and deduplicate with addToStore */
             std::string oldHashPart { scratchPath->hashPart() };
-            HashModuloSink caSink { outputHash.hashType, oldHashPart };
+            HashModuloSink caSink {outputHash.hashAlgo, oldHashPart };
             std::visit(overloaded {
                 [&](const TextIngestionMethod &) {
                     readFile(actualPath, caSink);
@@ -2511,7 +2511,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
                                std::string(newInfo0.path.hashPart())}});
             }
 
-            HashResult narHashAndSize = hashPath(htSHA256, actualPath);
+            HashResult narHashAndSize = hashPath(HashAlgorithm::SHA256, actualPath);
             newInfo0.narHash = narHashAndSize.first;
             newInfo0.narSize = narHashAndSize.second;
 
@@ -2531,7 +2531,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
                         std::string { scratchPath->hashPart() },
                         std::string { requiredFinalPath.hashPart() });
                 rewriteOutput(outputRewrites);
-                auto narHashAndSize = hashPath(htSHA256, actualPath);
+                auto narHashAndSize = hashPath(HashAlgorithm::SHA256, actualPath);
                 ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first };
                 newInfo0.narSize = narHashAndSize.second;
                 auto refs = rewriteRefs();
@@ -2546,7 +2546,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
 
                 auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating {
                     .method = dof.ca.method,
-                    .hashType = wanted.type,
+                    .hashAlgo = wanted.algo,
                 });
 
                 /* Check wanted hash */
@@ -2583,7 +2583,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
             [&](const DerivationOutput::Impure & doi) {
                 return newInfoFromCA(DerivationOutput::CAFloating {
                     .method = doi.method,
-                    .hashType = doi.hashType,
+                    .hashAlgo = doi.hashAlgo,
                 });
             },
 
@@ -2945,7 +2945,7 @@ StorePath LocalDerivationGoal::makeFallbackPath(OutputNameView outputName)
 {
     return worker.store.makeStorePath(
         "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName),
-        Hash(htSHA256), outputPathName(drv->name, outputName));
+        Hash(HashAlgorithm::SHA256), outputPathName(drv->name, outputName));
 }
 
 
@@ -2953,7 +2953,7 @@ StorePath LocalDerivationGoal::makeFallbackPath(const StorePath & path)
 {
     return worker.store.makeStorePath(
         "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()),
-        Hash(htSHA256), path.name());
+        Hash(HashAlgorithm::SHA256), path.name());
 }
 
 
diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc
index 01f52e7ab..9b8c36286 100644
--- a/src/libstore/build/worker.cc
+++ b/src/libstore/build/worker.cc
@@ -519,8 +519,8 @@ bool Worker::pathContentsGood(const StorePath & path)
     if (!pathExists(store.printStorePath(path)))
         res = false;
     else {
-        HashResult current = hashPath(info->narHash.type, store.printStorePath(path));
-        Hash nullHash(htSHA256);
+        HashResult current = hashPath(info->narHash.algo, store.printStorePath(path));
+        Hash nullHash(HashAlgorithm::SHA256);
         res = info->narHash == nullHash || info->narHash == current.first;
     }
     pathContentsGoodCache.insert_or_assign(path, res);
diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc
index 357800333..2086bd0b9 100644
--- a/src/libstore/builtins/fetchurl.cc
+++ b/src/libstore/builtins/fetchurl.cc
@@ -63,9 +63,9 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
         for (auto hashedMirror : settings.hashedMirrors.get())
             try {
                 if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
-                std::optional<HashType> ht = parseHashTypeOpt(getAttr("outputHashAlgo"));
+                std::optional<HashAlgorithm> ht = parseHashAlgoOpt(getAttr("outputHashAlgo"));
                 Hash h = newHashAllowEmpty(getAttr("outputHash"), ht);
-                fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(HashFormat::Base16, false));
+                fetch(hashedMirror + printHashAlgo(h.algo) + "/" + h.to_string(HashFormat::Base16, false));
                 return;
             } catch (Error & e) {
                 debug(e.what());
diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc
index a5f7cdf81..de8194f73 100644
--- a/src/libstore/content-address.cc
+++ b/src/libstore/content-address.cc
@@ -38,14 +38,14 @@ ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m)
     return FileIngestionMethod::Flat;
 }
 
-std::string ContentAddressMethod::render(HashType ht) const
+std::string ContentAddressMethod::render(HashAlgorithm ha) const
 {
     return std::visit(overloaded {
         [&](const TextIngestionMethod & th) {
-            return std::string{"text:"} + printHashType(ht);
+            return std::string{"text:"} + printHashAlgo(ha);
         },
         [&](const FileIngestionMethod & fim) {
-            return "fixed:" + makeFileIngestionPrefix(fim) + printHashType(ht);
+            return "fixed:" + makeFileIngestionPrefix(fim) + printHashAlgo(ha);
         }
     }, raw);
 }
@@ -67,7 +67,7 @@ std::string ContentAddress::render() const
 /**
  * Parses content address strings up to the hash.
  */
-static std::pair<ContentAddressMethod, HashType> parseContentAddressMethodPrefix(std::string_view & rest)
+static std::pair<ContentAddressMethod, HashAlgorithm> parseContentAddressMethodPrefix(std::string_view & rest)
 {
     std::string_view wholeInput { rest };
 
@@ -83,27 +83,27 @@ static std::pair<ContentAddressMethod, HashType> parseContentAddressMethodPrefix
         auto hashTypeRaw = splitPrefixTo(rest, ':');
         if (!hashTypeRaw)
             throw UsageError("content address hash must be in form '<algo>:<hash>', but found: %s", wholeInput);
-        HashType hashType = parseHashType(*hashTypeRaw);
-        return hashType;
+        HashAlgorithm hashAlgo = parseHashAlgo(*hashTypeRaw);
+        return hashAlgo;
     };
 
     // Switch on prefix
     if (prefix == "text") {
         // No parsing of the ingestion method, "text" only support flat.
-        HashType hashType = parseHashType_();
+        HashAlgorithm hashAlgo = parseHashType_();
         return {
             TextIngestionMethod {},
-            std::move(hashType),
+            std::move(hashAlgo),
         };
     } else if (prefix == "fixed") {
         // Parse method
         auto method = FileIngestionMethod::Flat;
         if (splitPrefix(rest, "r:"))
             method = FileIngestionMethod::Recursive;
-        HashType hashType = parseHashType_();
+        HashAlgorithm hashAlgo = parseHashType_();
         return {
             std::move(method),
-            std::move(hashType),
+            std::move(hashAlgo),
         };
     } else
         throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix);
@@ -113,15 +113,15 @@ ContentAddress ContentAddress::parse(std::string_view rawCa)
 {
     auto rest = rawCa;
 
-    auto [caMethod, hashType] = parseContentAddressMethodPrefix(rest);
+    auto [caMethod, hashAlgo] = parseContentAddressMethodPrefix(rest);
 
     return ContentAddress {
         .method = std::move(caMethod),
-        .hash = Hash::parseNonSRIUnprefixed(rest, hashType),
+        .hash = Hash::parseNonSRIUnprefixed(rest, hashAlgo),
     };
 }
 
-std::pair<ContentAddressMethod, HashType> ContentAddressMethod::parse(std::string_view caMethod)
+std::pair<ContentAddressMethod, HashAlgorithm> ContentAddressMethod::parse(std::string_view caMethod)
 {
     std::string asPrefix = std::string{caMethod} + ":";
     // parseContentAddressMethodPrefix takes its argument by reference
@@ -144,7 +144,7 @@ std::string renderContentAddress(std::optional<ContentAddress> ca)
 std::string ContentAddress::printMethodAlgo() const
 {
     return method.renderPrefix()
-        + printHashType(hash.type);
+        + printHashAlgo(hash.algo);
 }
 
 bool StoreReferences::empty() const
diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh
index bdb558907..05234da38 100644
--- a/src/libstore/content-address.hh
+++ b/src/libstore/content-address.hh
@@ -94,7 +94,7 @@ struct ContentAddressMethod
     /**
      * Parse a content addressing method and hash type.
      */
-    static std::pair<ContentAddressMethod, HashType> parse(std::string_view rawCaMethod);
+    static std::pair<ContentAddressMethod, HashAlgorithm> parse(std::string_view rawCaMethod);
 
     /**
      * Render a content addressing method and hash type in a
@@ -102,7 +102,7 @@ struct ContentAddressMethod
      *
      * The rough inverse of `parse()`.
      */
-    std::string render(HashType ht) const;
+    std::string render(HashAlgorithm ha) const;
 };
 
 
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index be9b0b0d3..530b1a178 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -400,22 +400,22 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             logger->startWork();
             auto pathInfo = [&]() {
                 // NB: FramedSource must be out of scope before logger->stopWork();
-                auto [contentAddressMethod, hashType_] = ContentAddressMethod::parse(camStr);
-                auto hashType = hashType_; // work around clang bug
+                auto [contentAddressMethod, hashAlgo_] = ContentAddressMethod::parse(camStr);
+                auto hashAlgo = hashAlgo_; // work around clang bug
                 FramedSource source(from);
                 // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store.
                 return std::visit(overloaded {
                     [&](const TextIngestionMethod &) {
-                        if (hashType != htSHA256)
+                        if (hashAlgo != HashAlgorithm::SHA256)
                             throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given",
-                                name, printHashType(hashType));
+                                name, printHashAlgo(hashAlgo));
                         // We could stream this by changing Store
                         std::string contents = source.drain();
                         auto path = store->addTextToStore(name, contents, refs, repair);
                         return store->queryPathInfo(path);
                     },
                     [&](const FileIngestionMethod & fim) {
-                        auto path = store->addToStoreFromDump(source, name, fim, hashType, repair, refs);
+                        auto path = store->addToStoreFromDump(source, name, fim, hashAlgo, repair, refs);
                         return store->queryPathInfo(path);
                     },
                 }, contentAddressMethod.raw);
@@ -424,7 +424,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
 
             WorkerProto::Serialise<ValidPathInfo>::write(*store, wconn, *pathInfo);
         } else {
-            HashType hashAlgo;
+            HashAlgorithm hashAlgo;
             std::string baseName;
             FileIngestionMethod method;
             {
@@ -440,7 +440,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                     hashAlgoRaw = "sha256";
                     method = FileIngestionMethod::Recursive;
                 }
-                hashAlgo = parseHashType(hashAlgoRaw);
+                hashAlgo = parseHashAlgo(hashAlgoRaw);
             }
 
             auto dumpSource = sinkToSource([&](Sink & saved) {
@@ -883,7 +883,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         bool repair, dontCheckSigs;
         auto path = store->parseStorePath(readString(from));
         auto deriver = readString(from);
-        auto narHash = Hash::parseAny(readString(from), htSHA256);
+        auto narHash = Hash::parseAny(readString(from), HashAlgorithm::SHA256);
         ValidPathInfo info { path, narHash };
         if (deriver != "")
             info.deriver = store->parseStorePath(deriver);
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index dd87203b8..c68631c1a 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -215,25 +215,25 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
 
 static DerivationOutput parseDerivationOutput(
     const StoreDirConfig & store,
-    std::string_view pathS, std::string_view hashAlgo, std::string_view hashS,
+    std::string_view pathS, std::string_view hashAlgoStr, std::string_view hashS,
     const ExperimentalFeatureSettings & xpSettings)
 {
-    if (hashAlgo != "") {
-        ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgo);
+    if (hashAlgoStr != "") {
+        ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgoStr);
         if (method == TextIngestionMethod {})
             xpSettings.require(Xp::DynamicDerivations);
-        const auto hashType = parseHashType(hashAlgo);
+        const auto hashAlgo = parseHashAlgo(hashAlgoStr);
         if (hashS == "impure") {
             xpSettings.require(Xp::ImpureDerivations);
             if (pathS != "")
                 throw FormatError("impure derivation output should not specify output path");
             return DerivationOutput::Impure {
                 .method = std::move(method),
-                .hashType = std::move(hashType),
+                .hashAlgo = std::move(hashAlgo),
             };
         } else if (hashS != "") {
             validatePath(pathS);
-            auto hash = Hash::parseNonSRIUnprefixed(hashS, hashType);
+            auto hash = Hash::parseNonSRIUnprefixed(hashS, hashAlgo);
             return DerivationOutput::CAFixed {
                 .ca = ContentAddress {
                     .method = std::move(method),
@@ -246,7 +246,7 @@ static DerivationOutput parseDerivationOutput(
                 throw FormatError("content-addressed derivation output should not specify output path");
             return DerivationOutput::CAFloating {
                 .method = std::move(method),
-                .hashType = std::move(hashType),
+                .hashAlgo = std::move(hashAlgo),
             };
         }
     } else {
@@ -547,7 +547,7 @@ std::string Derivation::unparse(const StoreDirConfig & store, bool maskOutputs,
             },
             [&](const DerivationOutput::CAFloating & dof) {
                 s += ','; printUnquotedString(s, "");
-                s += ','; printUnquotedString(s, dof.method.renderPrefix() + printHashType(dof.hashType));
+                s += ','; printUnquotedString(s, dof.method.renderPrefix() + printHashAlgo(dof.hashAlgo));
                 s += ','; printUnquotedString(s, "");
             },
             [&](const DerivationOutput::Deferred &) {
@@ -558,7 +558,7 @@ std::string Derivation::unparse(const StoreDirConfig & store, bool maskOutputs,
             [&](const DerivationOutput::Impure & doi) {
                 // FIXME
                 s += ','; printUnquotedString(s, "");
-                s += ','; printUnquotedString(s, doi.method.renderPrefix() + printHashType(doi.hashType));
+                s += ','; printUnquotedString(s, doi.method.renderPrefix() + printHashAlgo(doi.hashAlgo));
                 s += ','; printUnquotedString(s, "impure");
             }
         }, i.second.raw);
@@ -631,7 +631,7 @@ DerivationType BasicDerivation::type() const
         floatingCAOutputs,
         deferredIAOutputs,
         impureOutputs;
-    std::optional<HashType> floatingHashType;
+    std::optional<HashAlgorithm> floatingHashAlgo;
 
     for (auto & i : outputs) {
         std::visit(overloaded {
@@ -643,10 +643,10 @@ DerivationType BasicDerivation::type() const
             },
             [&](const DerivationOutput::CAFloating & dof) {
                 floatingCAOutputs.insert(i.first);
-                if (!floatingHashType) {
-                    floatingHashType = dof.hashType;
+                if (!floatingHashAlgo) {
+                    floatingHashAlgo = dof.hashAlgo;
                 } else {
-                    if (*floatingHashType != dof.hashType)
+                    if (*floatingHashAlgo != dof.hashAlgo)
                         throw Error("all floating outputs must use the same hash type");
                 }
             },
@@ -774,7 +774,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
         std::map<std::string, Hash> outputHashes;
         for (const auto & i : drv.outputs) {
             auto & dof = std::get<DerivationOutput::CAFixed>(i.second.raw);
-            auto hash = hashString(htSHA256, "fixed:out:"
+            auto hash = hashString(HashAlgorithm::SHA256, "fixed:out:"
                 + dof.ca.printMethodAlgo() + ":"
                 + dof.ca.hash.to_string(HashFormat::Base16, false) + ":"
                 + store.printStorePath(dof.path(store, drv.name, i.first)));
@@ -825,7 +825,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
         }
     }
 
-    auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
+    auto hash = hashString(HashAlgorithm::SHA256, drv.unparse(store, maskOutputs, &inputs2));
 
     std::map<std::string, Hash> outputHashes;
     for (const auto & [outputName, _] : drv.outputs) {
@@ -930,7 +930,7 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva
             },
             [&](const DerivationOutput::CAFloating & dof) {
                 out << ""
-                    << (dof.method.renderPrefix() + printHashType(dof.hashType))
+                    << (dof.method.renderPrefix() + printHashAlgo(dof.hashAlgo))
                     << "";
             },
             [&](const DerivationOutput::Deferred &) {
@@ -940,7 +940,7 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva
             },
             [&](const DerivationOutput::Impure & doi) {
                 out << ""
-                    << (doi.method.renderPrefix() + printHashType(doi.hashType))
+                    << (doi.method.renderPrefix() + printHashAlgo(doi.hashAlgo))
                     << "impure";
             },
         }, i.second.raw);
@@ -958,7 +958,7 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva
 std::string hashPlaceholder(const OutputNameView outputName)
 {
     // FIXME: memoize?
-    return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(HashFormat::Base32, false);
+    return "/" + hashString(HashAlgorithm::SHA256, concatStrings("nix-output:", outputName)).to_string(HashFormat::Base32, false);
 }
 
 
@@ -1150,7 +1150,7 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const
 }
 
 
-const Hash impureOutputHash = hashString(htSHA256, "impure");
+const Hash impureOutputHash = hashString(HashAlgorithm::SHA256, "impure");
 
 nlohmann::json DerivationOutput::toJSON(
     const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const
@@ -1167,11 +1167,11 @@ nlohmann::json DerivationOutput::toJSON(
             // FIXME print refs?
         },
         [&](const DerivationOutput::CAFloating & dof) {
-            res["hashAlgo"] = dof.method.renderPrefix() + printHashType(dof.hashType);
+            res["hashAlgo"] = dof.method.renderPrefix() + printHashAlgo(dof.hashAlgo);
         },
         [&](const DerivationOutput::Deferred &) {},
         [&](const DerivationOutput::Impure & doi) {
-            res["hashAlgo"] = doi.method.renderPrefix() + printHashType(doi.hashType);
+            res["hashAlgo"] = doi.method.renderPrefix() + printHashAlgo(doi.hashAlgo);
             res["impure"] = true;
         },
     }, raw);
@@ -1191,15 +1191,15 @@ DerivationOutput DerivationOutput::fromJSON(
     for (const auto & [key, _] : json)
         keys.insert(key);
 
-    auto methodAlgo = [&]() -> std::pair<ContentAddressMethod, HashType> {
-        std::string hashAlgo = json["hashAlgo"];
+    auto methodAlgo = [&]() -> std::pair<ContentAddressMethod, HashAlgorithm> {
+        std::string hashAlgoStr = json["hashAlgo"];
         // remaining to parse, will be mutated by parsers
-        std::string_view s = hashAlgo;
+        std::string_view s = hashAlgoStr;
         ContentAddressMethod method = ContentAddressMethod::parsePrefix(s);
         if (method == TextIngestionMethod {})
             xpSettings.require(Xp::DynamicDerivations);
-        auto hashType = parseHashType(s);
-        return { std::move(method), std::move(hashType) };
+        auto hashAlgo = parseHashAlgo(s);
+        return { std::move(method), std::move(hashAlgo) };
     };
 
     if (keys == (std::set<std::string_view> { "path" })) {
@@ -1209,11 +1209,11 @@ DerivationOutput DerivationOutput::fromJSON(
     }
 
     else if (keys == (std::set<std::string_view> { "path", "hashAlgo", "hash" })) {
-        auto [method, hashType] = methodAlgo();
+        auto [method, hashAlgo] = methodAlgo();
         auto dof = DerivationOutput::CAFixed {
             .ca = ContentAddress {
                 .method = std::move(method),
-                .hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType),
+                .hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashAlgo),
             },
         };
         if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"]))
@@ -1223,10 +1223,10 @@ DerivationOutput DerivationOutput::fromJSON(
 
     else if (keys == (std::set<std::string_view> { "hashAlgo" })) {
         xpSettings.require(Xp::CaDerivations);
-        auto [method, hashType] = methodAlgo();
+        auto [method, hashAlgo] = methodAlgo();
         return DerivationOutput::CAFloating {
             .method = std::move(method),
-            .hashType = std::move(hashType),
+            .hashAlgo = std::move(hashAlgo),
         };
     }
 
@@ -1236,10 +1236,10 @@ DerivationOutput DerivationOutput::fromJSON(
 
     else if (keys == (std::set<std::string_view> { "hashAlgo", "impure" })) {
         xpSettings.require(Xp::ImpureDerivations);
-        auto [method, hashType] = methodAlgo();
+        auto [method, hashAlgo] = methodAlgo();
         return DerivationOutput::Impure {
             .method = std::move(method),
-            .hashType = hashType,
+            .hashAlgo = hashAlgo,
         };
     }
 
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 219e8e7d7..290abedcf 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -75,9 +75,9 @@ struct DerivationOutput
         /**
          * How the serialization will be hashed
          */
-        HashType hashType;
+        HashAlgorithm hashAlgo;
 
-        GENERATE_CMP(CAFloating, me->method, me->hashType);
+        GENERATE_CMP(CAFloating, me->method, me->hashAlgo);
     };
 
     /**
@@ -102,9 +102,9 @@ struct DerivationOutput
         /**
          * How the serialization will be hashed
          */
-        HashType hashType;
+        HashAlgorithm hashAlgo;
 
-        GENERATE_CMP(Impure, me->method, me->hashType);
+        GENERATE_CMP(Impure, me->method, me->hashAlgo);
     };
 
     typedef std::variant<
diff --git a/src/libstore/downstream-placeholder.cc b/src/libstore/downstream-placeholder.cc
index ca9f7476e..10df37fa4 100644
--- a/src/libstore/downstream-placeholder.cc
+++ b/src/libstore/downstream-placeholder.cc
@@ -19,7 +19,7 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput(
     auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4);
     auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName);
     return DownstreamPlaceholder {
-        hashString(htSHA256, clearText)
+        hashString(HashAlgorithm::SHA256, clearText)
     };
 }
 
@@ -34,7 +34,7 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation(
         + compressed.to_string(HashFormat::Base32, false)
         + ":" + std::string { outputName };
     return DownstreamPlaceholder {
-        hashString(htSHA256, clearText)
+        hashString(HashAlgorithm::SHA256, clearText)
     };
 }
 
diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc
index 52130f8f6..48718ef84 100644
--- a/src/libstore/export-import.cc
+++ b/src/libstore/export-import.cc
@@ -30,7 +30,7 @@ void Store::exportPath(const StorePath & path, Sink & sink)
 {
     auto info = queryPathInfo(path);
 
-    HashSink hashSink(htSHA256);
+    HashSink hashSink(HashAlgorithm::SHA256);
     TeeSink teeSink(sink, hashSink);
 
     narFromPath(path, teeSink);
@@ -39,7 +39,7 @@ void Store::exportPath(const StorePath & path, Sink & sink)
        filesystem corruption from spreading to other machines.
        Don't complain if the stored hash is zero (unknown). */
     Hash hash = hashSink.currentHash().first;
-    if (hash != info->narHash && info->narHash != Hash(info->narHash.type))
+    if (hash != info->narHash && info->narHash != Hash(info->narHash.algo))
         throw Error("hash of path '%s' has changed from '%s' to '%s'!",
             printStorePath(path), info->narHash.to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true));
 
@@ -79,7 +79,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
         auto references = CommonProto::Serialise<StorePathSet>::read(*this,
             CommonProto::ReadConn { .from = source });
         auto deriver = readString(source);
-        auto narHash = hashString(htSHA256, saved.s);
+        auto narHash = hashString(HashAlgorithm::SHA256, saved.s);
 
         ValidPathInfo info { path, narHash };
         if (deriver != "")
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 93fa60682..5c413aa77 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -50,7 +50,7 @@ static void makeSymlink(const Path & link, const Path & target)
 
 void LocalStore::addIndirectRoot(const Path & path)
 {
-    std::string hash = hashString(htSHA1, path).to_string(HashFormat::Base32, false);
+    std::string hash = hashString(HashAlgorithm::SHA1, path).to_string(HashFormat::Base32, false);
     Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", stateDir, gcRootsDir, hash));
     makeSymlink(realRoot, path);
 }
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 731457354..fb1580dd6 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -267,13 +267,13 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
     { unsupported("queryPathFromHashPart"); }
 
     StorePath addToStore(
-        std::string_view name,
-        const Path & srcPath,
-        FileIngestionMethod method,
-        HashType hashAlgo,
-        PathFilter & filter,
-        RepairFlag repair,
-        const StorePathSet & references) override
+            std::string_view name,
+            const Path & srcPath,
+            FileIngestionMethod method,
+            HashAlgorithm hashAlgo,
+            PathFilter & filter,
+            RepairFlag repair,
+            const StorePathSet & references) override
     { unsupported("addToStore"); }
 
     StorePath addTextToStore(
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index c8962f574..ef7dd7985 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -955,7 +955,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
         StorePathSet paths;
 
         for (auto & [_, i] : infos) {
-            assert(i.narHash.type == htSHA256);
+            assert(i.narHash.algo == HashAlgorithm::SHA256);
             if (isValidPath_(*state, i.path))
                 updatePathInfo(*state, i);
             else
@@ -1069,7 +1069,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
             /* While restoring the path from the NAR, compute the hash
                of the NAR. */
-            HashSink hashSink(htSHA256);
+            HashSink hashSink(HashAlgorithm::SHA256);
 
             TeeSource wrapperSource { source, hashSink };
 
@@ -1090,7 +1090,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
                 auto & specified = *info.ca;
                 auto actualHash = hashCAPath(
                     specified.method,
-                    specified.hash.type,
+                    specified.hash.algo,
                     info.path
                 );
                 if (specified.hash != actualHash.hash) {
@@ -1116,7 +1116,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
 
 StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name,
-    FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references)
+                                         FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references)
 {
     /* For computing the store path. */
     auto hashSink = std::make_unique<HashSink>(hashAlgo);
@@ -1220,8 +1220,8 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name
             /* For computing the nar hash. In recursive SHA-256 mode, this
                is the same as the store hash, so no need to do it again. */
             auto narHash = std::pair { hash, size };
-            if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) {
-                HashSink narSink { htSHA256 };
+            if (method != FileIngestionMethod::Recursive || hashAlgo != HashAlgorithm::SHA256) {
+                HashSink narSink { HashAlgorithm::SHA256 };
                 dumpPath(realPath, narSink);
                 narHash = narSink.finish();
             }
@@ -1252,7 +1252,7 @@ StorePath LocalStore::addTextToStore(
     std::string_view s,
     const StorePathSet & references, RepairFlag repair)
 {
-    auto hash = hashString(htSHA256, s);
+    auto hash = hashString(HashAlgorithm::SHA256, s);
     auto dstPath = makeTextPath(name, TextInfo {
         .hash = hash,
         .references = references,
@@ -1278,7 +1278,7 @@ StorePath LocalStore::addTextToStore(
 
             StringSink sink;
             dumpString(s, sink);
-            auto narHash = hashString(htSHA256, sink.s);
+            auto narHash = hashString(HashAlgorithm::SHA256, sink.s);
 
             optimisePath(realPath, repair);
 
@@ -1389,7 +1389,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
         for (auto & link : readDirectory(linksDir)) {
             printMsg(lvlTalkative, "checking contents of '%s'", link.name);
             Path linkPath = linksDir + "/" + link.name;
-            std::string hash = hashPath(htSHA256, linkPath).first.to_string(HashFormat::Base32, false);
+            std::string hash = hashPath(HashAlgorithm::SHA256, linkPath).first.to_string(HashFormat::Base32, false);
             if (hash != link.name) {
                 printError("link '%s' was modified! expected hash '%s', got '%s'",
                     linkPath, link.name, hash);
@@ -1406,7 +1406,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 
         printInfo("checking store hashes...");
 
-        Hash nullHash(htSHA256);
+        Hash nullHash(HashAlgorithm::SHA256);
 
         for (auto & i : validPaths) {
             try {
@@ -1415,7 +1415,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
                 /* Check the content hash (optionally - slow). */
                 printMsg(lvlTalkative, "checking contents of '%s'", printStorePath(i));
 
-                auto hashSink = HashSink(info->narHash.type);
+                auto hashSink = HashSink(info->narHash.algo);
 
                 dumpPath(Store::toRealPath(i), hashSink);
                 auto current = hashSink.finish();
@@ -1697,20 +1697,20 @@ void LocalStore::queryRealisationUncached(const DrvOutput & id,
 }
 
 ContentAddress LocalStore::hashCAPath(
-    const ContentAddressMethod & method, const HashType & hashType,
+    const ContentAddressMethod & method, const HashAlgorithm & hashAlgo,
     const StorePath & path)
 {
-    return hashCAPath(method, hashType, Store::toRealPath(path), path.hashPart());
+    return hashCAPath(method, hashAlgo, Store::toRealPath(path), path.hashPart());
 }
 
 ContentAddress LocalStore::hashCAPath(
     const ContentAddressMethod & method,
-    const HashType & hashType,
+    const HashAlgorithm & hashAlgo,
     const Path & path,
     const std::string_view pathHash
 )
 {
-    HashModuloSink caSink ( hashType, std::string(pathHash) );
+    HashModuloSink caSink ( hashAlgo, std::string(pathHash) );
     std::visit(overloaded {
         [&](const TextIngestionMethod &) {
             readFile(path, caSink);
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 8f0ffd2a2..ee605b5a2 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -178,7 +178,7 @@ public:
         RepairFlag repair, CheckSigsFlag checkSigs) override;
 
     StorePath addToStoreFromDump(Source & dump, std::string_view name,
-        FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override;
+                                 FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) override;
 
     StorePath addTextToStore(
         std::string_view name,
@@ -353,12 +353,12 @@ private:
     // XXX: Make a generic `Store` method
     ContentAddress hashCAPath(
         const ContentAddressMethod & method,
-        const HashType & hashType,
+        const HashAlgorithm & hashAlgo,
         const StorePath & path);
 
     ContentAddress hashCAPath(
         const ContentAddressMethod & method,
-        const HashType & hashType,
+        const HashAlgorithm & hashAlgo,
         const Path & path,
         const std::string_view pathHash
     );
diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc
index 253609ed2..170fe67b9 100644
--- a/src/libstore/make-content-addressed.cc
+++ b/src/libstore/make-content-addressed.cc
@@ -43,7 +43,7 @@ std::map<StorePath, StorePath> makeContentAddressed(
 
         sink.s = rewriteStrings(sink.s, rewrites);
 
-        HashModuloSink hashModuloSink(htSHA256, oldHashPart);
+        HashModuloSink hashModuloSink(HashAlgorithm::SHA256, oldHashPart);
         hashModuloSink(sink.s);
 
         auto narModuloHash = hashModuloSink.finish().first;
@@ -66,7 +66,7 @@ std::map<StorePath, StorePath> makeContentAddressed(
         rsink2(sink.s);
         rsink2.flush();
 
-        info.narHash = hashString(htSHA256, sink2.s);
+        info.narHash = hashString(HashAlgorithm::SHA256, sink2.s);
         info.narSize = sink.s.size();
 
         StringSource source(sink2.s);
diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc
index 1060a6c8b..25e2a7d7b 100644
--- a/src/libstore/nar-info.cc
+++ b/src/libstore/nar-info.cc
@@ -113,10 +113,10 @@ std::string NarInfo::to_string(const Store & store) const
     res += "URL: " + url + "\n";
     assert(compression != "");
     res += "Compression: " + compression + "\n";
-    assert(fileHash && fileHash->type == htSHA256);
+    assert(fileHash && fileHash->algo == HashAlgorithm::SHA256);
     res += "FileHash: " + fileHash->to_string(HashFormat::Base32, true) + "\n";
     res += "FileSize: " + std::to_string(fileSize) + "\n";
-    assert(narHash.type == htSHA256);
+    assert(narHash.algo == HashAlgorithm::SHA256);
     res += "NarHash: " + narHash.to_string(HashFormat::Base32, true) + "\n";
     res += "NarSize: " + std::to_string(narSize) + "\n";
 
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index 0fa977545..cadf88347 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -146,7 +146,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
        Also note that if `path' is a symlink, then we're hashing the
        contents of the symlink (i.e. the result of readlink()), not
        the contents of the target (which may not even exist). */
-    Hash hash = hashPath(htSHA256, path).first;
+    Hash hash = hashPath(HashAlgorithm::SHA256, path).first;
     debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Base32, true));
 
     /* Check if this is a known hash. */
@@ -156,7 +156,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
     if (pathExists(linkPath)) {
         auto stLink = lstat(linkPath);
         if (st.st_size != stLink.st_size
-            || (repair && hash != hashPath(htSHA256, linkPath).first))
+            || (repair && hash != hashPath(HashAlgorithm::SHA256, linkPath).first))
         {
             // XXX: Consider overwriting linkPath with our valid version.
             warn("removing corrupted link '%s'", linkPath);
diff --git a/src/libstore/path-references.cc b/src/libstore/path-references.cc
index 274b596c0..15f52ec9d 100644
--- a/src/libstore/path-references.cc
+++ b/src/libstore/path-references.cc
@@ -49,7 +49,7 @@ std::pair<StorePathSet, HashResult> scanForReferences(
     const std::string & path,
     const StorePathSet & refs)
 {
-    HashSink hashSink { htSHA256 };
+    HashSink hashSink { HashAlgorithm::SHA256 };
     auto found = scanForReferences(hashSink, path, refs);
     auto hash = hashSink.finish();
     return std::pair<StorePathSet, HashResult>(found, hash);
diff --git a/src/libstore/path.cc b/src/libstore/path.cc
index 69f6d7356..d5257c939 100644
--- a/src/libstore/path.cc
+++ b/src/libstore/path.cc
@@ -49,7 +49,7 @@ StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x");
 
 StorePath StorePath::random(std::string_view name)
 {
-    Hash hash(htSHA1);
+    Hash hash(HashAlgorithm::SHA1);
     randombytes_buf(hash.hash, hash.hashSize);
     return StorePath(hash, name);
 }
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 3d3919882..cc26c2a94 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -417,12 +417,12 @@ std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string &
 
 
 ref<const ValidPathInfo> RemoteStore::addCAToStore(
-    Source & dump,
-    std::string_view name,
-    ContentAddressMethod caMethod,
-    HashType hashType,
-    const StorePathSet & references,
-    RepairFlag repair)
+        Source & dump,
+        std::string_view name,
+        ContentAddressMethod caMethod,
+        HashAlgorithm hashAlgo,
+        const StorePathSet & references,
+        RepairFlag repair)
 {
     std::optional<ConnectionHandle> conn_(getConnection());
     auto & conn = *conn_;
@@ -432,7 +432,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
         conn->to
             << WorkerProto::Op::AddToStore
             << name
-            << caMethod.render(hashType);
+            << caMethod.render(hashAlgo);
         WorkerProto::write(*this, *conn, references);
         conn->to << repair;
 
@@ -453,9 +453,9 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
 
         std::visit(overloaded {
             [&](const TextIngestionMethod & thm) -> void {
-                if (hashType != htSHA256)
+                if (hashAlgo != HashAlgorithm::SHA256)
                     throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given",
-                        name, printHashType(hashType));
+                        name, printHashAlgo(hashAlgo));
                 std::string s = dump.drain();
                 conn->to << WorkerProto::Op::AddTextToStore << name << s;
                 WorkerProto::write(*this, *conn, references);
@@ -465,9 +465,9 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
                 conn->to
                     << WorkerProto::Op::AddToStore
                     << name
-                    << ((hashType == htSHA256 && fim == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */
+                    << ((hashAlgo == HashAlgorithm::SHA256 && fim == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */
                     << (fim == FileIngestionMethod::Recursive ? 1 : 0)
-                    << printHashType(hashType);
+                    << printHashAlgo(hashAlgo);
 
                 try {
                     conn->to.written = 0;
@@ -503,9 +503,9 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
 
 
 StorePath RemoteStore::addToStoreFromDump(Source & dump, std::string_view name,
-      FileIngestionMethod method, HashType hashType, RepairFlag repair, const StorePathSet & references)
+                                          FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references)
 {
-    return addCAToStore(dump, name, method, hashType, references, repair)->path;
+    return addCAToStore(dump, name, method, hashAlgo, references, repair)->path;
 }
 
 
@@ -610,7 +610,7 @@ StorePath RemoteStore::addTextToStore(
     RepairFlag repair)
 {
     StringSource source(s);
-    return addCAToStore(source, name, TextIngestionMethod {}, htSHA256, references, repair)->path;
+    return addCAToStore(source, name, TextIngestionMethod {}, HashAlgorithm::SHA256, references, repair)->path;
 }
 
 void RemoteStore::registerDrvOutput(const Realisation & info)
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 1cc11af86..f2e34c1a3 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -74,18 +74,18 @@ public:
      * Add a content-addressable store path. `dump` will be drained.
      */
     ref<const ValidPathInfo> addCAToStore(
-        Source & dump,
-        std::string_view name,
-        ContentAddressMethod caMethod,
-        HashType hashType,
-        const StorePathSet & references,
-        RepairFlag repair);
+            Source & dump,
+            std::string_view name,
+            ContentAddressMethod caMethod,
+            HashAlgorithm hashAlgo,
+            const StorePathSet & references,
+            RepairFlag repair);
 
     /**
      * Add a content-addressable store path. Does not support references. `dump` will be drained.
      */
     StorePath addToStoreFromDump(Source & dump, std::string_view name,
-        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override;
+                                 FileIngestionMethod method = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override;
 
     void addToStore(const ValidPathInfo & info, Source & nar,
         RepairFlag repair, CheckSigsFlag checkSigs) override;
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 8601e0857..800df7fa0 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -153,7 +153,7 @@ StorePath StoreDirConfig::makeStorePath(std::string_view type,
     /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
     auto s = std::string(type) + ":" + std::string(hash)
         + ":" + storeDir + ":" + std::string(name);
-    auto h = compressHash(hashString(htSHA256, s), 20);
+    auto h = compressHash(hashString(HashAlgorithm::SHA256, s), 20);
     return StorePath(h, name);
 }
 
@@ -191,12 +191,12 @@ static std::string makeType(
 
 StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const
 {
-    if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) {
+    if (info.hash.algo == HashAlgorithm::SHA256 && info.method == FileIngestionMethod::Recursive) {
         return makeStorePath(makeType(*this, "source", info.references), info.hash, name);
     } else {
         assert(info.references.size() == 0);
         return makeStorePath("output:out",
-            hashString(htSHA256,
+            hashString(HashAlgorithm::SHA256,
                 "fixed:out:"
                 + makeFileIngestionPrefix(info.method)
                 + info.hash.to_string(HashFormat::Base16, true) + ":"),
@@ -207,7 +207,7 @@ StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const Fixed
 
 StorePath StoreDirConfig::makeTextPath(std::string_view name, const TextInfo & info) const
 {
-    assert(info.hash.type == htSHA256);
+    assert(info.hash.algo == HashAlgorithm::SHA256);
     return makeStorePath(
         makeType(*this, "text", StoreReferences {
             .others = info.references,
@@ -233,11 +233,11 @@ StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const
 
 
 std::pair<StorePath, Hash> StoreDirConfig::computeStorePathFromDump(
-    Source & dump,
-    std::string_view name,
-    FileIngestionMethod method,
-    HashType hashAlgo,
-    const StorePathSet & references) const
+        Source & dump,
+        std::string_view name,
+        FileIngestionMethod method,
+        HashAlgorithm hashAlgo,
+        const StorePathSet & references) const
 {
     HashSink sink(hashAlgo);
     dump.drainInto(sink);
@@ -257,20 +257,20 @@ StorePath StoreDirConfig::computeStorePathForText(
     const StorePathSet & references) const
 {
     return makeTextPath(name, TextInfo {
-        .hash = hashString(htSHA256, s),
+        .hash = hashString(HashAlgorithm::SHA256, s),
         .references = references,
     });
 }
 
 
 StorePath Store::addToStore(
-    std::string_view name,
-    const Path & _srcPath,
-    FileIngestionMethod method,
-    HashType hashAlgo,
-    PathFilter & filter,
-    RepairFlag repair,
-    const StorePathSet & references)
+        std::string_view name,
+        const Path & _srcPath,
+        FileIngestionMethod method,
+        HashAlgorithm hashAlgo,
+        PathFilter & filter,
+        RepairFlag repair,
+        const StorePathSet & references)
 {
     Path srcPath(absPath(_srcPath));
     auto source = sinkToSource([&](Sink & sink) {
@@ -405,10 +405,10 @@ digraph graphname {
 }
 */
 ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
-    FileIngestionMethod method, HashType hashAlgo,
-    std::optional<Hash> expectedCAHash)
+                                    FileIngestionMethod method, HashAlgorithm hashAlgo,
+                                    std::optional<Hash> expectedCAHash)
 {
-    HashSink narHashSink { htSHA256 };
+    HashSink narHashSink { HashAlgorithm::SHA256 };
     HashSink caHashSink { hashAlgo };
 
     /* Note that fileSink and unusualHashTee must be mutually exclusive, since
@@ -417,7 +417,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
     RegularFileSink fileSink { caHashSink };
     TeeSink unusualHashTee { narHashSink, caHashSink };
 
-    auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != htSHA256
+    auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != HashAlgorithm::SHA256
         ? static_cast<Sink &>(unusualHashTee)
         : narHashSink;
 
@@ -445,7 +445,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
        finish. */
     auto [narHash, narSize] = narHashSink.finish();
 
-    auto hash = method == FileIngestionMethod::Recursive && hashAlgo == htSHA256
+    auto hash = method == FileIngestionMethod::Recursive && hashAlgo == HashAlgorithm::SHA256
         ? narHash
         : caHashSink.finish().first;
 
@@ -1205,7 +1205,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istre
     if (!hashGiven) {
         std::string s;
         getline(str, s);
-        auto narHash = Hash::parseAny(s, htSHA256);
+        auto narHash = Hash::parseAny(s, HashAlgorithm::SHA256);
         getline(str, s);
         auto narSize = string2Int<uint64_t>(s);
         if (!narSize) throw Error("number expected");
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 5860d0ea6..ada6699d5 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -427,13 +427,13 @@ public:
      * libutil/archive.hh).
      */
     virtual StorePath addToStore(
-        std::string_view name,
-        const Path & srcPath,
-        FileIngestionMethod method = FileIngestionMethod::Recursive,
-        HashType hashAlgo = htSHA256,
-        PathFilter & filter = defaultPathFilter,
-        RepairFlag repair = NoRepair,
-        const StorePathSet & references = StorePathSet());
+            std::string_view name,
+            const Path & srcPath,
+            FileIngestionMethod method = FileIngestionMethod::Recursive,
+            HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
+            PathFilter & filter = defaultPathFilter,
+            RepairFlag repair = NoRepair,
+            const StorePathSet & references = StorePathSet());
 
     /**
      * Copy the contents of a path to the store and register the
@@ -441,8 +441,8 @@ public:
      * memory.
      */
     ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath,
-        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
-        std::optional<Hash> expectedCAHash = {});
+                                 FileIngestionMethod method = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
+                                 std::optional<Hash> expectedCAHash = {});
 
     /**
      * Like addToStore(), but the contents of the path are contained
@@ -454,8 +454,8 @@ public:
      * \todo remove?
      */
     virtual StorePath addToStoreFromDump(Source & dump, std::string_view name,
-        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair,
-        const StorePathSet & references = StorePathSet())
+                                         FileIngestionMethod method = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, RepairFlag repair = NoRepair,
+                                         const StorePathSet & references = StorePathSet())
     { unsupported("addToStoreFromDump"); }
 
     /**
diff --git a/src/libstore/store-dir-config.hh b/src/libstore/store-dir-config.hh
index 53843d663..8dafca096 100644
--- a/src/libstore/store-dir-config.hh
+++ b/src/libstore/store-dir-config.hh
@@ -98,7 +98,7 @@ struct StoreDirConfig : public Config
         Source & dump,
         std::string_view name,
         FileIngestionMethod method = FileIngestionMethod::Recursive,
-        HashType hashAlgo = htSHA256,
+        HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
         const StorePathSet & references = {}) const;
 
     /**
diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc
index 43654d7e8..2a379e75e 100644
--- a/src/libstore/worker-protocol.cc
+++ b/src/libstore/worker-protocol.cc
@@ -160,7 +160,7 @@ void WorkerProto::Serialise<ValidPathInfo>::write(const StoreDirConfig & store,
 UnkeyedValidPathInfo WorkerProto::Serialise<UnkeyedValidPathInfo>::read(const StoreDirConfig & store, ReadConn conn)
 {
     auto deriver = readString(conn.from);
-    auto narHash = Hash::parseAny(readString(conn.from), htSHA256);
+    auto narHash = Hash::parseAny(readString(conn.from), HashAlgorithm::SHA256);
     UnkeyedValidPathInfo info(narHash);
     if (deriver != "") info.deriver = store.parseStorePath(deriver);
     info.references = WorkerProto::Serialise<StorePathSet>::read(store, conn);
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index c4b2975ee..ac3727d11 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -546,32 +546,32 @@ nlohmann::json Args::toJSON()
 
 static void hashTypeCompleter(AddCompletions & completions, size_t index, std::string_view prefix)
 {
-    for (auto & type : hashTypes)
+    for (auto & type : hashAlgorithms)
         if (hasPrefix(type, prefix))
             completions.add(type);
 }
 
-Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht)
+Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashAlgorithm * ha)
 {
     return Flag {
         .longName = std::move(longName),
         .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')",
         .labels = {"hash-algo"},
-        .handler = {[ht](std::string s) {
-            *ht = parseHashType(s);
+        .handler = {[ha](std::string s) {
+            *ha = parseHashAlgo(s);
         }},
         .completer = hashTypeCompleter,
     };
 }
 
-Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional<HashType> * oht)
+Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional<HashAlgorithm> * oha)
 {
     return Flag {
         .longName = std::move(longName),
         .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512'). Optional as can also be gotten from SRI hash itself.",
         .labels = {"hash-algo"},
-        .handler = {[oht](std::string s) {
-            *oht = std::optional<HashType> { parseHashType(s) };
+        .handler = {[oha](std::string s) {
+            *oha = std::optional<HashAlgorithm> {parseHashAlgo(s) };
         }},
         .completer = hashTypeCompleter,
     };
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 72278dccc..0cff76158 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -14,7 +14,7 @@
 
 namespace nix {
 
-enum HashType : char;
+enum struct HashAlgorithm : char;
 
 class MultiCommand;
 
@@ -175,8 +175,8 @@ protected:
 
         std::optional<ExperimentalFeature> experimentalFeature;
 
-        static Flag mkHashTypeFlag(std::string && longName, HashType * ht);
-        static Flag mkHashTypeOptFlag(std::string && longName, std::optional<HashType> * oht);
+        static Flag mkHashTypeFlag(std::string && longName, HashAlgorithm * ha);
+        static Flag mkHashTypeOptFlag(std::string && longName, std::optional<HashAlgorithm> * oha);
     };
 
     /**
diff --git a/src/libutil/git.cc b/src/libutil/git.cc
index a4bd60096..296b75628 100644
--- a/src/libutil/git.cc
+++ b/src/libutil/git.cc
@@ -106,7 +106,7 @@ void parse(
             std::string hashs = getString(source, 20);
             left -= 20;
 
-            Hash hash(htSHA1);
+            Hash hash(HashAlgorithm::SHA1);
             std::copy(hashs.begin(), hashs.end(), hash.hash);
 
             hook(name, TreeEntry {
@@ -241,12 +241,12 @@ Mode dump(
 
 
 TreeEntry dumpHash(
-    HashType ht,
-    SourceAccessor & accessor, const CanonPath & path, PathFilter & filter)
+        HashAlgorithm ha,
+        SourceAccessor & accessor, const CanonPath & path, PathFilter & filter)
 {
     std::function<DumpHook> hook;
     hook = [&](const CanonPath & path) -> TreeEntry {
-        auto hashSink = HashSink(ht);
+        auto hashSink = HashSink(ha);
         auto mode = dump(accessor, path, hashSink, hook, filter);
         auto hash = hashSink.finish().first;
         return {
diff --git a/src/libutil/git.hh b/src/libutil/git.hh
index 303460072..b24b25dd3 100644
--- a/src/libutil/git.hh
+++ b/src/libutil/git.hh
@@ -123,9 +123,9 @@ Mode dump(
  * A smaller wrapper around `dump`.
  */
 TreeEntry dumpHash(
-    HashType ht,
-    SourceAccessor & accessor, const CanonPath & path,
-    PathFilter & filter = defaultPathFilter);
+            HashAlgorithm ha,
+            SourceAccessor & accessor, const CanonPath & path,
+            PathFilter & filter = defaultPathFilter);
 
 /**
  * A line from the output of `git ls-remote --symref`.
diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index 144f7ae7e..38a29c459 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -16,23 +16,23 @@
 
 namespace nix {
 
-static size_t regularHashSize(HashType type) {
+static size_t regularHashSize(HashAlgorithm type) {
     switch (type) {
-    case htMD5: return md5HashSize;
-    case htSHA1: return sha1HashSize;
-    case htSHA256: return sha256HashSize;
-    case htSHA512: return sha512HashSize;
+    case HashAlgorithm::MD5: return md5HashSize;
+    case HashAlgorithm::SHA1: return sha1HashSize;
+    case HashAlgorithm::SHA256: return sha256HashSize;
+    case HashAlgorithm::SHA512: return sha512HashSize;
     }
     abort();
 }
 
 
-std::set<std::string> hashTypes = { "md5", "sha1", "sha256", "sha512" };
+std::set<std::string> hashAlgorithms = {"md5", "sha1", "sha256", "sha512" };
 
 
-Hash::Hash(HashType type) : type(type)
+Hash::Hash(HashAlgorithm algo) : algo(algo)
 {
-    hashSize = regularHashSize(type);
+    hashSize = regularHashSize(algo);
     assert(hashSize <= maxHashSize);
     memset(hash, 0, maxHashSize);
 }
@@ -109,16 +109,16 @@ static std::string printHash32(const Hash & hash)
 
 std::string printHash16or32(const Hash & hash)
 {
-    assert(hash.type);
-    return hash.to_string(hash.type == htMD5 ? HashFormat::Base16 : HashFormat::Base32, false);
+    assert(static_cast<char>(hash.algo));
+    return hash.to_string(hash.algo == HashAlgorithm::MD5 ? HashFormat::Base16 : HashFormat::Base32, false);
 }
 
 
-std::string Hash::to_string(HashFormat hashFormat, bool includeType) const
+std::string Hash::to_string(HashFormat hashFormat, bool includeAlgo) const
 {
     std::string s;
-    if (hashFormat == HashFormat::SRI || includeType) {
-        s += printHashType(type);
+    if (hashFormat == HashFormat::SRI || includeAlgo) {
+        s += printHashAlgo(algo);
         s += hashFormat == HashFormat::SRI ? '-' : ':';
     }
     switch (hashFormat) {
@@ -136,7 +136,7 @@ std::string Hash::to_string(HashFormat hashFormat, bool includeType) const
     return s;
 }
 
-Hash Hash::dummy(htSHA256);
+Hash Hash::dummy(HashAlgorithm::SHA256);
 
 Hash Hash::parseSRI(std::string_view original) {
     auto rest = original;
@@ -145,18 +145,18 @@ Hash Hash::parseSRI(std::string_view original) {
     auto hashRaw = splitPrefixTo(rest, '-');
     if (!hashRaw)
         throw BadHash("hash '%s' is not SRI", original);
-    HashType parsedType = parseHashType(*hashRaw);
+    HashAlgorithm parsedType = parseHashAlgo(*hashRaw);
 
     return Hash(rest, parsedType, true);
 }
 
 // Mutates the string to eliminate the prefixes when found
-static std::pair<std::optional<HashType>, bool> getParsedTypeAndSRI(std::string_view & rest)
+static std::pair<std::optional<HashAlgorithm>, bool> getParsedTypeAndSRI(std::string_view & rest)
 {
     bool isSRI = false;
 
     // Parse the hash type before the separator, if there was one.
-    std::optional<HashType> optParsedType;
+    std::optional<HashAlgorithm> optParsedType;
     {
         auto hashRaw = splitPrefixTo(rest, ':');
 
@@ -166,7 +166,7 @@ static std::pair<std::optional<HashType>, bool> getParsedTypeAndSRI(std::string_
                 isSRI = true;
         }
         if (hashRaw)
-            optParsedType = parseHashType(*hashRaw);
+            optParsedType = parseHashAlgo(*hashRaw);
     }
 
     return {optParsedType, isSRI};
@@ -185,29 +185,29 @@ Hash Hash::parseAnyPrefixed(std::string_view original)
     return Hash(rest, *optParsedType, isSRI);
 }
 
-Hash Hash::parseAny(std::string_view original, std::optional<HashType> optType)
+Hash Hash::parseAny(std::string_view original, std::optional<HashAlgorithm> optAlgo)
 {
     auto rest = original;
     auto [optParsedType, isSRI] = getParsedTypeAndSRI(rest);
 
     // Either the string or user must provide the type, if they both do they
     // must agree.
-    if (!optParsedType && !optType)
+    if (!optParsedType && !optAlgo)
         throw BadHash("hash '%s' does not include a type, nor is the type otherwise known from context", rest);
-    else if (optParsedType && optType && *optParsedType != *optType)
-        throw BadHash("hash '%s' should have type '%s'", original, printHashType(*optType));
+    else if (optParsedType && optAlgo && *optParsedType != *optAlgo)
+        throw BadHash("hash '%s' should have type '%s'", original, printHashAlgo(*optAlgo));
 
-    HashType hashType = optParsedType ? *optParsedType : *optType;
-    return Hash(rest, hashType, isSRI);
+    HashAlgorithm hashAlgo = optParsedType ? *optParsedType : *optAlgo;
+    return Hash(rest, hashAlgo, isSRI);
 }
 
-Hash Hash::parseNonSRIUnprefixed(std::string_view s, HashType type)
+Hash Hash::parseNonSRIUnprefixed(std::string_view s, HashAlgorithm algo)
 {
-    return Hash(s, type, false);
+    return Hash(s, algo, false);
 }
 
-Hash::Hash(std::string_view rest, HashType type, bool isSRI)
-    : Hash(type)
+Hash::Hash(std::string_view rest, HashAlgorithm algo, bool isSRI)
+    : Hash(algo)
 {
     if (!isSRI && rest.size() == base16Len()) {
 
@@ -257,19 +257,19 @@ Hash::Hash(std::string_view rest, HashType type, bool isSRI)
     }
 
     else
-        throw BadHash("hash '%s' has wrong length for hash type '%s'", rest, printHashType(this->type));
+        throw BadHash("hash '%s' has wrong length for hash algorithm '%s'", rest, printHashAlgo(this->algo));
 }
 
-Hash newHashAllowEmpty(std::string_view hashStr, std::optional<HashType> ht)
+Hash newHashAllowEmpty(std::string_view hashStr, std::optional<HashAlgorithm> ha)
 {
     if (hashStr.empty()) {
-        if (!ht)
+        if (!ha)
             throw BadHash("empty hash requires explicit hash type");
-        Hash h(*ht);
+        Hash h(*ha);
         warn("found empty hash, assuming '%s'", h.to_string(HashFormat::SRI, true));
         return h;
     } else
-        return Hash::parseAny(hashStr, ht);
+        return Hash::parseAny(hashStr, ha);
 }
 
 
@@ -282,58 +282,58 @@ union Ctx
 };
 
 
-static void start(HashType ht, Ctx & ctx)
+static void start(HashAlgorithm ha, Ctx & ctx)
 {
-    if (ht == htMD5) MD5_Init(&ctx.md5);
-    else if (ht == htSHA1) SHA1_Init(&ctx.sha1);
-    else if (ht == htSHA256) SHA256_Init(&ctx.sha256);
-    else if (ht == htSHA512) SHA512_Init(&ctx.sha512);
+    if (ha == HashAlgorithm::MD5) MD5_Init(&ctx.md5);
+    else if (ha == HashAlgorithm::SHA1) SHA1_Init(&ctx.sha1);
+    else if (ha == HashAlgorithm::SHA256) SHA256_Init(&ctx.sha256);
+    else if (ha == HashAlgorithm::SHA512) SHA512_Init(&ctx.sha512);
 }
 
 
-static void update(HashType ht, Ctx & ctx,
-    std::string_view data)
+static void update(HashAlgorithm ha, Ctx & ctx,
+                   std::string_view data)
 {
-    if (ht == htMD5) MD5_Update(&ctx.md5, data.data(), data.size());
-    else if (ht == htSHA1) SHA1_Update(&ctx.sha1, data.data(), data.size());
-    else if (ht == htSHA256) SHA256_Update(&ctx.sha256, data.data(), data.size());
-    else if (ht == htSHA512) SHA512_Update(&ctx.sha512, data.data(), data.size());
+    if (ha == HashAlgorithm::MD5) MD5_Update(&ctx.md5, data.data(), data.size());
+    else if (ha == HashAlgorithm::SHA1) SHA1_Update(&ctx.sha1, data.data(), data.size());
+    else if (ha == HashAlgorithm::SHA256) SHA256_Update(&ctx.sha256, data.data(), data.size());
+    else if (ha == HashAlgorithm::SHA512) SHA512_Update(&ctx.sha512, data.data(), data.size());
 }
 
 
-static void finish(HashType ht, Ctx & ctx, unsigned char * hash)
+static void finish(HashAlgorithm ha, Ctx & ctx, unsigned char * hash)
 {
-    if (ht == htMD5) MD5_Final(hash, &ctx.md5);
-    else if (ht == htSHA1) SHA1_Final(hash, &ctx.sha1);
-    else if (ht == htSHA256) SHA256_Final(hash, &ctx.sha256);
-    else if (ht == htSHA512) SHA512_Final(hash, &ctx.sha512);
+    if (ha == HashAlgorithm::MD5) MD5_Final(hash, &ctx.md5);
+    else if (ha == HashAlgorithm::SHA1) SHA1_Final(hash, &ctx.sha1);
+    else if (ha == HashAlgorithm::SHA256) SHA256_Final(hash, &ctx.sha256);
+    else if (ha == HashAlgorithm::SHA512) SHA512_Final(hash, &ctx.sha512);
 }
 
 
-Hash hashString(HashType ht, std::string_view s)
+Hash hashString(HashAlgorithm ha, std::string_view s)
 {
     Ctx ctx;
-    Hash hash(ht);
-    start(ht, ctx);
-    update(ht, ctx, s);
-    finish(ht, ctx, hash.hash);
+    Hash hash(ha);
+    start(ha, ctx);
+    update(ha, ctx, s);
+    finish(ha, ctx, hash.hash);
     return hash;
 }
 
 
-Hash hashFile(HashType ht, const Path & path)
+Hash hashFile(HashAlgorithm ha, const Path & path)
 {
-    HashSink sink(ht);
+    HashSink sink(ha);
     readFile(path, sink);
     return sink.finish().first;
 }
 
 
-HashSink::HashSink(HashType ht) : ht(ht)
+HashSink::HashSink(HashAlgorithm ha) : ha(ha)
 {
     ctx = new Ctx;
     bytes = 0;
-    start(ht, *ctx);
+    start(ha, *ctx);
 }
 
 HashSink::~HashSink()
@@ -345,14 +345,14 @@ HashSink::~HashSink()
 void HashSink::writeUnbuffered(std::string_view data)
 {
     bytes += data.size();
-    update(ht, *ctx, data);
+    update(ha, *ctx, data);
 }
 
 HashResult HashSink::finish()
 {
     flush();
-    Hash hash(ht);
-    nix::finish(ht, *ctx, hash.hash);
+    Hash hash(ha);
+    nix::finish(ha, *ctx, hash.hash);
     return HashResult(hash, bytes);
 }
 
@@ -360,16 +360,16 @@ HashResult HashSink::currentHash()
 {
     flush();
     Ctx ctx2 = *ctx;
-    Hash hash(ht);
-    nix::finish(ht, ctx2, hash.hash);
+    Hash hash(ha);
+    nix::finish(ha, ctx2, hash.hash);
     return HashResult(hash, bytes);
 }
 
 
 HashResult hashPath(
-    HashType ht, const Path & path, PathFilter & filter)
+        HashAlgorithm ha, const Path & path, PathFilter & filter)
 {
-    HashSink sink(ht);
+    HashSink sink(ha);
     dumpPath(path, sink, filter);
     return sink.finish();
 }
@@ -377,7 +377,7 @@ HashResult hashPath(
 
 Hash compressHash(const Hash & hash, unsigned int newSize)
 {
-    Hash h(hash.type);
+    Hash h(hash.algo);
     h.hashSize = newSize;
     for (unsigned int i = 0; i < hash.hashSize; ++i)
         h.hash[i % newSize] ^= hash.hash[i];
@@ -420,31 +420,31 @@ std::string_view printHashFormat(HashFormat HashFormat)
     }
 }
 
-std::optional<HashType> parseHashTypeOpt(std::string_view s)
+std::optional<HashAlgorithm> parseHashAlgoOpt(std::string_view s)
 {
-    if (s == "md5") return htMD5;
-    if (s == "sha1") return htSHA1;
-    if (s == "sha256") return htSHA256;
-    if (s == "sha512") return htSHA512;
+    if (s == "md5") return HashAlgorithm::MD5;
+    if (s == "sha1") return HashAlgorithm::SHA1;
+    if (s == "sha256") return HashAlgorithm::SHA256;
+    if (s == "sha512") return HashAlgorithm::SHA512;
     return std::nullopt;
 }
 
-HashType parseHashType(std::string_view s)
+HashAlgorithm parseHashAlgo(std::string_view s)
 {
-    auto opt_h = parseHashTypeOpt(s);
+    auto opt_h = parseHashAlgoOpt(s);
     if (opt_h)
         return *opt_h;
     else
         throw UsageError("unknown hash algorithm '%1%', expect 'md5', 'sha1', 'sha256', or 'sha512'", s);
 }
 
-std::string_view printHashType(HashType ht)
+std::string_view printHashAlgo(HashAlgorithm ha)
 {
-    switch (ht) {
-    case htMD5: return "md5";
-    case htSHA1: return "sha1";
-    case htSHA256: return "sha256";
-    case htSHA512: return "sha512";
+    switch (ha) {
+    case HashAlgorithm::MD5: return "md5";
+    case HashAlgorithm::SHA1: return "sha1";
+    case HashAlgorithm::SHA256: return "sha256";
+    case HashAlgorithm::SHA512: return "sha512";
     default:
         // illegal hash type enum value internally, as opposed to external input
         // which should be validated with nice error message.
diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh
index 820154e7a..3c97ed4b1 100644
--- a/src/libutil/hash.hh
+++ b/src/libutil/hash.hh
@@ -12,7 +12,7 @@ namespace nix {
 MakeError(BadHash, Error);
 
 
-enum HashType : char { htMD5 = 42, htSHA1, htSHA256, htSHA512 };
+enum struct HashAlgorithm : char { MD5 = 42, SHA1, SHA256, SHA512 };
 
 
 const int md5HashSize = 16;
@@ -20,7 +20,7 @@ const int sha1HashSize = 20;
 const int sha256HashSize = 32;
 const int sha512HashSize = 64;
 
-extern std::set<std::string> hashTypes;
+extern std::set<std::string> hashAlgorithms;
 
 extern const std::string base32Chars;
 
@@ -46,12 +46,12 @@ struct Hash
     size_t hashSize = 0;
     uint8_t hash[maxHashSize] = {};
 
-    HashType type;
+    HashAlgorithm algo;
 
     /**
      * Create a zero-filled hash object.
      */
-    explicit Hash(HashType type);
+    explicit Hash(HashAlgorithm algo);
 
     /**
      * Parse the hash from a string representation in the format
@@ -60,7 +60,7 @@ struct Hash
      * is not present, then the hash type must be specified in the
      * string.
      */
-    static Hash parseAny(std::string_view s, std::optional<HashType> type);
+    static Hash parseAny(std::string_view s, std::optional<HashAlgorithm> optAlgo);
 
     /**
      * Parse a hash from a string representation like the above, except the
@@ -72,7 +72,7 @@ struct Hash
      * Parse a plain hash that musst not have any prefix indicating the type.
      * The type is passed in to disambiguate.
      */
-    static Hash parseNonSRIUnprefixed(std::string_view s, HashType type);
+    static Hash parseNonSRIUnprefixed(std::string_view s, HashAlgorithm algo);
 
     static Hash parseSRI(std::string_view original);
 
@@ -81,7 +81,7 @@ private:
      * The type must be provided, the string view must not include <type>
      * prefix. `isSRI` helps disambigate the various base-* encodings.
      */
-    Hash(std::string_view s, HashType type, bool isSRI);
+    Hash(std::string_view s, HashAlgorithm algo, bool isSRI);
 
 public:
     /**
@@ -125,10 +125,10 @@ public:
 
     /**
      * Return a string representation of the hash, in base-16, base-32
-     * or base-64. By default, this is prefixed by the hash type
+     * or base-64. By default, this is prefixed by the hash algo
      * (e.g. "sha256:").
      */
-    [[nodiscard]] std::string to_string(HashFormat hashFormat, bool includeType) const;
+    [[nodiscard]] std::string to_string(HashFormat hashFormat, bool includeAlgo) const;
 
     [[nodiscard]] std::string gitRev() const
     {
@@ -146,7 +146,7 @@ public:
 /**
  * Helper that defaults empty hashes to the 0 hash.
  */
-Hash newHashAllowEmpty(std::string_view hashStr, std::optional<HashType> ht);
+Hash newHashAllowEmpty(std::string_view hashStr, std::optional<HashAlgorithm> ha);
 
 /**
  * Print a hash in base-16 if it's MD5, or base-32 otherwise.
@@ -156,14 +156,14 @@ std::string printHash16or32(const Hash & hash);
 /**
  * Compute the hash of the given string.
  */
-Hash hashString(HashType ht, std::string_view s);
+Hash hashString(HashAlgorithm ha, std::string_view s);
 
 /**
  * Compute the hash of the given file, hashing its contents directly.
  *
  * (Metadata, such as the executable permission bit, is ignored.)
  */
-Hash hashFile(HashType ht, const Path & path);
+Hash hashFile(HashAlgorithm ha, const Path & path);
 
 /**
  * Compute the hash of the given path, serializing as a Nix Archive and
@@ -172,8 +172,8 @@ Hash hashFile(HashType ht, const Path & path);
  * The hash is defined as (essentially) hashString(ht, dumpPath(path)).
  */
 typedef std::pair<Hash, uint64_t> HashResult;
-HashResult hashPath(HashType ht, const Path & path,
-    PathFilter & filter = defaultPathFilter);
+HashResult hashPath(HashAlgorithm ha, const Path & path,
+                    PathFilter & filter = defaultPathFilter);
 
 /**
  * Compress a hash to the specified number of bytes by cyclically
@@ -199,17 +199,17 @@ std::string_view printHashFormat(HashFormat hashFormat);
 /**
  * Parse a string representing a hash type.
  */
-HashType parseHashType(std::string_view s);
+HashAlgorithm parseHashAlgo(std::string_view s);
 
 /**
  * Will return nothing on parse error
  */
-std::optional<HashType> parseHashTypeOpt(std::string_view s);
+std::optional<HashAlgorithm> parseHashAlgoOpt(std::string_view s);
 
 /**
  * And the reverse.
  */
-std::string_view printHashType(HashType ht);
+std::string_view printHashAlgo(HashAlgorithm ha);
 
 
 union Ctx;
@@ -222,12 +222,12 @@ struct AbstractHashSink : virtual Sink
 class HashSink : public BufferedSink, public AbstractHashSink
 {
 private:
-    HashType ht;
+    HashAlgorithm ha;
     Ctx * ctx;
     uint64_t bytes;
 
 public:
-    HashSink(HashType ht);
+    HashSink(HashAlgorithm ha);
     HashSink(const HashSink & h);
     ~HashSink();
     void writeUnbuffered(std::string_view data) override;
diff --git a/src/libutil/references.cc b/src/libutil/references.cc
index 9d75606ef..d82d51945 100644
--- a/src/libutil/references.cc
+++ b/src/libutil/references.cc
@@ -110,8 +110,8 @@ void RewritingSink::flush()
     prev.clear();
 }
 
-HashModuloSink::HashModuloSink(HashType ht, const std::string & modulus)
-    : hashSink(ht)
+HashModuloSink::HashModuloSink(HashAlgorithm ha, const std::string & modulus)
+    : hashSink(ha)
     , rewritingSink(modulus, std::string(modulus.size(), 0), hashSink)
 {
 }
diff --git a/src/libutil/references.hh b/src/libutil/references.hh
index f0baeffe1..8bc9f7ec9 100644
--- a/src/libutil/references.hh
+++ b/src/libutil/references.hh
@@ -46,7 +46,7 @@ struct HashModuloSink : AbstractHashSink
     HashSink hashSink;
     RewritingSink rewritingSink;
 
-    HashModuloSink(HashType ht, const std::string & modulus);
+    HashModuloSink(HashAlgorithm ha, const std::string & modulus);
 
     void operator () (std::string_view data) override;
 
diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc
index 7813433a7..afbbbe1a9 100644
--- a/src/libutil/source-accessor.cc
+++ b/src/libutil/source-accessor.cc
@@ -39,11 +39,11 @@ void SourceAccessor::readFile(
 }
 
 Hash SourceAccessor::hashPath(
-    const CanonPath & path,
-    PathFilter & filter,
-    HashType ht)
+        const CanonPath & path,
+        PathFilter & filter,
+        HashAlgorithm ha)
 {
-    HashSink sink(ht);
+    HashSink sink(ha);
     dumpPath(path, sink, filter);
     return sink.finish().first;
 }
diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh
index 264caab16..3ca12d624 100644
--- a/src/libutil/source-accessor.hh
+++ b/src/libutil/source-accessor.hh
@@ -97,9 +97,9 @@ struct SourceAccessor
         PathFilter & filter = defaultPathFilter);
 
     Hash hashPath(
-        const CanonPath & path,
-        PathFilter & filter = defaultPathFilter,
-        HashType ht = htSHA256);
+            const CanonPath & path,
+            PathFilter & filter = defaultPathFilter,
+            HashAlgorithm ha = HashAlgorithm::SHA256);
 
     /**
      * Return a corresponding path in the root filesystem, if
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 25f0107bc..75ad4e75f 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -193,7 +193,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs)
     if (opArgs.empty())
         throw UsageError("first argument must be hash algorithm");
 
-    HashType hashAlgo = parseHashType(opArgs.front());
+    HashAlgorithm hashAlgo = parseHashAlgo(opArgs.front());
     opArgs.pop_front();
 
     for (auto & i : opArgs)
@@ -214,7 +214,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs)
         throw UsageError("'--print-fixed-path' requires three arguments");
 
     Strings::iterator i = opArgs.begin();
-    HashType hashAlgo = parseHashType(*i++);
+    HashAlgorithm hashAlgo = parseHashAlgo(*i++);
     std::string hash = *i++;
     std::string name = *i++;
 
@@ -405,7 +405,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
                 for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) {
                     auto info = store->queryPathInfo(j);
                     if (query == qHash) {
-                        assert(info->narHash.type == htSHA256);
+                        assert(info->narHash.algo == HashAlgorithm::SHA256);
                         cout << fmt("%s\n", info->narHash.to_string(HashFormat::Base32, true));
                     } else if (query == qSize)
                         cout << fmt("%d\n", info->narSize);
@@ -541,7 +541,7 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise)
             if (canonicalise)
                 canonicalisePathMetaData(store->printStorePath(info->path), {});
             if (!hashGiven) {
-                HashResult hash = hashPath(htSHA256, store->printStorePath(info->path));
+                HashResult hash = hashPath(HashAlgorithm::SHA256, store->printStorePath(info->path));
                 info->narHash = hash.first;
                 info->narSize = hash.second;
             }
@@ -763,7 +763,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs)
         auto path = store->followLinksToStorePath(i);
         printMsg(lvlTalkative, "checking path '%s'...", store->printStorePath(path));
         auto info = store->queryPathInfo(path);
-        HashSink sink(info->narHash.type);
+        HashSink sink(info->narHash.algo);
         store->narFromPath(path, sink);
         auto current = sink.finish();
         if (current.first != info->narHash) {
@@ -979,7 +979,7 @@ static void opServe(Strings opFlags, Strings opArgs)
                 auto deriver = readString(in);
                 ValidPathInfo info {
                     store->parseStorePath(path),
-                    Hash::parseAny(readString(in), htSHA256),
+                    Hash::parseAny(readString(in), HashAlgorithm::SHA256),
                 };
                 if (deriver != "")
                     info.deriver = store->parseStorePath(deriver);
diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index f9d487ada..02de796b5 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -60,11 +60,11 @@ struct CmdAddToStore : MixDryRun, StoreCommand
         StringSink sink;
         dumpPath(path, sink);
 
-        auto narHash = hashString(htSHA256, sink.s);
+        auto narHash = hashString(HashAlgorithm::SHA256, sink.s);
 
         Hash hash = narHash;
         if (ingestionMethod == FileIngestionMethod::Flat) {
-            HashSink hsink(htSHA256);
+            HashSink hsink(HashAlgorithm::SHA256);
             readFile(path, hsink);
             hash = hsink.finish().first;
         }
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index 62f96ef1d..638178afa 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -18,7 +18,7 @@ struct CmdHashBase : Command
     FileIngestionMethod mode;
     HashFormat hashFormat = HashFormat::SRI;
     bool truncate = false;
-    HashType ht = htSHA256;
+    HashAlgorithm ha = HashAlgorithm::SHA256;
     std::vector<std::string> paths;
     std::optional<std::string> modulus;
 
@@ -48,7 +48,7 @@ struct CmdHashBase : Command
             .handler = {&hashFormat, HashFormat::Base16},
         });
 
-        addFlag(Flag::mkHashTypeFlag("type", &ht));
+        addFlag(Flag::mkHashTypeFlag("type", &ha));
 
         #if 0
         addFlag({
@@ -84,9 +84,9 @@ struct CmdHashBase : Command
 
             std::unique_ptr<AbstractHashSink> hashSink;
             if (modulus)
-                hashSink = std::make_unique<HashModuloSink>(ht, *modulus);
+                hashSink = std::make_unique<HashModuloSink>(ha, *modulus);
             else
-                hashSink = std::make_unique<HashSink>(ht);
+                hashSink = std::make_unique<HashSink>(ha);
 
             switch (mode) {
             case FileIngestionMethod::Flat:
@@ -107,7 +107,7 @@ struct CmdHashBase : Command
 struct CmdToBase : Command
 {
     HashFormat hashFormat;
-    std::optional<HashType> ht;
+    std::optional<HashAlgorithm> ht;
     std::vector<std::string> args;
 
     CmdToBase(HashFormat hashFormat) : hashFormat(hashFormat)
@@ -139,7 +139,7 @@ struct CmdHashConvert : Command
 {
     std::optional<HashFormat> from;
     HashFormat to;
-    std::optional<HashType> type;
+    std::optional<HashAlgorithm> type;
     std::vector<std::string> hashStrings;
 
     CmdHashConvert(): to(HashFormat::SRI) {
@@ -166,7 +166,7 @@ struct CmdHashConvert : Command
             .description = "Specify the algorithm if it can't be auto-detected.",
             .labels = {"hash algorithm"},
             .handler = {[this](std::string str) {
-                type = parseHashType(str);
+                type = parseHashAlgo(str);
             }},
         });
         expectArgs({
@@ -223,7 +223,7 @@ static auto rCmdHash = registerCommand<CmdHash>("hash");
 /* Legacy nix-hash command. */
 static int compatNixHash(int argc, char * * argv)
 {
-    std::optional<HashType> ht;
+    std::optional<HashAlgorithm> ha;
     bool flat = false;
     HashFormat hashFormat = HashFormat::Base16;
     bool truncate = false;
@@ -243,7 +243,7 @@ static int compatNixHash(int argc, char * * argv)
         else if (*arg == "--truncate") truncate = true;
         else if (*arg == "--type") {
             std::string s = getArg(*arg, arg, end);
-            ht = parseHashType(s);
+            ha = parseHashAlgo(s);
         }
         else if (*arg == "--to-base16") {
             op = opTo;
@@ -270,8 +270,8 @@ static int compatNixHash(int argc, char * * argv)
 
     if (op == opHash) {
         CmdHashBase cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive);
-        if (!ht.has_value()) ht = htMD5;
-        cmd.ht = ht.value();
+        if (!ha.has_value()) ha = HashAlgorithm::MD5;
+        cmd.ha = ha.value();
         cmd.hashFormat = hashFormat;
         cmd.truncate = truncate;
         cmd.paths = ss;
@@ -281,7 +281,7 @@ static int compatNixHash(int argc, char * * argv)
     else {
         CmdToBase cmd(hashFormat);
         cmd.args = ss;
-        if (ht.has_value()) cmd.ht = ht;
+        if (ha.has_value()) cmd.ht = ha;
         cmd.run();
     }
 
diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc
index 3ed7946a8..09f33a51e 100644
--- a/src/nix/prefetch.cc
+++ b/src/nix/prefetch.cc
@@ -46,13 +46,13 @@ std::string resolveMirrorUrl(EvalState & state, const std::string & url)
 }
 
 std::tuple<StorePath, Hash> prefetchFile(
-    ref<Store> store,
-    std::string_view url,
-    std::optional<std::string> name,
-    HashType hashType,
-    std::optional<Hash> expectedHash,
-    bool unpack,
-    bool executable)
+        ref<Store> store,
+        std::string_view url,
+        std::optional<std::string> name,
+        HashAlgorithm hashAlgo,
+        std::optional<Hash> expectedHash,
+        bool unpack,
+        bool executable)
 {
     auto ingestionMethod = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
 
@@ -69,7 +69,7 @@ std::tuple<StorePath, Hash> prefetchFile(
     /* If an expected hash is given, the file may already exist in
        the store. */
     if (expectedHash) {
-        hashType = expectedHash->type;
+        hashAlgo = expectedHash->algo;
         storePath = store->makeFixedOutputPath(*name, FixedOutputInfo {
             .method = ingestionMethod,
             .hash = *expectedHash,
@@ -122,7 +122,7 @@ std::tuple<StorePath, Hash> prefetchFile(
         Activity act(*logger, lvlChatty, actUnknown,
             fmt("adding '%s' to the store", url));
 
-        auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash);
+        auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashAlgo, expectedHash);
         storePath = info.path;
         assert(info.ca);
         hash = info.ca->hash;
@@ -134,7 +134,7 @@ std::tuple<StorePath, Hash> prefetchFile(
 static int main_nix_prefetch_url(int argc, char * * argv)
 {
     {
-        HashType ht = htSHA256;
+        HashAlgorithm ha = HashAlgorithm::SHA256;
         std::vector<std::string> args;
         bool printPath = getEnv("PRINT_PATH") == "1";
         bool fromExpr = false;
@@ -155,7 +155,7 @@ static int main_nix_prefetch_url(int argc, char * * argv)
                 printVersion("nix-prefetch-url");
             else if (*arg == "--type") {
                 auto s = getArg(*arg, arg, end);
-                ht = parseHashType(s);
+                ha = parseHashAlgo(s);
             }
             else if (*arg == "--print-path")
                 printPath = true;
@@ -233,10 +233,10 @@ static int main_nix_prefetch_url(int argc, char * * argv)
 
         std::optional<Hash> expectedHash;
         if (args.size() == 2)
-            expectedHash = Hash::parseAny(args[1], ht);
+            expectedHash = Hash::parseAny(args[1], ha);
 
         auto [storePath, hash] = prefetchFile(
-            store, resolveMirrorUrl(*state, url), name, ht, expectedHash, unpack, executable);
+            store, resolveMirrorUrl(*state, url), name, ha, expectedHash, unpack, executable);
 
         stopProgressBar();
 
@@ -258,7 +258,7 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON
     std::string url;
     bool executable = false;
     std::optional<std::string> name;
-    HashType hashType = htSHA256;
+    HashAlgorithm hashAlgo = HashAlgorithm::SHA256;
     std::optional<Hash> expectedHash;
 
     CmdStorePrefetchFile()
@@ -275,11 +275,11 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON
             .description = "The expected hash of the file.",
             .labels = {"hash"},
             .handler = {[&](std::string s) {
-                expectedHash = Hash::parseAny(s, hashType);
+                expectedHash = Hash::parseAny(s, hashAlgo);
             }}
         });
 
-        addFlag(Flag::mkHashTypeFlag("hash-type", &hashType));
+        addFlag(Flag::mkHashTypeFlag("hash-type", &hashAlgo));
 
         addFlag({
             .longName = "executable",
@@ -305,7 +305,7 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON
     }
     void run(ref<Store> store) override
     {
-        auto [storePath, hash] = prefetchFile(store, url, name, hashType, expectedHash, false, executable);
+        auto [storePath, hash] = prefetchFile(store, url, name, hashAlgo, expectedHash, false, executable);
 
         if (json) {
             auto res = nlohmann::json::object();
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index 147b4680b..9d9492da9 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -216,7 +216,7 @@ struct ProfileManifest
         StringSink sink;
         dumpPath(tempDir, sink);
 
-        auto narHash = hashString(htSHA256, sink.s);
+        auto narHash = hashString(HashAlgorithm::SHA256, sink.s);
 
         ValidPathInfo info {
             *store,
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index 78cb765ce..cd0f6d95f 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -98,7 +98,7 @@ struct CmdVerify : StorePathsCommand
 
                 if (!noContents) {
 
-                    auto hashSink = HashSink(info->narHash.type);
+                    auto hashSink = HashSink(info->narHash.algo);
 
                     store->narFromPath(info->path, hashSink);
 
diff --git a/tests/unit/libstore/common-protocol.cc b/tests/unit/libstore/common-protocol.cc
index c09ac6a3e..d23805fc3 100644
--- a/tests/unit/libstore/common-protocol.cc
+++ b/tests/unit/libstore/common-protocol.cc
@@ -84,15 +84,15 @@ CHARACTERIZATION_TEST(
     (std::tuple<ContentAddress, ContentAddress, ContentAddress> {
         ContentAddress {
             .method = TextIngestionMethod {},
-            .hash = hashString(HashType::htSHA256, "Derive(...)"),
+            .hash = hashString(HashAlgorithm::SHA256, "Derive(...)"),
         },
         ContentAddress {
             .method = FileIngestionMethod::Flat,
-            .hash = hashString(HashType::htSHA1, "blob blob..."),
+            .hash = hashString(HashAlgorithm::SHA1, "blob blob..."),
         },
         ContentAddress {
             .method = FileIngestionMethod::Recursive,
-            .hash = hashString(HashType::htSHA256, "(...)"),
+            .hash = hashString(HashAlgorithm::SHA256, "(...)"),
         },
     }))
 
@@ -179,7 +179,7 @@ CHARACTERIZATION_TEST(
         std::optional {
             ContentAddress {
                 .method = FileIngestionMethod::Flat,
-                .hash = hashString(HashType::htSHA1, "blob blob..."),
+                .hash = hashString(HashAlgorithm::SHA1, "blob blob..."),
             },
         },
     }))
diff --git a/tests/unit/libstore/derivation.cc b/tests/unit/libstore/derivation.cc
index a7f4488fa..7a4b1403a 100644
--- a/tests/unit/libstore/derivation.cc
+++ b/tests/unit/libstore/derivation.cc
@@ -134,7 +134,7 @@ TEST_JSON(DynDerivationTest, caFixedText,
 TEST_JSON(CaDerivationTest, caFloating,
     (DerivationOutput::CAFloating {
         .method = FileIngestionMethod::Recursive,
-        .hashType = htSHA256,
+        .hashAlgo = HashAlgorithm::SHA256,
     }),
     "drv-name", "output-name")
 
@@ -145,7 +145,7 @@ TEST_JSON(DerivationTest, deferred,
 TEST_JSON(ImpureDerivationTest, impure,
     (DerivationOutput::Impure {
         .method = FileIngestionMethod::Recursive,
-        .hashType = htSHA256,
+        .hashAlgo = HashAlgorithm::SHA256,
     }),
     "drv-name", "output-name")
 
diff --git a/tests/unit/libstore/nar-info.cc b/tests/unit/libstore/nar-info.cc
index 4f124e89e..bd10602e7 100644
--- a/tests/unit/libstore/nar-info.cc
+++ b/tests/unit/libstore/nar-info.cc
@@ -26,7 +26,7 @@ static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) {
         "foo",
         FixedOutputInfo {
             .method = FileIngestionMethod::Recursive,
-            .hash = hashString(HashType::htSHA256, "(...)"),
+            .hash = hashString(HashAlgorithm::SHA256, "(...)"),
 
             .references = {
                 .others = {
diff --git a/tests/unit/libstore/path-info.cc b/tests/unit/libstore/path-info.cc
index 18f00ca19..80d6fcfed 100644
--- a/tests/unit/libstore/path-info.cc
+++ b/tests/unit/libstore/path-info.cc
@@ -25,7 +25,7 @@ static UnkeyedValidPathInfo makePathInfo(const Store & store, bool includeImpure
         "foo",
         FixedOutputInfo {
             .method = FileIngestionMethod::Recursive,
-            .hash = hashString(HashType::htSHA256, "(...)"),
+            .hash = hashString(HashAlgorithm::SHA256, "(...)"),
 
             .references = {
                 .others = {
diff --git a/tests/unit/libstore/serve-protocol.cc b/tests/unit/libstore/serve-protocol.cc
index c8ac87a04..6d2054f7d 100644
--- a/tests/unit/libstore/serve-protocol.cc
+++ b/tests/unit/libstore/serve-protocol.cc
@@ -53,15 +53,15 @@ VERSIONED_CHARACTERIZATION_TEST(
     (std::tuple<ContentAddress, ContentAddress, ContentAddress> {
         ContentAddress {
             .method = TextIngestionMethod {},
-            .hash = hashString(HashType::htSHA256, "Derive(...)"),
+            .hash = hashString(HashAlgorithm::SHA256, "Derive(...)"),
         },
         ContentAddress {
             .method = FileIngestionMethod::Flat,
-            .hash = hashString(HashType::htSHA1, "blob blob..."),
+            .hash = hashString(HashAlgorithm::SHA1, "blob blob..."),
         },
         ContentAddress {
             .method = FileIngestionMethod::Recursive,
-            .hash = hashString(HashType::htSHA256, "(...)"),
+            .hash = hashString(HashAlgorithm::SHA256, "(...)"),
         },
     }))
 
@@ -271,7 +271,7 @@ VERSIONED_CHARACTERIZATION_TEST(
         std::optional {
             ContentAddress {
                 .method = FileIngestionMethod::Flat,
-                .hash = hashString(HashType::htSHA1, "blob blob..."),
+                .hash = hashString(HashAlgorithm::SHA1, "blob blob..."),
             },
         },
     }))
diff --git a/tests/unit/libstore/worker-protocol.cc b/tests/unit/libstore/worker-protocol.cc
index ad5943c69..91f804f0c 100644
--- a/tests/unit/libstore/worker-protocol.cc
+++ b/tests/unit/libstore/worker-protocol.cc
@@ -55,15 +55,15 @@ VERSIONED_CHARACTERIZATION_TEST(
     (std::tuple<ContentAddress, ContentAddress, ContentAddress> {
         ContentAddress {
             .method = TextIngestionMethod {},
-            .hash = hashString(HashType::htSHA256, "Derive(...)"),
+            .hash = hashString(HashAlgorithm::SHA256, "Derive(...)"),
         },
         ContentAddress {
             .method = FileIngestionMethod::Flat,
-            .hash = hashString(HashType::htSHA1, "blob blob..."),
+            .hash = hashString(HashAlgorithm::SHA1, "blob blob..."),
         },
         ContentAddress {
             .method = FileIngestionMethod::Recursive,
-            .hash = hashString(HashType::htSHA256, "(...)"),
+            .hash = hashString(HashAlgorithm::SHA256, "(...)"),
         },
     }))
 
@@ -464,7 +464,7 @@ VERSIONED_CHARACTERIZATION_TEST(
                 "foo",
                 FixedOutputInfo {
                     .method = FileIngestionMethod::Recursive,
-                    .hash = hashString(HashType::htSHA256, "(...)"),
+                    .hash = hashString(HashAlgorithm::SHA256, "(...)"),
                     .references = {
                         .others = {
                             StorePath {
@@ -539,7 +539,7 @@ VERSIONED_CHARACTERIZATION_TEST(
         std::optional {
             ContentAddress {
                 .method = FileIngestionMethod::Flat,
-                .hash = hashString(HashType::htSHA1, "blob blob..."),
+                .hash = hashString(HashAlgorithm::SHA1, "blob blob..."),
             },
         },
     }))
diff --git a/tests/unit/libutil/git.cc b/tests/unit/libutil/git.cc
index 551a2d105..141a55816 100644
--- a/tests/unit/libutil/git.cc
+++ b/tests/unit/libutil/git.cc
@@ -95,7 +95,7 @@ const static Tree tree = {
         {
             .mode = Mode::Regular,
             // hello world with special chars from above
-            .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", htSHA1),
+            .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", HashAlgorithm::SHA1),
         },
     },
     {
@@ -103,7 +103,7 @@ const static Tree tree = {
         {
             .mode = Mode::Executable,
             // ditto
-            .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", htSHA1),
+            .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", HashAlgorithm::SHA1),
         },
     },
     {
@@ -111,7 +111,7 @@ const static Tree tree = {
         {
             .mode = Mode::Directory,
             // Empty directory hash
-            .hash = Hash::parseAny("4b825dc642cb6eb9a060e54bf8d69288fbee4904", htSHA1),
+            .hash = Hash::parseAny("4b825dc642cb6eb9a060e54bf8d69288fbee4904", HashAlgorithm::SHA1),
         },
     },
 };
@@ -174,7 +174,7 @@ TEST_F(GitTest, both_roundrip) {
     std::function<DumpHook> dumpHook;
     dumpHook = [&](const CanonPath & path) {
         StringSink s;
-        HashSink hashSink { htSHA1 };
+        HashSink hashSink { HashAlgorithm::SHA1 };
         TeeSink s2 { s, hashSink };
         auto mode = dump(
             files, path, s2, dumpHook,
diff --git a/tests/unit/libutil/hash.cc b/tests/unit/libutil/hash.cc
index 92291afce..4d82c7f09 100644
--- a/tests/unit/libutil/hash.cc
+++ b/tests/unit/libutil/hash.cc
@@ -13,28 +13,28 @@ namespace nix {
     TEST(hashString, testKnownMD5Hashes1) {
         // values taken from: https://tools.ietf.org/html/rfc1321
         auto s1 = "";
-        auto hash = hashString(HashType::htMD5, s1);
+        auto hash = hashString(HashAlgorithm::MD5, s1);
         ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "md5:d41d8cd98f00b204e9800998ecf8427e");
     }
 
     TEST(hashString, testKnownMD5Hashes2) {
         // values taken from: https://tools.ietf.org/html/rfc1321
         auto s2 = "abc";
-        auto hash = hashString(HashType::htMD5, s2);
+        auto hash = hashString(HashAlgorithm::MD5, s2);
         ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "md5:900150983cd24fb0d6963f7d28e17f72");
     }
 
     TEST(hashString, testKnownSHA1Hashes1) {
         // values taken from: https://tools.ietf.org/html/rfc3174
         auto s = "abc";
-        auto hash = hashString(HashType::htSHA1, s);
+        auto hash = hashString(HashAlgorithm::SHA1, s);
         ASSERT_EQ(hash.to_string(HashFormat::Base16, true),"sha1:a9993e364706816aba3e25717850c26c9cd0d89d");
     }
 
     TEST(hashString, testKnownSHA1Hashes2) {
         // values taken from: https://tools.ietf.org/html/rfc3174
         auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq";
-        auto hash = hashString(HashType::htSHA1, s);
+        auto hash = hashString(HashAlgorithm::SHA1, s);
         ASSERT_EQ(hash.to_string(HashFormat::Base16, true),"sha1:84983e441c3bd26ebaae4aa1f95129e5e54670f1");
     }
 
@@ -42,7 +42,7 @@ namespace nix {
         // values taken from: https://tools.ietf.org/html/rfc4634
         auto s = "abc";
 
-        auto hash = hashString(HashType::htSHA256, s);
+        auto hash = hashString(HashAlgorithm::SHA256, s);
         ASSERT_EQ(hash.to_string(HashFormat::Base16, true),
                 "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad");
     }
@@ -50,7 +50,7 @@ namespace nix {
     TEST(hashString, testKnownSHA256Hashes2) {
         // values taken from: https://tools.ietf.org/html/rfc4634
         auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq";
-        auto hash = hashString(HashType::htSHA256, s);
+        auto hash = hashString(HashAlgorithm::SHA256, s);
         ASSERT_EQ(hash.to_string(HashFormat::Base16, true),
                 "sha256:248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1");
     }
@@ -58,7 +58,7 @@ namespace nix {
     TEST(hashString, testKnownSHA512Hashes1) {
         // values taken from: https://tools.ietf.org/html/rfc4634
         auto s = "abc";
-        auto hash = hashString(HashType::htSHA512, s);
+        auto hash = hashString(HashAlgorithm::SHA512, s);
         ASSERT_EQ(hash.to_string(HashFormat::Base16, true),
                 "sha512:ddaf35a193617abacc417349ae20413112e6fa4e89a9"
                 "7ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd"
@@ -68,7 +68,7 @@ namespace nix {
         // values taken from: https://tools.ietf.org/html/rfc4634
         auto s = "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu";
 
-        auto hash = hashString(HashType::htSHA512, s);
+        auto hash = hashString(HashAlgorithm::SHA512, s);
         ASSERT_EQ(hash.to_string(HashFormat::Base16, true),
                 "sha512:8e959b75dae313da8cf4f72814fc143f8f7779c6eb9f7fa1"
                 "7299aeadb6889018501d289e4900f7e4331b99dec4b5433a"

From 837b889c41543b32154ceade2363ec6ad6dff15d Mon Sep 17 00:00:00 2001
From: Peter Kolloch <info@eigenvalue.net>
Date: Tue, 28 Nov 2023 14:41:44 +0100
Subject: [PATCH 267/294] Further HashType renaming + using mkHashAlgoOptFlag
 for new conversion

https://github.com/NixOS/nix/issues/8876
---
 src/libutil/args.cc | 38 +++++++++++++++++++-------------------
 src/libutil/args.hh |  4 ++--
 src/nix/hash.cc     | 17 +++++------------
 src/nix/prefetch.cc |  2 +-
 4 files changed, 27 insertions(+), 34 deletions(-)

diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index ac3727d11..7ea1647d9 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -544,36 +544,36 @@ nlohmann::json Args::toJSON()
     return res;
 }
 
-static void hashTypeCompleter(AddCompletions & completions, size_t index, std::string_view prefix)
+static void hashAlgoCompleter(AddCompletions & completions, size_t index, std::string_view prefix)
 {
     for (auto & type : hashAlgorithms)
         if (hasPrefix(type, prefix))
             completions.add(type);
 }
 
-Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashAlgorithm * ha)
+Args::Flag Args::Flag::mkHashAlgoFlag(std::string && longName, HashAlgorithm * ha)
 {
-    return Flag {
-        .longName = std::move(longName),
-        .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')",
-        .labels = {"hash-algo"},
-        .handler = {[ha](std::string s) {
-            *ha = parseHashAlgo(s);
-        }},
-        .completer = hashTypeCompleter,
+    return Flag{
+            .longName = std::move(longName),
+            .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')",
+            .labels = {"hash-algo"},
+            .handler = {[ha](std::string s) {
+                *ha = parseHashAlgo(s);
+            }},
+            .completer = hashAlgoCompleter,
     };
 }
 
-Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional<HashAlgorithm> * oha)
+Args::Flag Args::Flag::mkHashAlgoOptFlag(std::string && longName, std::optional<HashAlgorithm> * oha)
 {
-    return Flag {
-        .longName = std::move(longName),
-        .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512'). Optional as can also be gotten from SRI hash itself.",
-        .labels = {"hash-algo"},
-        .handler = {[oha](std::string s) {
-            *oha = std::optional<HashAlgorithm> {parseHashAlgo(s) };
-        }},
-        .completer = hashTypeCompleter,
+    return Flag{
+            .longName = std::move(longName),
+            .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512'). Optional as can also be gotten from SRI hash itself.",
+            .labels = {"hash-algo"},
+            .handler = {[oha](std::string s) {
+                *oha = std::optional<HashAlgorithm>{parseHashAlgo(s)};
+            }},
+            .completer = hashAlgoCompleter,
     };
 }
 
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 0cff76158..653a9bbd6 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -175,8 +175,8 @@ protected:
 
         std::optional<ExperimentalFeature> experimentalFeature;
 
-        static Flag mkHashTypeFlag(std::string && longName, HashAlgorithm * ha);
-        static Flag mkHashTypeOptFlag(std::string && longName, std::optional<HashAlgorithm> * oha);
+        static Flag mkHashAlgoFlag(std::string && longName, HashAlgorithm * ha);
+        static Flag mkHashAlgoOptFlag(std::string && longName, std::optional<HashAlgorithm> * oha);
     };
 
     /**
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index 638178afa..173043c8a 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -48,7 +48,7 @@ struct CmdHashBase : Command
             .handler = {&hashFormat, HashFormat::Base16},
         });
 
-        addFlag(Flag::mkHashTypeFlag("type", &ha));
+        addFlag(Flag::mkHashAlgoFlag("type", &ha));
 
         #if 0
         addFlag({
@@ -112,7 +112,7 @@ struct CmdToBase : Command
 
     CmdToBase(HashFormat hashFormat) : hashFormat(hashFormat)
     {
-        addFlag(Flag::mkHashTypeOptFlag("type", &ht));
+        addFlag(Flag::mkHashAlgoOptFlag("type", &ht));
         expectArgs("strings", &args);
     }
 
@@ -139,7 +139,7 @@ struct CmdHashConvert : Command
 {
     std::optional<HashFormat> from;
     HashFormat to;
-    std::optional<HashAlgorithm> type;
+    std::optional<HashAlgorithm> algo;
     std::vector<std::string> hashStrings;
 
     CmdHashConvert(): to(HashFormat::SRI) {
@@ -161,14 +161,7 @@ struct CmdHashConvert : Command
                 to = parseHashFormat(str);
             }},
         });
-        addFlag({
-            .longName = "algo",
-            .description = "Specify the algorithm if it can't be auto-detected.",
-            .labels = {"hash algorithm"},
-            .handler = {[this](std::string str) {
-                type = parseHashAlgo(str);
-            }},
-        });
+        addFlag(Args::Flag::mkHashAlgoOptFlag("algo", &algo));
         expectArgs({
            .label = "hashes",
            .handler = {&hashStrings},
@@ -184,7 +177,7 @@ struct CmdHashConvert : Command
 
     void run() override {
         for (const auto& s: hashStrings) {
-            Hash h = Hash::parseAny(s, type);
+            Hash h = Hash::parseAny(s, algo);
             if (from && h.to_string(*from, from == HashFormat::SRI) != s) {
                 auto from_as_string = printHashFormat(*from);
                 throw BadHash("input hash '%s' does not have the expected format '--from %s'", s, from_as_string);
diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc
index 09f33a51e..bbfeb8aa4 100644
--- a/src/nix/prefetch.cc
+++ b/src/nix/prefetch.cc
@@ -279,7 +279,7 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON
             }}
         });
 
-        addFlag(Flag::mkHashTypeFlag("hash-type", &hashAlgo));
+        addFlag(Flag::mkHashAlgoFlag("hash-type", &hashAlgo));
 
         addFlag({
             .longName = "executable",

From fc6f29053aa69b6b14bcad93cb273b1c266e74fe Mon Sep 17 00:00:00 2001
From: Peter Kolloch <info@eigenvalue.net>
Date: Tue, 28 Nov 2023 15:38:15 +0100
Subject: [PATCH 268/294] Renamed HashFormat::Base32 to HashFormat::Nix32

...and also adjusted parsing accordingly.

Also added CLI completion for HashFormats.

https://github.com/NixOS/nix/issues/8876
---
 src/libexpr/primops/fetchTree.cc              |   2 +-
 src/libfetchers/git.cc                        |   2 +-
 src/libfetchers/mercurial.cc                  |   2 +-
 src/libstore/binary-cache-store.cc            |   4 +-
 src/libstore/build/local-derivation-goal.cc   |   2 +-
 src/libstore/content-address.cc               |   2 +-
 src/libstore/derivations.cc                   |   2 +-
 src/libstore/downstream-placeholder.cc        |   4 +-
 src/libstore/export-import.cc                 |   2 +-
 src/libstore/gc.cc                            |   2 +-
 src/libstore/local-store.cc                   |  10 +-
 src/libstore/nar-info-disk-cache.cc           |   4 +-
 src/libstore/nar-info.cc                      |   4 +-
 src/libstore/optimise-store.cc                |   4 +-
 src/libstore/parsed-derivations.cc            |   2 +-
 src/libstore/path-info.cc                     |   6 +-
 src/libstore/path.cc                          |   2 +-
 src/libutil/args.cc                           |  40 ++++++-
 src/libutil/args.hh                           |   3 +
 src/libutil/hash.cc                           |  25 ++--
 src/libutil/hash.hh                           |  10 +-
 src/libutil/references.cc                     |   4 +-
 src/nix-store/nix-store.cc                    |   8 +-
 src/nix/hash.cc                               |  40 +++----
 src/nix/verify.cc                             |   4 +-
 tests/functional/hash.sh                      |   6 +-
 .../lang/eval-okay-convertHash.err.exp        | 108 ++++++++++++++++++
 .../functional/lang/eval-okay-convertHash.exp |   2 +-
 .../functional/lang/eval-okay-convertHash.nix |   2 +
 tests/unit/libutil/hash.cc                    |   2 +-
 30 files changed, 228 insertions(+), 82 deletions(-)
 create mode 100644 tests/functional/lang/eval-okay-convertHash.err.exp

diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index ef80c634f..15f870a95 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -304,7 +304,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
             : hashFile(HashAlgorithm::SHA256, state.store->toRealPath(storePath));
         if (hash != *expectedHash)
             state.debugThrowLastTrace(EvalError((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n  specified: %s\n  got:       %s",
-                *url, expectedHash->to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true)));
+                                                *url, expectedHash->to_string(HashFormat::Nix32, true), hash.to_string(HashFormat::Nix32, true)));
     }
 
     state.allowAndSetStorePathString(storePath, v);
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index a89acc1c0..9e6ba8963 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -52,7 +52,7 @@ bool touchCacheFile(const Path & path, time_t touch_time)
 Path getCachePath(std::string_view key)
 {
     return getCacheDir() + "/nix/gitv3/" +
-        hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Base32, false);
+        hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Nix32, false);
 }
 
 // Returns the name of the HEAD branch.
diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc
index 713f24bbb..6056b9a3c 100644
--- a/src/libfetchers/mercurial.cc
+++ b/src/libfetchers/mercurial.cc
@@ -267,7 +267,7 @@ struct MercurialInputScheme : InputScheme
             }
         }
 
-        Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(HashAlgorithm::SHA256, actualUrl).to_string(HashFormat::Base32, false));
+        Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(HashAlgorithm::SHA256, actualUrl).to_string(HashFormat::Nix32, false));
 
         /* If this is a commit hash that we already have, we don't
            have to pull again. */
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index f287d72a8..2837e8934 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -165,8 +165,8 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     auto [fileHash, fileSize] = fileHashSink.finish();
     narInfo->fileHash = fileHash;
     narInfo->fileSize = fileSize;
-    narInfo->url = "nar/" + narInfo->fileHash->to_string(HashFormat::Base32, false) + ".nar"
-        + (compression == "xz" ? ".xz" :
+    narInfo->url = "nar/" + narInfo->fileHash->to_string(HashFormat::Nix32, false) + ".nar"
+                   + (compression == "xz" ? ".xz" :
            compression == "bzip2" ? ".bz2" :
            compression == "zstd" ? ".zst" :
            compression == "lzip" ? ".lzip" :
diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 4c3dc1f5c..802b39f84 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -1067,7 +1067,7 @@ void LocalDerivationGoal::initTmpDir() {
                 env[i.first] = i.second;
             } else {
                 auto hash = hashString(HashAlgorithm::SHA256, i.first);
-                std::string fn = ".attr-" + hash.to_string(HashFormat::Base32, false);
+                std::string fn = ".attr-" + hash.to_string(HashFormat::Nix32, false);
                 Path p = tmpDir + "/" + fn;
                 writeFile(p, rewriteStrings(i.second, inputRewrites));
                 chownToBuilder(p);
diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc
index de8194f73..f42a13126 100644
--- a/src/libstore/content-address.cc
+++ b/src/libstore/content-address.cc
@@ -61,7 +61,7 @@ std::string ContentAddress::render() const
                 + makeFileIngestionPrefix(method);
         },
     }, method.raw)
-        + this->hash.to_string(HashFormat::Base32, true);
+        + this->hash.to_string(HashFormat::Nix32, true);
 }
 
 /**
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index c68631c1a..664ab7556 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -958,7 +958,7 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva
 std::string hashPlaceholder(const OutputNameView outputName)
 {
     // FIXME: memoize?
-    return "/" + hashString(HashAlgorithm::SHA256, concatStrings("nix-output:", outputName)).to_string(HashFormat::Base32, false);
+    return "/" + hashString(HashAlgorithm::SHA256, concatStrings("nix-output:", outputName)).to_string(HashFormat::Nix32, false);
 }
 
 
diff --git a/src/libstore/downstream-placeholder.cc b/src/libstore/downstream-placeholder.cc
index 10df37fa4..91d47f946 100644
--- a/src/libstore/downstream-placeholder.cc
+++ b/src/libstore/downstream-placeholder.cc
@@ -5,7 +5,7 @@ namespace nix {
 
 std::string DownstreamPlaceholder::render() const
 {
-    return "/" + hash.to_string(HashFormat::Base32, false);
+    return "/" + hash.to_string(HashFormat::Nix32, false);
 }
 
 
@@ -31,7 +31,7 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation(
     xpSettings.require(Xp::DynamicDerivations);
     auto compressed = compressHash(placeholder.hash, 20);
     auto clearText = "nix-computed-output:"
-        + compressed.to_string(HashFormat::Base32, false)
+        + compressed.to_string(HashFormat::Nix32, false)
         + ":" + std::string { outputName };
     return DownstreamPlaceholder {
         hashString(HashAlgorithm::SHA256, clearText)
diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc
index 48718ef84..d57b25bd7 100644
--- a/src/libstore/export-import.cc
+++ b/src/libstore/export-import.cc
@@ -41,7 +41,7 @@ void Store::exportPath(const StorePath & path, Sink & sink)
     Hash hash = hashSink.currentHash().first;
     if (hash != info->narHash && info->narHash != Hash(info->narHash.algo))
         throw Error("hash of path '%s' has changed from '%s' to '%s'!",
-            printStorePath(path), info->narHash.to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true));
+                    printStorePath(path), info->narHash.to_string(HashFormat::Nix32, true), hash.to_string(HashFormat::Nix32, true));
 
     teeSink
         << exportMagic
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 5c413aa77..2bd3a2edc 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -50,7 +50,7 @@ static void makeSymlink(const Path & link, const Path & target)
 
 void LocalStore::addIndirectRoot(const Path & path)
 {
-    std::string hash = hashString(HashAlgorithm::SHA1, path).to_string(HashFormat::Base32, false);
+    std::string hash = hashString(HashAlgorithm::SHA1, path).to_string(HashFormat::Nix32, false);
     Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", stateDir, gcRootsDir, hash));
     makeSymlink(realRoot, path);
 }
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index ef7dd7985..7e82bae28 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1080,7 +1080,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
             if (hashResult.first != info.narHash)
                 throw Error("hash mismatch importing path '%s';\n  specified: %s\n  got:       %s",
-                    printStorePath(info.path), info.narHash.to_string(HashFormat::Base32, true), hashResult.first.to_string(HashFormat::Base32, true));
+                            printStorePath(info.path), info.narHash.to_string(HashFormat::Nix32, true), hashResult.first.to_string(HashFormat::Nix32, true));
 
             if (hashResult.second != info.narSize)
                 throw Error("size mismatch importing path '%s';\n  specified: %s\n  got:       %s",
@@ -1096,8 +1096,8 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
                 if (specified.hash != actualHash.hash) {
                     throw Error("ca hash mismatch importing path '%s';\n  specified: %s\n  got:       %s",
                         printStorePath(info.path),
-                        specified.hash.to_string(HashFormat::Base32, true),
-                        actualHash.hash.to_string(HashFormat::Base32, true));
+                        specified.hash.to_string(HashFormat::Nix32, true),
+                        actualHash.hash.to_string(HashFormat::Nix32, true));
                 }
             }
 
@@ -1389,7 +1389,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
         for (auto & link : readDirectory(linksDir)) {
             printMsg(lvlTalkative, "checking contents of '%s'", link.name);
             Path linkPath = linksDir + "/" + link.name;
-            std::string hash = hashPath(HashAlgorithm::SHA256, linkPath).first.to_string(HashFormat::Base32, false);
+            std::string hash = hashPath(HashAlgorithm::SHA256, linkPath).first.to_string(HashFormat::Nix32, false);
             if (hash != link.name) {
                 printError("link '%s' was modified! expected hash '%s', got '%s'",
                     linkPath, link.name, hash);
@@ -1422,7 +1422,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 
                 if (info->narHash != nullHash && info->narHash != current.first) {
                     printError("path '%s' was modified! expected hash '%s', got '%s'",
-                        printStorePath(i), info->narHash.to_string(HashFormat::Base32, true), current.first.to_string(HashFormat::Base32, true));
+                               printStorePath(i), info->narHash.to_string(HashFormat::Nix32, true), current.first.to_string(HashFormat::Nix32, true));
                     if (repair) repairPath(i); else errors = true;
                 } else {
 
diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index e50c15939..310105c75 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -333,9 +333,9 @@ public:
                     (std::string(info->path.name()))
                     (narInfo ? narInfo->url : "", narInfo != 0)
                     (narInfo ? narInfo->compression : "", narInfo != 0)
-                    (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(HashFormat::Base32, true) : "", narInfo && narInfo->fileHash)
+                    (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(HashFormat::Nix32, true) : "", narInfo && narInfo->fileHash)
                     (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize)
-                    (info->narHash.to_string(HashFormat::Base32, true))
+                    (info->narHash.to_string(HashFormat::Nix32, true))
                     (info->narSize)
                     (concatStringsSep(" ", info->shortRefs()))
                     (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver)
diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc
index 25e2a7d7b..d9618d04c 100644
--- a/src/libstore/nar-info.cc
+++ b/src/libstore/nar-info.cc
@@ -114,10 +114,10 @@ std::string NarInfo::to_string(const Store & store) const
     assert(compression != "");
     res += "Compression: " + compression + "\n";
     assert(fileHash && fileHash->algo == HashAlgorithm::SHA256);
-    res += "FileHash: " + fileHash->to_string(HashFormat::Base32, true) + "\n";
+    res += "FileHash: " + fileHash->to_string(HashFormat::Nix32, true) + "\n";
     res += "FileSize: " + std::to_string(fileSize) + "\n";
     assert(narHash.algo == HashAlgorithm::SHA256);
-    res += "NarHash: " + narHash.to_string(HashFormat::Base32, true) + "\n";
+    res += "NarHash: " + narHash.to_string(HashFormat::Nix32, true) + "\n";
     res += "NarSize: " + std::to_string(narSize) + "\n";
 
     res += "References: " + concatStringsSep(" ", shortRefs()) + "\n";
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index cadf88347..b395453d1 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -147,10 +147,10 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
        contents of the symlink (i.e. the result of readlink()), not
        the contents of the target (which may not even exist). */
     Hash hash = hashPath(HashAlgorithm::SHA256, path).first;
-    debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Base32, true));
+    debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Nix32, true));
 
     /* Check if this is a known hash. */
-    Path linkPath = linksDir + "/" + hash.to_string(HashFormat::Base32, false);
+    Path linkPath = linksDir + "/" + hash.to_string(HashFormat::Nix32, false);
 
     /* Maybe delete the link, if it has been corrupted. */
     if (pathExists(linkPath)) {
diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc
index 73e55a96c..72f45143d 100644
--- a/src/libstore/parsed-derivations.cc
+++ b/src/libstore/parsed-derivations.cc
@@ -146,7 +146,7 @@ static nlohmann::json pathInfoToJSON(
         auto info = store.queryPathInfo(storePath);
 
         auto & jsonPath = jsonList.emplace_back(
-            info->toJSON(store, false, HashFormat::Base32));
+            info->toJSON(store, false, HashFormat::Nix32));
 
         // Add the path to the object whose metadata we are including.
         jsonPath["path"] = store.printStorePath(storePath);
diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc
index 2d7dc972f..f58e31bfd 100644
--- a/src/libstore/path-info.cc
+++ b/src/libstore/path-info.cc
@@ -31,9 +31,9 @@ std::string ValidPathInfo::fingerprint(const Store & store) const
         throw Error("cannot calculate fingerprint of path '%s' because its size is not known",
             store.printStorePath(path));
     return
-        "1;" + store.printStorePath(path) + ";"
-        + narHash.to_string(HashFormat::Base32, true) + ";"
-        + std::to_string(narSize) + ";"
+            "1;" + store.printStorePath(path) + ";"
+            + narHash.to_string(HashFormat::Nix32, true) + ";"
+            + std::to_string(narSize) + ";"
         + concatStringsSep(",", store.printStorePathSet(references));
 }
 
diff --git a/src/libstore/path.cc b/src/libstore/path.cc
index d5257c939..1afd10af7 100644
--- a/src/libstore/path.cc
+++ b/src/libstore/path.cc
@@ -35,7 +35,7 @@ StorePath::StorePath(std::string_view _baseName)
 }
 
 StorePath::StorePath(const Hash & hash, std::string_view _name)
-    : baseName((hash.to_string(HashFormat::Base32, false) + "-").append(std::string(_name)))
+    : baseName((hash.to_string(HashFormat::Nix32, false) + "-").append(std::string(_name)))
 {
     checkName(baseName, name());
 }
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 7ea1647d9..e2668c673 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -544,11 +544,45 @@ nlohmann::json Args::toJSON()
     return res;
 }
 
+static void hashFormatCompleter(AddCompletions & completions, size_t index, std::string_view prefix)
+{
+    for (auto & format : hashFormats) {
+        if (hasPrefix(format, prefix)) {
+            completions.add(format);
+        }
+    }
+}
+
+Args::Flag Args::Flag::mkHashFormatFlagWithDefault(std::string &&longName, HashFormat * hf) {
+    assert(*hf == nix::HashFormat::SRI);
+    return Flag{
+            .longName = std::move(longName),
+            .description = "hash format ('base16', 'nix32', 'base64', 'sri'). Default: 'sri'",
+            .labels = {"hash-format"},
+            .handler = {[hf](std::string s) {
+                *hf = parseHashFormat(s);
+            }},
+            .completer = hashFormatCompleter,
+    };
+}
+
+Args::Flag Args::Flag::mkHashFormatOptFlag(std::string && longName, std::optional<HashFormat> * ohf) {
+    return Flag{
+            .longName = std::move(longName),
+            .description = "hash format ('base16', 'nix32', 'base64', 'sri').",
+            .labels = {"hash-format"},
+            .handler = {[ohf](std::string s) {
+                *ohf = std::optional<HashFormat>{parseHashFormat(s)};
+            }},
+            .completer = hashFormatCompleter,
+    };
+}
+
 static void hashAlgoCompleter(AddCompletions & completions, size_t index, std::string_view prefix)
 {
-    for (auto & type : hashAlgorithms)
-        if (hasPrefix(type, prefix))
-            completions.add(type);
+    for (auto & algo : hashAlgorithms)
+        if (hasPrefix(algo, prefix))
+            completions.add(algo);
 }
 
 Args::Flag Args::Flag::mkHashAlgoFlag(std::string && longName, HashAlgorithm * ha)
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 653a9bbd6..18b0ae583 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -15,6 +15,7 @@
 namespace nix {
 
 enum struct HashAlgorithm : char;
+enum struct HashFormat : int;
 
 class MultiCommand;
 
@@ -177,6 +178,8 @@ protected:
 
         static Flag mkHashAlgoFlag(std::string && longName, HashAlgorithm * ha);
         static Flag mkHashAlgoOptFlag(std::string && longName, std::optional<HashAlgorithm> * oha);
+        static Flag mkHashFormatFlagWithDefault(std::string && longName, HashFormat * hf);
+        static Flag mkHashFormatOptFlag(std::string && longName, std::optional<HashFormat> * ohf);
     };
 
     /**
diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index 38a29c459..30456ae5c 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -27,8 +27,9 @@ static size_t regularHashSize(HashAlgorithm type) {
 }
 
 
-std::set<std::string> hashAlgorithms = {"md5", "sha1", "sha256", "sha512" };
+const std::set<std::string> hashAlgorithms = {"md5", "sha1", "sha256", "sha512" };
 
+const std::set<std::string> hashFormats = {"base64", "nix32", "base16", "sri" };
 
 Hash::Hash(HashAlgorithm algo) : algo(algo)
 {
@@ -81,7 +82,7 @@ static std::string printHash16(const Hash & hash)
 
 
 // omitted: E O U T
-const std::string base32Chars = "0123456789abcdfghijklmnpqrsvwxyz";
+const std::string nix32Chars = "0123456789abcdfghijklmnpqrsvwxyz";
 
 
 static std::string printHash32(const Hash & hash)
@@ -100,7 +101,7 @@ static std::string printHash32(const Hash & hash)
         unsigned char c =
             (hash.hash[i] >> j)
             | (i >= hash.hashSize - 1 ? 0 : hash.hash[i + 1] << (8 - j));
-        s.push_back(base32Chars[c & 0x1f]);
+        s.push_back(nix32Chars[c & 0x1f]);
     }
 
     return s;
@@ -110,7 +111,7 @@ static std::string printHash32(const Hash & hash)
 std::string printHash16or32(const Hash & hash)
 {
     assert(static_cast<char>(hash.algo));
-    return hash.to_string(hash.algo == HashAlgorithm::MD5 ? HashFormat::Base16 : HashFormat::Base32, false);
+    return hash.to_string(hash.algo == HashAlgorithm::MD5 ? HashFormat::Base16 : HashFormat::Nix32, false);
 }
 
 
@@ -125,7 +126,7 @@ std::string Hash::to_string(HashFormat hashFormat, bool includeAlgo) const
     case HashFormat::Base16:
         s += printHash16(*this);
         break;
-    case HashFormat::Base32:
+    case HashFormat::Nix32:
         s += printHash32(*this);
         break;
     case HashFormat::Base64:
@@ -230,8 +231,8 @@ Hash::Hash(std::string_view rest, HashAlgorithm algo, bool isSRI)
         for (unsigned int n = 0; n < rest.size(); ++n) {
             char c = rest[rest.size() - n - 1];
             unsigned char digit;
-            for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */
-                if (base32Chars[digit] == c) break;
+            for (digit = 0; digit < nix32Chars.size(); ++digit) /* !!! slow */
+                if (nix32Chars[digit] == c) break;
             if (digit >= 32)
                 throw BadHash("invalid base-32 hash '%s'", rest);
             unsigned int b = n * 5;
@@ -388,7 +389,11 @@ Hash compressHash(const Hash & hash, unsigned int newSize)
 std::optional<HashFormat> parseHashFormatOpt(std::string_view hashFormatName)
 {
     if (hashFormatName == "base16") return HashFormat::Base16;
-    if (hashFormatName == "base32") return HashFormat::Base32;
+    if (hashFormatName == "nix32") return HashFormat::Nix32;
+    if (hashFormatName == "base32") {
+        warn(R"("base32" is a deprecated alias for hash format "nix32".)");
+        return HashFormat::Nix32;
+    }
     if (hashFormatName == "base64") return HashFormat::Base64;
     if (hashFormatName == "sri") return HashFormat::SRI;
     return std::nullopt;
@@ -407,8 +412,8 @@ std::string_view printHashFormat(HashFormat HashFormat)
     switch (HashFormat) {
     case HashFormat::Base64:
         return "base64";
-    case HashFormat::Base32:
-        return "base32";
+    case HashFormat::Nix32:
+        return "nix32";
     case HashFormat::Base16:
         return "base16";
     case HashFormat::SRI:
diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh
index 3c97ed4b1..7bed9e2bd 100644
--- a/src/libutil/hash.hh
+++ b/src/libutil/hash.hh
@@ -20,9 +20,9 @@ const int sha1HashSize = 20;
 const int sha256HashSize = 32;
 const int sha512HashSize = 64;
 
-extern std::set<std::string> hashAlgorithms;
+extern const std::set<std::string> hashAlgorithms;
 
-extern const std::string base32Chars;
+extern const std::string nix32Chars;
 
 /**
  * @brief Enumeration representing the hash formats.
@@ -31,8 +31,8 @@ enum struct HashFormat : int {
     /// @brief Base 64 encoding.
     /// @see [IETF RFC 4648, section 4](https://datatracker.ietf.org/doc/html/rfc4648#section-4).
     Base64,
-    /// @brief Nix-specific base-32 encoding. @see base32Chars
-    Base32,
+    /// @brief Nix-specific base-32 encoding. @see nix32Chars
+    Nix32,
     /// @brief Lowercase hexadecimal encoding. @see base16Chars
     Base16,
     /// @brief "<hash algo>:<Base 64 hash>", format of the SRI integrity attribute.
@@ -40,6 +40,8 @@ enum struct HashFormat : int {
     SRI
 };
 
+extern const std::set<std::string> hashFormats;
+
 struct Hash
 {
     constexpr static size_t maxHashSize = 64;
diff --git a/src/libutil/references.cc b/src/libutil/references.cc
index d82d51945..b30e62c7b 100644
--- a/src/libutil/references.cc
+++ b/src/libutil/references.cc
@@ -23,8 +23,8 @@ static void search(
     static bool isBase32[256];
     std::call_once(initialised, [](){
         for (unsigned int i = 0; i < 256; ++i) isBase32[i] = false;
-        for (unsigned int i = 0; i < base32Chars.size(); ++i)
-            isBase32[(unsigned char) base32Chars[i]] = true;
+        for (unsigned int i = 0; i < nix32Chars.size(); ++i)
+            isBase32[(unsigned char) nix32Chars[i]] = true;
     });
 
     for (size_t i = 0; i + refLength <= s.size(); ) {
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 75ad4e75f..db45be2a8 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -406,7 +406,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
                     auto info = store->queryPathInfo(j);
                     if (query == qHash) {
                         assert(info->narHash.algo == HashAlgorithm::SHA256);
-                        cout << fmt("%s\n", info->narHash.to_string(HashFormat::Base32, true));
+                        cout << fmt("%s\n", info->narHash.to_string(HashFormat::Nix32, true));
                     } else if (query == qSize)
                         cout << fmt("%d\n", info->narSize);
                 }
@@ -769,8 +769,8 @@ static void opVerifyPath(Strings opFlags, Strings opArgs)
         if (current.first != info->narHash) {
             printError("path '%s' was modified! expected hash '%s', got '%s'",
                 store->printStorePath(path),
-                info->narHash.to_string(HashFormat::Base32, true),
-                current.first.to_string(HashFormat::Base32, true));
+                info->narHash.to_string(HashFormat::Nix32, true),
+                current.first.to_string(HashFormat::Nix32, true));
             status = 1;
         }
     }
@@ -898,7 +898,7 @@ static void opServe(Strings opFlags, Strings opArgs)
                         out << info->narSize // downloadSize
                             << info->narSize;
                         if (GET_PROTOCOL_MINOR(clientVersion) >= 4)
-                            out << info->narHash.to_string(HashFormat::Base32, true)
+                            out << info->narHash.to_string(HashFormat::Nix32, true)
                                 << renderContentAddress(info->ca)
                                 << info->sigs;
                     } catch (InvalidPath &) {
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index 173043c8a..f9c7592a3 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -39,7 +39,7 @@ struct CmdHashBase : Command
         addFlag({
             .longName = "base32",
             .description = "Print the hash in base-32 (Nix-specific) format.",
-            .handler = {&hashFormat, HashFormat::Base32},
+            .handler = {&hashFormat, HashFormat::Nix32},
         });
 
         addFlag({
@@ -120,7 +120,7 @@ struct CmdToBase : Command
     {
         return fmt("convert a hash to %s representation",
             hashFormat == HashFormat::Base16 ? "base-16" :
-            hashFormat == HashFormat::Base32 ? "base-32" :
+            hashFormat == HashFormat::Nix32 ? "base-32" :
             hashFormat == HashFormat::Base64 ? "base-64" :
             "SRI");
     }
@@ -143,24 +143,8 @@ struct CmdHashConvert : Command
     std::vector<std::string> hashStrings;
 
     CmdHashConvert(): to(HashFormat::SRI) {
-        addFlag({
-            .longName = "from",
-            // TODO: List format choices. Maybe introduce a constant?
-            .description = "The format of the input hash.",
-            .labels = {"hash format"},
-            .handler = {[this](std::string str) {
-                from = parseHashFormat(str);
-            }},
-        });
-        addFlag({
-            .longName = "to",
-            // TODO: List format choices. Maybe introduce a constant?
-            .description = "The format of the output hash.",
-            .labels = {"hash format"},
-            .handler = {[this](std::string str) {
-                to = parseHashFormat(str);
-            }},
-        });
+        addFlag(Args::Flag::mkHashFormatOptFlag("from", &from));
+        addFlag(Args::Flag::mkHashFormatFlagWithDefault("to", &to));
         addFlag(Args::Flag::mkHashAlgoOptFlag("algo", &algo));
         expectArgs({
            .label = "hashes",
@@ -170,7 +154,15 @@ struct CmdHashConvert : Command
 
     std::string description() override
     {
-        return "convert between different hash formats, e.g. base16, nix32, base64 and sri.";
+        std::string descr( "convert between different hash formats. Choose from: ");
+        auto iter = hashFormats.begin();
+        assert(iter != hashFormats.end());
+        descr += *iter++;
+        while (iter != hashFormats.end()) {
+            descr += ", " + *iter++;
+        }
+
+        return descr;
     }
 
     Category category() override { return catUtility; }
@@ -197,7 +189,7 @@ struct CmdHash : NixMultiCommand
                 {"file", []() { return make_ref<CmdHashBase>(FileIngestionMethod::Flat);; }},
                 {"path", []() { return make_ref<CmdHashBase>(FileIngestionMethod::Recursive); }},
                 {"to-base16", []() { return make_ref<CmdToBase>(HashFormat::Base16); }},
-                {"to-base32", []() { return make_ref<CmdToBase>(HashFormat::Base32); }},
+                {"to-base32", []() { return make_ref<CmdToBase>(HashFormat::Nix32); }},
                 {"to-base64", []() { return make_ref<CmdToBase>(HashFormat::Base64); }},
                 {"to-sri", []() { return make_ref<CmdToBase>(HashFormat::SRI); }},
           })
@@ -230,7 +222,7 @@ static int compatNixHash(int argc, char * * argv)
             printVersion("nix-hash");
         else if (*arg == "--flat") flat = true;
         else if (*arg == "--base16") hashFormat = HashFormat::Base16;
-        else if (*arg == "--base32") hashFormat = HashFormat::Base32;
+        else if (*arg == "--base32") hashFormat = HashFormat::Nix32;
         else if (*arg == "--base64") hashFormat = HashFormat::Base64;
         else if (*arg == "--sri") hashFormat = HashFormat::SRI;
         else if (*arg == "--truncate") truncate = true;
@@ -244,7 +236,7 @@ static int compatNixHash(int argc, char * * argv)
         }
         else if (*arg == "--to-base32") {
             op = opTo;
-            hashFormat = HashFormat::Base32;
+            hashFormat = HashFormat::Nix32;
         }
         else if (*arg == "--to-base64") {
             op = opTo;
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index cd0f6d95f..f0234f7be 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -109,8 +109,8 @@ struct CmdVerify : StorePathsCommand
                         act2.result(resCorruptedPath, store->printStorePath(info->path));
                         printError("path '%s' was modified! expected hash '%s', got '%s'",
                             store->printStorePath(info->path),
-                            info->narHash.to_string(HashFormat::Base32, true),
-                            hash.first.to_string(HashFormat::Base32, true));
+                            info->narHash.to_string(HashFormat::Nix32, true),
+                            hash.first.to_string(HashFormat::Nix32, true));
                     }
                 }
 
diff --git a/tests/functional/hash.sh b/tests/functional/hash.sh
index 031e33adf..278ed83b9 100644
--- a/tests/functional/hash.sh
+++ b/tests/functional/hash.sh
@@ -163,7 +163,7 @@ try3() {
 
     sri=$(nix hash convert --algo "$1" --from base16 "$2")
     [ "$sri" = "$1-$4" ]
-    sri=$(nix hash convert --algo "$1" --from base32 "$3")
+    sri=$(nix hash convert --algo "$1" --from nix32 "$3")
     [ "$sri" = "$1-$4" ]
     sri=$(nix hash convert --algo "$1" --from base64 "$4")
     [ "$sri" = "$1-$4" ]
@@ -172,11 +172,11 @@ try3() {
     # Asserting input format fails.
     #
 
-    fail=$(nix hash convert --algo "$1" --from base32 "$2" 2>&1 || echo "exit: $?")
+    fail=$(nix hash convert --algo "$1" --from nix32 "$2" 2>&1 || echo "exit: $?")
     [[ "$fail" == "error: input hash"*"exit: 1" ]]
     fail=$(nix hash convert --algo "$1" --from base16 "$3" 2>&1 || echo "exit: $?")
     [[ "$fail" == "error: input hash"*"exit: 1" ]]
-    fail=$(nix hash convert --algo "$1" --from base32 "$4" 2>&1 || echo "exit: $?")
+    fail=$(nix hash convert --algo "$1" --from nix32 "$4" 2>&1 || echo "exit: $?")
     [[ "$fail" == "error: input hash"*"exit: 1" ]]
 
 }
diff --git a/tests/functional/lang/eval-okay-convertHash.err.exp b/tests/functional/lang/eval-okay-convertHash.err.exp
new file mode 100644
index 000000000..41d746725
--- /dev/null
+++ b/tests/functional/lang/eval-okay-convertHash.err.exp
@@ -0,0 +1,108 @@
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
diff --git a/tests/functional/lang/eval-okay-convertHash.exp b/tests/functional/lang/eval-okay-convertHash.exp
index 60e0a3c49..16b0240e5 100644
--- a/tests/functional/lang/eval-okay-convertHash.exp
+++ b/tests/functional/lang/eval-okay-convertHash.exp
@@ -1 +1 @@
-{ hashesBase16 = [ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ]; hashesBase32 = [ "3y8bwfr609h3lh9ch0izcqq7fl" "26mrvc0v1nslch8r0w45zywsbc" "1v4gi57l97pmnylq6lmgxkhd5v" "143xibwh31h9bvxzalr0sjvbbvpa6ffs" "i4hj30pkrfdpgc5dbcgcydqviibfhm6d" "fxz2p030yba2bza71qhss79k3l5y24kd" "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73" "0qy6iz9yh6a079757mxdmypx0gcmnzjd3ij5q78bzk00vxll82lh" "0mkygpci4r4yb8zz5rs2kxcgvw0a2yf5zlj6r8qgfll6pnrqf0xd" "0zdl9zrg8r3i9c1g90lgg9ip5ijzv3yhz91i0zzn3r8ap9ws784gkp9dk9j3aglhgf1amqb0pj21mh7h1nxcl18akqvvf7ggqsy30yg" "19ncrpp37dx0nzzjw4k6zaqkb9mzaq2myhgpzh5aff7qqcj5wwdxslg6ixwncm7gyq8l761gwf87fgsh2bwfyr52s53k2dkqvw8c24x" "2kz74snvckxldmmbisz9ikmy031d28cs6xfdbl6rhxx42glpyz4vww4lajrc5akklxwixl0js4g84233pxvmbykiic5m7i5m9r4nr11" ]; hashesBase64 = [ "1B2M2Y8AsgTpgAmY7PhCfg==" "bGnufyEcZAQZ1TZswHauRg==" "uzQ4+6vUYOptvSfRU+IjOw==" "2jmj7l5rSw0yVb/vlWAYkK/YBwk=" "zVToVowbN88eW62wd5vL84IhIYk=" "bRLhCx0zHa0hDkf9JdTyYIArfnc=" "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" "kApEad8AzL/QwUXG0eS3lT3Qr6+t11NOOkAZ6NOPxmM=" "rQOHs72GUvcwykbSX5wXCvD9WJ9C5/I/Wp5kEtl9flY=" "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" "nQiG+MaziTmKFiV7x5eA+rmDHH/BHIqwf6cyy3s0j+reOC+SYXycUwX++6CvAqtf05pYfTMJl/9b0NsZ92ZmUw==" "IWRLcqolnlpYjNOvuvsdQxD0iJaA9sg7nVMVlqWihPNNvr/0CdI7zIau5rrRDIkWBvB1xvR1XLU22ifbVpPzpw==" ]; hashesSRI = [ "md5-1B2M2Y8AsgTpgAmY7PhCfg==" "md5-bGnufyEcZAQZ1TZswHauRg==" "md5-uzQ4+6vUYOptvSfRU+IjOw==" "sha1-2jmj7l5rSw0yVb/vlWAYkK/YBwk=" "sha1-zVToVowbN88eW62wd5vL84IhIYk=" "sha1-bRLhCx0zHa0hDkf9JdTyYIArfnc=" "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" "sha256-kApEad8AzL/QwUXG0eS3lT3Qr6+t11NOOkAZ6NOPxmM=" "sha256-rQOHs72GUvcwykbSX5wXCvD9WJ9C5/I/Wp5kEtl9flY=" "sha512-z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" "sha512-nQiG+MaziTmKFiV7x5eA+rmDHH/BHIqwf6cyy3s0j+reOC+SYXycUwX++6CvAqtf05pYfTMJl/9b0NsZ92ZmUw==" "sha512-IWRLcqolnlpYjNOvuvsdQxD0iJaA9sg7nVMVlqWihPNNvr/0CdI7zIau5rrRDIkWBvB1xvR1XLU22ifbVpPzpw==" ]; }
+{ hashesBase16 = [ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ]; hashesBase32 = [ "3y8bwfr609h3lh9ch0izcqq7fl" "26mrvc0v1nslch8r0w45zywsbc" "1v4gi57l97pmnylq6lmgxkhd5v" "143xibwh31h9bvxzalr0sjvbbvpa6ffs" "i4hj30pkrfdpgc5dbcgcydqviibfhm6d" "fxz2p030yba2bza71qhss79k3l5y24kd" "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73" "0qy6iz9yh6a079757mxdmypx0gcmnzjd3ij5q78bzk00vxll82lh" "0mkygpci4r4yb8zz5rs2kxcgvw0a2yf5zlj6r8qgfll6pnrqf0xd" "0zdl9zrg8r3i9c1g90lgg9ip5ijzv3yhz91i0zzn3r8ap9ws784gkp9dk9j3aglhgf1amqb0pj21mh7h1nxcl18akqvvf7ggqsy30yg" "19ncrpp37dx0nzzjw4k6zaqkb9mzaq2myhgpzh5aff7qqcj5wwdxslg6ixwncm7gyq8l761gwf87fgsh2bwfyr52s53k2dkqvw8c24x" "2kz74snvckxldmmbisz9ikmy031d28cs6xfdbl6rhxx42glpyz4vww4lajrc5akklxwixl0js4g84233pxvmbykiic5m7i5m9r4nr11" ]; hashesBase64 = [ "1B2M2Y8AsgTpgAmY7PhCfg==" "bGnufyEcZAQZ1TZswHauRg==" "uzQ4+6vUYOptvSfRU+IjOw==" "2jmj7l5rSw0yVb/vlWAYkK/YBwk=" "zVToVowbN88eW62wd5vL84IhIYk=" "bRLhCx0zHa0hDkf9JdTyYIArfnc=" "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" "kApEad8AzL/QwUXG0eS3lT3Qr6+t11NOOkAZ6NOPxmM=" "rQOHs72GUvcwykbSX5wXCvD9WJ9C5/I/Wp5kEtl9flY=" "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" "nQiG+MaziTmKFiV7x5eA+rmDHH/BHIqwf6cyy3s0j+reOC+SYXycUwX++6CvAqtf05pYfTMJl/9b0NsZ92ZmUw==" "IWRLcqolnlpYjNOvuvsdQxD0iJaA9sg7nVMVlqWihPNNvr/0CdI7zIau5rrRDIkWBvB1xvR1XLU22ifbVpPzpw==" ]; hashesNix32 = [ "3y8bwfr609h3lh9ch0izcqq7fl" "26mrvc0v1nslch8r0w45zywsbc" "1v4gi57l97pmnylq6lmgxkhd5v" "143xibwh31h9bvxzalr0sjvbbvpa6ffs" "i4hj30pkrfdpgc5dbcgcydqviibfhm6d" "fxz2p030yba2bza71qhss79k3l5y24kd" "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73" "0qy6iz9yh6a079757mxdmypx0gcmnzjd3ij5q78bzk00vxll82lh" "0mkygpci4r4yb8zz5rs2kxcgvw0a2yf5zlj6r8qgfll6pnrqf0xd" "0zdl9zrg8r3i9c1g90lgg9ip5ijzv3yhz91i0zzn3r8ap9ws784gkp9dk9j3aglhgf1amqb0pj21mh7h1nxcl18akqvvf7ggqsy30yg" "19ncrpp37dx0nzzjw4k6zaqkb9mzaq2myhgpzh5aff7qqcj5wwdxslg6ixwncm7gyq8l761gwf87fgsh2bwfyr52s53k2dkqvw8c24x" "2kz74snvckxldmmbisz9ikmy031d28cs6xfdbl6rhxx42glpyz4vww4lajrc5akklxwixl0js4g84233pxvmbykiic5m7i5m9r4nr11" ]; hashesSRI = [ "md5-1B2M2Y8AsgTpgAmY7PhCfg==" "md5-bGnufyEcZAQZ1TZswHauRg==" "md5-uzQ4+6vUYOptvSfRU+IjOw==" "sha1-2jmj7l5rSw0yVb/vlWAYkK/YBwk=" "sha1-zVToVowbN88eW62wd5vL84IhIYk=" "sha1-bRLhCx0zHa0hDkf9JdTyYIArfnc=" "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" "sha256-kApEad8AzL/QwUXG0eS3lT3Qr6+t11NOOkAZ6NOPxmM=" "sha256-rQOHs72GUvcwykbSX5wXCvD9WJ9C5/I/Wp5kEtl9flY=" "sha512-z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" "sha512-nQiG+MaziTmKFiV7x5eA+rmDHH/BHIqwf6cyy3s0j+reOC+SYXycUwX++6CvAqtf05pYfTMJl/9b0NsZ92ZmUw==" "sha512-IWRLcqolnlpYjNOvuvsdQxD0iJaA9sg7nVMVlqWihPNNvr/0CdI7zIau5rrRDIkWBvB1xvR1XLU22ifbVpPzpw==" ]; }
diff --git a/tests/functional/lang/eval-okay-convertHash.nix b/tests/functional/lang/eval-okay-convertHash.nix
index cf4909aaf..a0191ee8d 100644
--- a/tests/functional/lang/eval-okay-convertHash.nix
+++ b/tests/functional/lang/eval-okay-convertHash.nix
@@ -5,12 +5,14 @@ let
   map2' = f: fsts: snds: map2 f { inherit fsts snds; };
   getOutputHashes = hashes: {
     hashesBase16 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base16";}) hashAlgos hashes;
+    hashesNix32 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "nix32";}) hashAlgos hashes;
     hashesBase32 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";}) hashAlgos hashes;
     hashesBase64 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base64";}) hashAlgos hashes;
     hashesSRI    = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "sri"   ;}) hashAlgos hashes;
   };
   getOutputHashesColon = hashes: {
     hashesBase16 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "base16";}) hashAlgos hashes;
+    hashesNix32 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "nix32";}) hashAlgos hashes;
     hashesBase32 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "base32";}) hashAlgos hashes;
     hashesBase64 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "base64";}) hashAlgos hashes;
     hashesSRI    = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "sri"   ;}) hashAlgos hashes;
diff --git a/tests/unit/libutil/hash.cc b/tests/unit/libutil/hash.cc
index 4d82c7f09..a88994d0b 100644
--- a/tests/unit/libutil/hash.cc
+++ b/tests/unit/libutil/hash.cc
@@ -80,7 +80,7 @@ namespace nix {
      * --------------------------------------------------------------------------*/
 
     TEST(hashFormat, testRoundTripPrintParse) {
-        for (const HashFormat hashFormat: { HashFormat::Base64, HashFormat::Base32, HashFormat::Base16, HashFormat::SRI}) {
+        for (const HashFormat hashFormat: { HashFormat::Base64, HashFormat::Nix32, HashFormat::Base16, HashFormat::SRI}) {
             ASSERT_EQ(parseHashFormat(printHashFormat(hashFormat)), hashFormat);
             ASSERT_EQ(*parseHashFormatOpt(printHashFormat(hashFormat)), hashFormat);
         }

From 7ff876b92b590fd9559472935f4adce1d3d5efb7 Mon Sep 17 00:00:00 2001
From: Peter Kolloch <info@eigenvalue.net>
Date: Tue, 28 Nov 2023 15:58:04 +0100
Subject: [PATCH 269/294] Add deprecation notice for old nix hash conversion
 subcommands.

(But not yet nix-hash since `nix hash` is still hidden behind a feature flag.)

https://github.com/NixOS/nix/issues/8876
---
 src/nix/hash.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index f9c7592a3..2c9deb0d5 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -127,6 +127,7 @@ struct CmdToBase : Command
 
     void run() override
     {
+        warn("The old format conversion sub commands of `nix hash` where deprecated in favor of `nix hash convert`.");
         for (auto s : args)
             logger->cout(Hash::parseAny(s, ht).to_string(hashFormat, hashFormat == HashFormat::SRI));
     }
@@ -208,6 +209,9 @@ static auto rCmdHash = registerCommand<CmdHash>("hash");
 /* Legacy nix-hash command. */
 static int compatNixHash(int argc, char * * argv)
 {
+    // Wait until `nix hash convert` is not hidden behind experimental flags anymore.
+    // warn("`nix-hash` has been deprecated in favor of `nix hash convert`.");
+
     std::optional<HashAlgorithm> ha;
     bool flat = false;
     HashFormat hashFormat = HashFormat::Base16;

From 8afeaf05c4063d48e65d2d82c31c3323c3237f7c Mon Sep 17 00:00:00 2001
From: Peter Kolloch <info@eigenvalue.net>
Date: Tue, 28 Nov 2023 19:02:15 +0100
Subject: [PATCH 270/294] Add docs/rl-notes for `nix hash convert` /
 `builtins.convertHash`

https://github.com/NixOS/nix/issues/8876
---
 doc/manual/rl-next/hash-format-nix32.md | 22 ++++++++++++
 doc/manual/rl-next/nix-hash-convert.md  | 47 +++++++++++++++++++++++++
 src/libexpr/primops.cc                  |  8 ++---
 src/nix/hash.cc                         |  2 +-
 4 files changed, 74 insertions(+), 5 deletions(-)
 create mode 100644 doc/manual/rl-next/hash-format-nix32.md
 create mode 100644 doc/manual/rl-next/nix-hash-convert.md

diff --git a/doc/manual/rl-next/hash-format-nix32.md b/doc/manual/rl-next/hash-format-nix32.md
new file mode 100644
index 000000000..20c557da9
--- /dev/null
+++ b/doc/manual/rl-next/hash-format-nix32.md
@@ -0,0 +1,22 @@
+synopsis: Rename hash format `base32` to `nix32`
+prs: #9452
+description: {
+
+Hash format `base32` was renamed to `nix32` since it used a special nix-specific character set for 
+[Base32](https://en.wikipedia.org/wiki/Base32).
+
+## Deprecation: Use `nix32` instead of `base32` as `toHashFormat`
+
+For the builtin `convertHash`, the `toHashFormat` parameter now accepts the same hash formats as the `--to`/`--from`
+parameters of the `nix hash conert` command: `"base16"`, `"nix32"`, `"base64"`, and `"sri"`. The former `"base32"` value
+remains as a deprecated alias for `"base32"`. Please convert your code from:
+
+```nix
+builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";}
+```
+
+to
+
+```nix
+builtins.convertHash { inherit hash hashAlgo; toHashFormat = "nix32";}
+```
\ No newline at end of file
diff --git a/doc/manual/rl-next/nix-hash-convert.md b/doc/manual/rl-next/nix-hash-convert.md
new file mode 100644
index 000000000..de4367c5b
--- /dev/null
+++ b/doc/manual/rl-next/nix-hash-convert.md
@@ -0,0 +1,47 @@
+synopsis: Add `nix hash convert`
+prs: #9452
+description: {
+
+New [`nix hash convert`](https://github.com/NixOS/nix/issues/8876) sub command with a fast track
+to stabilization! Examples:
+
+- Convert the hash to `nix32`.
+
+  ```bash
+  $ nix hash convert --algo "sha1" --to nix32 "800d59cfcd3c05e900cb4e214be48f6b886a08df"
+  vw46m23bizj4n8afrc0fj19wrp7mj3c0
+  ```
+  `nix32` is a base32 encoding with a nix-specific character set.
+  Explicitly specify the hashing algorithm (optional with SRI hashes) but detect hash format by the length of the input
+  hash.
+- Convert the hash to the `sri` format that includes an algorithm specification:
+  ```bash
+  nix hash convert --algo "sha1" "800d59cfcd3c05e900cb4e214be48f6b886a08df"
+  sha1-gA1Zz808BekAy04hS+SPa4hqCN8=
+  ```
+  or with an explicit `-to` format:
+  ```bash
+  nix hash convert --algo "sha1" --to sri "800d59cfcd3c05e900cb4e214be48f6b886a08df"
+  sha1-gA1Zz808BekAy04hS+SPa4hqCN8=
+  ```
+- Assert the input format of the hash:
+  ```bash
+  nix hash convert --algo "sha256" --from nix32 "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0="
+  error: input hash 'ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=' does not have the expected format '--from nix32'
+  nix hash convert --algo "sha256" --from nix32 "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s"
+  sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=
+  ```
+
+The `--to`/`--from`/`--algo` parameters have context-sensitive auto-completion.
+
+## Related Deprecations
+
+The following commands are still available but will emit a deprecation warning. Please convert your code to
+`nix hash convert`:
+
+- `nix hash to-base16 $hash1 $hash2`: Use `nix hash convert --to base16 $hash1 $hash2` instead.
+- `nix hash to-base32 $hash1 $hash2`: Use `nix hash convert --to nix32 $hash1 $hash2` instead.
+- `nix hash to-base64 $hash1 $hash2`: Use `nix hash convert --to base64 $hash1 $hash2` instead.
+- `nix hash to-sri $hash1 $hash2`: : Use `nix hash convert --to sri $hash1 $hash2`
+  or even just `nix hash convert $hash1 $hash2` instead.
+}
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 7831f3803..4162a8da3 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1339,7 +1339,7 @@ drvName, Bindings * attrs, Value & v)
                 .errPos = state.positions[noPos]
             });
 
-        auto ht = parseHashAlgoOpt(outputHashAlgo).value_or(HashAlgorithm::SHA256);
+        auto ha = parseHashAlgoOpt(outputHashAlgo).value_or(HashAlgorithm::SHA256);
         auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive);
 
         for (auto & i : outputs) {
@@ -1348,13 +1348,13 @@ drvName, Bindings * attrs, Value & v)
                 drv.outputs.insert_or_assign(i,
                     DerivationOutput::Impure {
                         .method = method,
-                        .hashAlgo = ht,
+                        .hashAlgo = ha,
                     });
             else
                 drv.outputs.insert_or_assign(i,
                     DerivationOutput::CAFloating {
                         .method = method,
-                        .hashAlgo = ht,
+                        .hashAlgo = ha,
                     });
         }
     }
@@ -3837,7 +3837,7 @@ static RegisterPrimOp primop_convertHash({
 
         The format of the resulting hash. Must be one of
         - `"base16"`
-        - `"base32"`
+        - `"nix32"`
         - `"base64"`
         - `"sri"`
 
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index 2c9deb0d5..0bba3b7d2 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -118,7 +118,7 @@ struct CmdToBase : Command
 
     std::string description() override
     {
-        return fmt("convert a hash to %s representation",
+        return fmt("convert a hash to %s representation (deprecated, use `nix hash convert` instead)",
             hashFormat == HashFormat::Base16 ? "base-16" :
             hashFormat == HashFormat::Nix32 ? "base-32" :
             hashFormat == HashFormat::Base64 ? "base-64" :

From d38ec1285573c98c987ec1421f7cec68754204f9 Mon Sep 17 00:00:00 2001
From: Peter Kolloch <info@eigenvalue.net>
Date: Sat, 2 Dec 2023 11:53:50 +0100
Subject: [PATCH 271/294] Update src/libexpr/primops.cc
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com>
---
 src/libexpr/primops.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 4162a8da3..828d118eb 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -3838,6 +3838,7 @@ static RegisterPrimOp primop_convertHash({
         The format of the resulting hash. Must be one of
         - `"base16"`
         - `"nix32"`
+        - `"base32"` (deprecated alias for `"nix32"`)
         - `"base64"`
         - `"sri"`
 

From bbba2055f0b77e9677ef318ceea3084906eccd7d Mon Sep 17 00:00:00 2001
From: Peter Kolloch <info@eigenvalue.net>
Date: Sat, 2 Dec 2023 16:43:52 +0100
Subject: [PATCH 272/294] Refactor concurrently added tests to use
 HashAlgorithm.

https://github.com/NixOS/nix/issues/8876
---
 tests/unit/libutil-support/tests/hash.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/unit/libutil-support/tests/hash.cc b/tests/unit/libutil-support/tests/hash.cc
index 577e9890e..50889cd33 100644
--- a/tests/unit/libutil-support/tests/hash.cc
+++ b/tests/unit/libutil-support/tests/hash.cc
@@ -11,7 +11,7 @@ using namespace nix;
 
 Gen<Hash> Arbitrary<Hash>::arbitrary()
 {
-    Hash hash(htSHA1);
+    Hash hash(HashAlgorithm::SHA1);
     for (size_t i = 0; i < hash.hashSize; ++i)
         hash.hash[i] = *gen::arbitrary<uint8_t>();
     return gen::just(hash);

From e9a5365db66737d1438fd91eba6529d278e1efca Mon Sep 17 00:00:00 2001
From: Peter Kolloch <info@eigenvalue.net>
Date: Sat, 2 Dec 2023 18:19:51 +0100
Subject: [PATCH 273/294] hash.sh: Make failure tests more tolerant of
 additional output

"warning: you don'\''t have Internet access; disabling some network-dependent features" ...

https://github.com/NixOS/nix/issues/8876
---
 tests/functional/hash.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/functional/hash.sh b/tests/functional/hash.sh
index 278ed83b9..47eed5178 100644
--- a/tests/functional/hash.sh
+++ b/tests/functional/hash.sh
@@ -173,11 +173,11 @@ try3() {
     #
 
     fail=$(nix hash convert --algo "$1" --from nix32 "$2" 2>&1 || echo "exit: $?")
-    [[ "$fail" == "error: input hash"*"exit: 1" ]]
+    [[ "$fail" == *"error: input hash"*"exit: 1" ]]
     fail=$(nix hash convert --algo "$1" --from base16 "$3" 2>&1 || echo "exit: $?")
-    [[ "$fail" == "error: input hash"*"exit: 1" ]]
+    [[ "$fail" == *"error: input hash"*"exit: 1" ]]
     fail=$(nix hash convert --algo "$1" --from nix32 "$4" 2>&1 || echo "exit: $?")
-    [[ "$fail" == "error: input hash"*"exit: 1" ]]
+    [[ "$fail" == *"error: input hash"*"exit: 1" ]]
 
 }
 

From 9a1a3c43bf11912ad32c433219c4c21a1b6ca9dd Mon Sep 17 00:00:00 2001
From: Peter Kolloch <info@eigenvalue.net>
Date: Sun, 3 Dec 2023 09:50:44 +0100
Subject: [PATCH 274/294] Store.xs: fix references to HashFormat::Nix32

https://github.com/NixOS/nix/issues/8876
---
 perl/lib/Nix/Store.xs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index 50148141b..82c7db608 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -78,7 +78,7 @@ SV * queryReferences(char * path)
 SV * queryPathHash(char * path)
     PPCODE:
         try {
-            auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(HashFormat::Base32, true);
+            auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(HashFormat::Nix32, true);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -104,7 +104,7 @@ SV * queryPathInfo(char * path, int base32)
                 XPUSHs(&PL_sv_undef);
             else
                 XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0)));
-            auto s = info->narHash.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, true);
+            auto s = info->narHash.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, true);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
             mXPUSHi(info->registrationTime);
             mXPUSHi(info->narSize);
@@ -206,7 +206,7 @@ SV * hashPath(char * algo, int base32, char * path)
     PPCODE:
         try {
             Hash h = hashPath(parseHashAlgo(algo), path).first;
-            auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false);
+            auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -217,7 +217,7 @@ SV * hashFile(char * algo, int base32, char * path)
     PPCODE:
         try {
             Hash h = hashFile(parseHashAlgo(algo), path);
-            auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false);
+            auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -228,7 +228,7 @@ SV * hashString(char * algo, int base32, char * s)
     PPCODE:
         try {
             Hash h = hashString(parseHashAlgo(algo), s);
-            auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false);
+            auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -239,7 +239,7 @@ SV * convertHash(char * algo, char * s, int toBase32)
     PPCODE:
         try {
             auto h = Hash::parseAny(s, parseHashAlgo(algo));
-            auto s = h.to_string(toBase32 ? HashFormat::Base32 : HashFormat::Base16, false);
+            auto s = h.to_string(toBase32 ? HashFormat::Nix32 : HashFormat::Base16, false);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());

From bf00d5ecef20c11eb7e49dff3482b9e536cf7abe Mon Sep 17 00:00:00 2001
From: Bryan Honof <bryanhonof@gmail.com>
Date: Thu, 7 Dec 2023 11:04:48 +0100
Subject: [PATCH 275/294] fix(libutil/tarfile): add option to libarchive so it
 behaves correctly with AppleDouble files

AppleDouble files were extracted differently on macOS machines than on other
UNIX's.
Setting `archive_read_set_format_option(this->archive, NULL ,"mac-ext",NULL)`
fixes this problem, since it just ignores the AppleDouble file and treats it as
a normal one.
This was a problem since it caused source archives to be different between macOS
and Linux.

Ref: nixos/nix#9290
---
 src/libutil/tarfile.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc
index 1733c791c..187b3e948 100644
--- a/src/libutil/tarfile.cc
+++ b/src/libutil/tarfile.cc
@@ -53,6 +53,7 @@ TarArchive::TarArchive(Source & source, bool raw) : buffer(65536)
         archive_read_support_format_raw(archive);
         archive_read_support_format_empty(archive);
     }
+    archive_read_set_option(archive, NULL, "mac-ext", NULL);
     check(archive_read_open(archive, (void *)this, callback_open, callback_read, callback_close), "Failed to open archive (%s)");
 }
 
@@ -63,6 +64,7 @@ TarArchive::TarArchive(const Path & path)
 
     archive_read_support_filter_all(archive);
     archive_read_support_format_all(archive);
+    archive_read_set_option(archive, NULL, "mac-ext", NULL);
     check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s");
 }
 

From a5521b7d9445af63a159d4fe7b44a0902c3a2a24 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 7 Dec 2023 10:49:29 -0500
Subject: [PATCH 276/294] Factor out
 `ServeProto::Serialiser<UnkeyedValidPathInfo>` and test

In the process, partially undo e89b5bd0bfeb4dfdd8fe7e6929544cb9ceb8a505
in that the ancient < 2.4 version is now supported again by the
serializer again. `LegacySSHStore`, instead of also asserting that the
version is at least 4, just checks that `narHash` is set.

This allows us to better test the serializer in isolation for both
versions (< 4 and >= 4).
---
 src/libstore/legacy-ssh-store.cc              |  22 ++---
 src/libstore/serve-protocol.cc                |  44 ++++++++++
 src/libstore/serve-protocol.hh                |   3 +
 src/nix-store/nix-store.cc                    |  12 +--
 .../unkeyed-valid-path-info-2.3.bin           | Bin 0 -> 184 bytes
 .../unkeyed-valid-path-info-2.4.bin           | Bin 0 -> 648 bytes
 tests/unit/libstore/serve-protocol.cc         |  77 ++++++++++++++++++
 7 files changed, 131 insertions(+), 27 deletions(-)
 create mode 100644 tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.3.bin
 create mode 100644 tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.4.bin

diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index fb1580dd6..277445ee6 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -172,24 +172,12 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
             if (p.empty()) return callback(nullptr);
             auto path2 = parseStorePath(p);
             assert(path == path2);
-            /* Hash will be set below. FIXME construct ValidPathInfo at end. */
-            auto info = std::make_shared<ValidPathInfo>(path, Hash::dummy);
+            auto info = std::make_shared<ValidPathInfo>(
+                path,
+                ServeProto::Serialise<UnkeyedValidPathInfo>::read(*this, *conn));
 
-            auto deriver = readString(conn->from);
-            if (deriver != "")
-                info->deriver = parseStorePath(deriver);
-            info->references = ServeProto::Serialise<StorePathSet>::read(*this, *conn);
-            readLongLong(conn->from); // download size
-            info->narSize = readLongLong(conn->from);
-
-            {
-                auto s = readString(conn->from);
-                if (s == "")
-                    throw Error("NAR hash is now mandatory");
-                info->narHash = Hash::parseAnyPrefixed(s);
-            }
-            info->ca = ContentAddress::parseOpt(readString(conn->from));
-            info->sigs = readStrings<StringSet>(conn->from);
+            if (info->narHash == Hash::dummy)
+                throw Error("NAR hash is now mandatory");
 
             auto s = readString(conn->from);
             assert(s == "");
diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc
index fb33553c5..c37b3095c 100644
--- a/src/libstore/serve-protocol.cc
+++ b/src/libstore/serve-protocol.cc
@@ -5,6 +5,7 @@
 #include "serve-protocol.hh"
 #include "serve-protocol-impl.hh"
 #include "archive.hh"
+#include "path-info.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -54,4 +55,47 @@ void ServeProto::Serialise<BuildResult>::write(const StoreDirConfig & store, Ser
     }
 }
 
+
+UnkeyedValidPathInfo ServeProto::Serialise<UnkeyedValidPathInfo>::read(const StoreDirConfig & store, ReadConn conn)
+{
+    /* Hash should be set below unless very old `nix-store --serve`.
+       Caller should assert that it did set it. */
+    UnkeyedValidPathInfo info { Hash::dummy };
+
+    auto deriver = readString(conn.from);
+    if (deriver != "")
+        info.deriver = store.parseStorePath(deriver);
+    info.references = ServeProto::Serialise<StorePathSet>::read(store, conn);
+
+    readLongLong(conn.from); // download size, unused
+    info.narSize = readLongLong(conn.from);
+
+    if (GET_PROTOCOL_MINOR(conn.version) >= 4) {
+        auto s = readString(conn.from);
+        if (!s.empty())
+            info.narHash = Hash::parseAnyPrefixed(s);
+        info.ca = ContentAddress::parseOpt(readString(conn.from));
+        info.sigs = readStrings<StringSet>(conn.from);
+    }
+
+    return info;
+}
+
+void ServeProto::Serialise<UnkeyedValidPathInfo>::write(const StoreDirConfig & store, WriteConn conn, const UnkeyedValidPathInfo & info)
+{
+    conn.to
+        << (info.deriver ? store.printStorePath(*info.deriver) : "");
+
+    ServeProto::write(store, conn, info.references);
+    // !!! Maybe we want compression?
+    conn.to
+        << info.narSize // downloadSize, lie a little
+        << info.narSize;
+    if (GET_PROTOCOL_MINOR(conn.version) >= 4)
+        conn.to
+            << info.narHash.to_string(HashFormat::Nix32, true)
+            << renderContentAddress(info.ca)
+            << info.sigs;
+}
+
 }
diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh
index 6e9d66e2d..ada67a149 100644
--- a/src/libstore/serve-protocol.hh
+++ b/src/libstore/serve-protocol.hh
@@ -18,6 +18,7 @@ struct Source;
 
 // items being serialised
 struct BuildResult;
+struct UnkeyedValidPathInfo;
 
 
 /**
@@ -141,6 +142,8 @@ inline std::ostream & operator << (std::ostream & s, ServeProto::Command op)
 
 template<>
 DECLARE_SERVE_SERIALISER(BuildResult);
+template<>
+DECLARE_SERVE_SERIALISER(UnkeyedValidPathInfo);
 
 template<typename T>
 DECLARE_SERVE_SERIALISER(std::vector<T>);
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index db45be2a8..45af7879c 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -891,16 +891,8 @@ static void opServe(Strings opFlags, Strings opArgs)
                 for (auto & i : paths) {
                     try {
                         auto info = store->queryPathInfo(i);
-                        out << store->printStorePath(info->path)
-                            << (info->deriver ? store->printStorePath(*info->deriver) : "");
-                        ServeProto::write(*store, wconn, info->references);
-                        // !!! Maybe we want compression?
-                        out << info->narSize // downloadSize
-                            << info->narSize;
-                        if (GET_PROTOCOL_MINOR(clientVersion) >= 4)
-                            out << info->narHash.to_string(HashFormat::Nix32, true)
-                                << renderContentAddress(info->ca)
-                                << info->sigs;
+                        out << store->printStorePath(info->path);
+                        ServeProto::write(*store, wconn, static_cast<const UnkeyedValidPathInfo &>(*info));
                     } catch (InvalidPath &) {
                     }
                 }
diff --git a/tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.3.bin b/tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.3.bin
new file mode 100644
index 0000000000000000000000000000000000000000..8056ec055ed2039814ab2654aad984f07115518f
GIT binary patch
literal 184
zcmZQzKm~Rk5I&4HhDz(_Wmf1Hm*f|v>Zco)n`cxS7viFIlM;*cQi{sJIvJt*ahg+E
XS&435ArJt~*G<dMhnoX)4~zx?9JnZs

literal 0
HcmV?d00001

diff --git a/tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.4.bin b/tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.4.bin
new file mode 100644
index 0000000000000000000000000000000000000000..521b5c423c68ba6b3ec7562e39e6137983e77dc6
GIT binary patch
literal 648
zcmb`D+fIcr5Jk~P{{^A6rGkksehIXcUbwXwpnQR!?eWl>cp=fmdD+P%du3+9=$K@X
zTQa|lwn<vB65*gZ%eb!WJ+sTTc;ox<jN_d9$}^X_@MAuXADpMx0}%4j^JY<HI4{$}
z^{e4n7KSjyda|KODQb*LjinRrhJC07%33~Ci3P2V={@0doZk7%e{Vf53Hdi~(GR@=
zubLIyT7;*HAjxq%?S&$}<ZVW{c=r?qFCqWS&$Iu7%dKIhExm8hS@1xXoKX#|5}C^9
tWs;im9mCai(AbXFHA?Z`2>`cL3n8wg;O8!IE8WDiY-4aNzU{MP{{~7(rMCb8

literal 0
HcmV?d00001

diff --git a/tests/unit/libstore/serve-protocol.cc b/tests/unit/libstore/serve-protocol.cc
index 6d2054f7d..c2298c6db 100644
--- a/tests/unit/libstore/serve-protocol.cc
+++ b/tests/unit/libstore/serve-protocol.cc
@@ -225,6 +225,83 @@ VERSIONED_CHARACTERIZATION_TEST(
         t;
     }))
 
+VERSIONED_CHARACTERIZATION_TEST(
+    ServeProtoTest,
+    unkeyedValidPathInfo_2_3,
+    "unkeyed-valid-path-info-2.3",
+    2 << 8 | 3,
+    (std::tuple<UnkeyedValidPathInfo, UnkeyedValidPathInfo> {
+        ({
+            UnkeyedValidPathInfo info { Hash::dummy };
+            info.narSize = 34878;
+            info;
+        }),
+        ({
+            UnkeyedValidPathInfo info { Hash::dummy };
+            info.deriver = StorePath {
+                "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
+            };
+            info.references = {
+                StorePath {
+                    "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo.drv",
+                },
+            };
+            info.narSize = 34878;
+            info;
+        }),
+    }))
+
+VERSIONED_CHARACTERIZATION_TEST(
+    ServeProtoTest,
+    unkeyedValidPathInfo_2_4,
+    "unkeyed-valid-path-info-2.4",
+    2 << 8 | 4,
+    (std::tuple<UnkeyedValidPathInfo, UnkeyedValidPathInfo> {
+        ({
+            UnkeyedValidPathInfo info {
+                Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
+            };
+            info.deriver = StorePath {
+                "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
+            };
+            info.references = {
+                StorePath {
+                    "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo.drv",
+                },
+            };
+            info.narSize = 34878;
+            info;
+        }),
+        ({
+            ValidPathInfo info {
+                *LibStoreTest::store,
+                "foo",
+                FixedOutputInfo {
+                    .method = FileIngestionMethod::Recursive,
+                    .hash = hashString(HashAlgorithm::SHA256, "(...)"),
+                    .references = {
+                        .others = {
+                            StorePath {
+                                "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar",
+                            },
+                        },
+                        .self = true,
+                    },
+                },
+                Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
+            };
+            info.deriver = StorePath {
+                "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
+            };
+            info.narSize = 34878;
+            info.sigs = {
+                "fake-sig-1",
+                "fake-sig-2",
+            },
+            static_cast<UnkeyedValidPathInfo>(std::move(info));
+        }),
+    }))
+
 VERSIONED_CHARACTERIZATION_TEST(
     ServeProtoTest,
     vector,

From 0b80935c22f367b1deecffeddb97c90d7ed985e9 Mon Sep 17 00:00:00 2001
From: Rebecca Turner <rbt@sent.as>
Date: Thu, 7 Dec 2023 10:01:42 -0800
Subject: [PATCH 277/294] Pass positions when evaluating
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This includes position information in more places, making debugging
easier.

Before:

```
$ nix-instantiate --show-trace --eval tests/functional/lang/eval-fail-using-set-as-attr-name.nix
error:
       … while evaluating an attribute name

         at «none»:0: (source not available)

       error: value is a set while a string was expected
```

After:

```
error:
       … while evaluating an attribute name

         at /pwd/lang/eval-fail-using-set-as-attr-name.nix:5:10:

            4| in
            5|   attr.${key}
             |          ^
            6|

       error: value is a set while a string was expected
```
---
 .../rl-next/source-positions-in-errors.md     | 45 +++++++++++++++++++
 src/libexpr/eval-inline.hh                    | 12 ++---
 src/libexpr/eval.cc                           | 18 ++++----
 src/libexpr/nixexpr.hh                        |  1 +
 .../lang/eval-fail-attr-name-type.err.exp     | 20 +++++++++
 .../lang/eval-fail-attr-name-type.nix         |  7 +++
 .../lang/eval-fail-call-primop.err.exp        | 12 +++++
 .../functional/lang/eval-fail-call-primop.nix |  1 +
 .../lang/eval-fail-not-throws.err.exp         | 18 ++++++++
 .../functional/lang/eval-fail-not-throws.nix  |  1 +
 .../eval-fail-using-set-as-attr-name.err.exp  | 11 +++++
 .../lang/eval-fail-using-set-as-attr-name.nix |  5 +++
 12 files changed, 137 insertions(+), 14 deletions(-)
 create mode 100644 doc/manual/rl-next/source-positions-in-errors.md
 create mode 100644 tests/functional/lang/eval-fail-attr-name-type.err.exp
 create mode 100644 tests/functional/lang/eval-fail-attr-name-type.nix
 create mode 100644 tests/functional/lang/eval-fail-call-primop.err.exp
 create mode 100644 tests/functional/lang/eval-fail-call-primop.nix
 create mode 100644 tests/functional/lang/eval-fail-not-throws.err.exp
 create mode 100644 tests/functional/lang/eval-fail-not-throws.nix
 create mode 100644 tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp
 create mode 100644 tests/functional/lang/eval-fail-using-set-as-attr-name.nix

diff --git a/doc/manual/rl-next/source-positions-in-errors.md b/doc/manual/rl-next/source-positions-in-errors.md
new file mode 100644
index 000000000..00f0b27e8
--- /dev/null
+++ b/doc/manual/rl-next/source-positions-in-errors.md
@@ -0,0 +1,45 @@
+synopsis: Source locations are printed more consistently in errors
+issues: #561
+prs: #9555
+description: {
+
+Source location information is now included in error messages more
+consistently. Given this code:
+
+```nix
+let
+  attr = {foo = "bar";};
+  key = {};
+in
+  attr.${key}
+```
+
+Previously, Nix would show this unhelpful message when attempting to evaluate
+it:
+
+```
+error:
+       … while evaluating an attribute name
+
+         at «none»:0: (source not available)
+
+       error: value is a set while a string was expected
+```
+
+Now, the error message displays where the problematic value was found:
+
+```
+error:
+       … while evaluating an attribute name
+
+         at bad.nix:4:11:
+
+            3|   key = {};
+            4| in attr.${key}
+             |           ^
+            5|
+
+       error: value is a set while a string was expected
+```
+
+}
diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh
index a988fa40c..c37b1d62b 100644
--- a/src/libexpr/eval-inline.hh
+++ b/src/libexpr/eval-inline.hh
@@ -103,8 +103,10 @@ void EvalState::forceValue(Value & v, Callable getPos)
             throw;
         }
     }
-    else if (v.isApp())
-        callFunction(*v.app.left, *v.app.right, v, noPos);
+    else if (v.isApp()) {
+        PosIdx pos = getPos();
+        callFunction(*v.app.left, *v.app.right, v, pos);
+    }
     else if (v.isBlackhole())
         error("infinite recursion encountered").atPos(getPos()).template debugThrow<EvalError>();
 }
@@ -121,9 +123,9 @@ template <typename Callable>
 [[gnu::always_inline]]
 inline void EvalState::forceAttrs(Value & v, Callable getPos, std::string_view errorCtx)
 {
-    forceValue(v, noPos);
+    PosIdx pos = getPos();
+    forceValue(v, pos);
     if (v.type() != nAttrs) {
-        PosIdx pos = getPos();
         error("value is %1% while a set was expected", showType(v)).withTrace(pos, errorCtx).debugThrow<TypeError>();
     }
 }
@@ -132,7 +134,7 @@ inline void EvalState::forceAttrs(Value & v, Callable getPos, std::string_view e
 [[gnu::always_inline]]
 inline void EvalState::forceList(Value & v, const PosIdx pos, std::string_view errorCtx)
 {
-    forceValue(v, noPos);
+    forceValue(v, pos);
     if (!v.isList()) {
         error("value is %1% while a list was expected", showType(v)).withTrace(pos, errorCtx).debugThrow<TypeError>();
     }
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 7e68e6f9b..8a6e07fb0 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -344,7 +344,7 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env)
     } else {
         Value nameValue;
         name.expr->eval(state, env, nameValue);
-        state.forceStringNoCtx(nameValue, noPos, "while evaluating an attribute name");
+        state.forceStringNoCtx(nameValue, name.expr->getPos(), "while evaluating an attribute name");
         return state.symbols.create(nameValue.string_view());
     }
 }
@@ -1514,7 +1514,7 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v)
     e->eval(state, env, vTmp);
 
     for (auto & i : attrPath) {
-        state.forceValue(*vAttrs, noPos);
+        state.forceValue(*vAttrs, getPos());
         Bindings::iterator j;
         auto name = getName(i, state, env);
         if (vAttrs->type() != nAttrs ||
@@ -1683,7 +1683,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                 if (countCalls) primOpCalls[name]++;
 
                 try {
-                    vCur.primOp->fun(*this, noPos, args, vCur);
+                    vCur.primOp->fun(*this, vCur.determinePos(noPos), args, vCur);
                 } catch (Error & e) {
                     addErrorTrace(e, pos, "while calling the '%1%' builtin", name);
                     throw;
@@ -1731,7 +1731,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                     // 1. Unify this and above code. Heavily redundant.
                     // 2. Create a fake env (arg1, arg2, etc.) and a fake expr (arg1: arg2: etc: builtins.name arg1 arg2 etc)
                     //    so the debugger allows to inspect the wrong parameters passed to the builtin.
-                    primOp->primOp->fun(*this, noPos, vArgs, vCur);
+                    primOp->primOp->fun(*this, vCur.determinePos(noPos), vArgs, vCur);
                 } catch (Error & e) {
                     addErrorTrace(e, pos, "while calling the '%1%' builtin", name);
                     throw;
@@ -1839,7 +1839,7 @@ https://nixos.org/manual/nix/stable/language/constructs.html#functions.)", symbo
         }
     }
 
-    callFunction(fun, allocValue()->mkAttrs(attrs), res, noPos);
+    callFunction(fun, allocValue()->mkAttrs(attrs), res, pos);
 }
 
 
@@ -1875,7 +1875,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v)
 
 void ExprOpNot::eval(EvalState & state, Env & env, Value & v)
 {
-    v.mkBool(!state.evalBool(env, e, noPos, "in the argument of the not operator")); // XXX: FIXME: !
+    v.mkBool(!state.evalBool(env, e, getPos(), "in the argument of the not operator")); // XXX: FIXME: !
 }
 
 
@@ -2316,7 +2316,7 @@ BackedStringView EvalState::coerceToString(
             std::string result;
             for (auto [n, v2] : enumerate(v.listItems())) {
                 try {
-                    result += *coerceToString(noPos, *v2, context,
+                    result += *coerceToString(pos, *v2, context,
                             "while evaluating one element of the list",
                             coerceMore, copyToStore, canonicalizePath);
                 } catch (Error & e) {
@@ -2463,8 +2463,8 @@ SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value &
 
 bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx)
 {
-    forceValue(v1, noPos);
-    forceValue(v2, noPos);
+    forceValue(v1, pos);
+    forceValue(v2, pos);
 
     /* !!! Hack to support some old broken code that relies on pointer
        equality tests between sets.  (Specifically, builderDefs calls
diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh
index 10099d49e..020286815 100644
--- a/src/libexpr/nixexpr.hh
+++ b/src/libexpr/nixexpr.hh
@@ -405,6 +405,7 @@ struct ExprOpNot : Expr
 {
     Expr * e;
     ExprOpNot(Expr * e) : e(e) { };
+    PosIdx getPos() const override { return e->getPos(); }
     COMMON_METHODS
 };
 
diff --git a/tests/functional/lang/eval-fail-attr-name-type.err.exp b/tests/functional/lang/eval-fail-attr-name-type.err.exp
new file mode 100644
index 000000000..5f9a073dd
--- /dev/null
+++ b/tests/functional/lang/eval-fail-attr-name-type.err.exp
@@ -0,0 +1,20 @@
+error:
+       … while evaluating the attribute 'puppy."${key}"'
+
+         at /pwd/lang/eval-fail-attr-name-type.nix:3:5:
+
+            2|   attrs = {
+            3|     puppy.doggy = {};
+             |     ^
+            4|   };
+
+       … while evaluating an attribute name
+
+         at /pwd/lang/eval-fail-attr-name-type.nix:7:17:
+
+            6| in
+            7|   attrs.puppy.${key}
+             |                 ^
+            8|
+
+       error: value is an integer while a string was expected
diff --git a/tests/functional/lang/eval-fail-attr-name-type.nix b/tests/functional/lang/eval-fail-attr-name-type.nix
new file mode 100644
index 000000000..a0e76004a
--- /dev/null
+++ b/tests/functional/lang/eval-fail-attr-name-type.nix
@@ -0,0 +1,7 @@
+let
+  attrs = {
+    puppy.doggy = {};
+  };
+  key = 1;
+in
+  attrs.puppy.${key}
diff --git a/tests/functional/lang/eval-fail-call-primop.err.exp b/tests/functional/lang/eval-fail-call-primop.err.exp
new file mode 100644
index 000000000..19b407c47
--- /dev/null
+++ b/tests/functional/lang/eval-fail-call-primop.err.exp
@@ -0,0 +1,12 @@
+error:
+       … while calling the 'length' builtin
+
+         at /pwd/lang/eval-fail-call-primop.nix:1:1:
+
+            1| builtins.length 1
+             | ^
+            2|
+
+       … while evaluating the first argument passed to builtins.length
+
+       error: value is an integer while a list was expected
diff --git a/tests/functional/lang/eval-fail-call-primop.nix b/tests/functional/lang/eval-fail-call-primop.nix
new file mode 100644
index 000000000..972eb72c7
--- /dev/null
+++ b/tests/functional/lang/eval-fail-call-primop.nix
@@ -0,0 +1 @@
+builtins.length 1
diff --git a/tests/functional/lang/eval-fail-not-throws.err.exp b/tests/functional/lang/eval-fail-not-throws.err.exp
new file mode 100644
index 000000000..b290afb0a
--- /dev/null
+++ b/tests/functional/lang/eval-fail-not-throws.err.exp
@@ -0,0 +1,18 @@
+error:
+       … in the argument of the not operator
+
+         at /pwd/lang/eval-fail-not-throws.nix:1:4:
+
+            1| ! (throw "uh oh!")
+             |    ^
+            2|
+
+       … while calling the 'throw' builtin
+
+         at /pwd/lang/eval-fail-not-throws.nix:1:4:
+
+            1| ! (throw "uh oh!")
+             |    ^
+            2|
+
+       error: uh oh!
diff --git a/tests/functional/lang/eval-fail-not-throws.nix b/tests/functional/lang/eval-fail-not-throws.nix
new file mode 100644
index 000000000..a74ce4ebe
--- /dev/null
+++ b/tests/functional/lang/eval-fail-not-throws.nix
@@ -0,0 +1 @@
+! (throw "uh oh!")
diff --git a/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp b/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp
new file mode 100644
index 000000000..811d01b03
--- /dev/null
+++ b/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp
@@ -0,0 +1,11 @@
+error:
+       … while evaluating an attribute name
+
+         at /pwd/lang/eval-fail-using-set-as-attr-name.nix:5:10:
+
+            4| in
+            5|   attr.${key}
+             |          ^
+            6|
+
+       error: value is a set while a string was expected
diff --git a/tests/functional/lang/eval-fail-using-set-as-attr-name.nix b/tests/functional/lang/eval-fail-using-set-as-attr-name.nix
new file mode 100644
index 000000000..48e071a41
--- /dev/null
+++ b/tests/functional/lang/eval-fail-using-set-as-attr-name.nix
@@ -0,0 +1,5 @@
+let
+  attr = {foo = "bar";};
+  key = {};
+in
+  attr.${key}

From 96dd757b0c0f3d6702f8e38467a8bf467b43154e Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 8 Dec 2023 00:44:55 -0500
Subject: [PATCH 278/294] Give `Derivation::tryResolve` an `evalStore` argument

This is needed for building CA deriations with a src store / dest store
split. In particular it is needed for Hydra.

https://github.com/NixOS/hydra/issues/838 currently puts realizations,
and thus build outputs, in the local store, but it should not.
---
 src/libstore/build/derivation-goal.cc | 2 +-
 src/libstore/derivations.cc           | 4 ++--
 src/libstore/derivations.hh           | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 81eef7c47..d4da374ba 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -558,7 +558,7 @@ void DerivationGoal::inputsRealised()
                  inputDrvOutputs statefully, sometimes it gets out of sync with
                  the real source of truth (store). So we query the store
                  directly if there's a problem. */
-              attempt = fullDrv.tryResolve(worker.store);
+              attempt = fullDrv.tryResolve(worker.store, &worker.evalStore);
             }
             assert(attempt);
             Derivation drvResolved { std::move(*attempt) };
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 664ab7556..c35150b57 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -1002,13 +1002,13 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
 
 }
 
-std::optional<BasicDerivation> Derivation::tryResolve(Store & store) const
+std::optional<BasicDerivation> Derivation::tryResolve(Store & store, Store * evalStore) const
 {
     std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
 
     std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accum;
     accum = [&](auto & inputDrv, auto & node) {
-        for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(inputDrv)) {
+        for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(inputDrv, evalStore)) {
             if (outputPath) {
                 inputDrvOutputs.insert_or_assign({inputDrv, outputName}, *outputPath);
                 if (auto p = get(node.childMap, outputName))
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 290abedcf..2a326b578 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -342,7 +342,7 @@ struct Derivation : BasicDerivation
      * 2. Input placeholders are replaced with realized input store
      *    paths.
      */
-    std::optional<BasicDerivation> tryResolve(Store & store) const;
+    std::optional<BasicDerivation> tryResolve(Store & store, Store * evalStore = nullptr) const;
 
     /**
      * Like the above, but instead of querying the Nix database for

From f0ac2a35d5e9dfb3a53e6cc810e871fe119cbf4b Mon Sep 17 00:00:00 2001
From: Rebecca Turner <rbt@sent.as>
Date: Fri, 8 Dec 2023 11:36:57 -0500
Subject: [PATCH 279/294] Print the value in `error: cannot coerce` messages
 (#9553)

* Print the value in `error: cannot coerce` messages

This extends the `error: cannot coerce a TYPE to a string` message
to print the value that could not be coerced. This helps with debugging
by making it easier to track down where the value is being produced
from, especially in errors with deep or unhelpful stack traces.

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
---
 .../rl-next/print-value-in-coercion-error.md  | 50 +++++++++++++++++++
 .../src/language/string-interpolation.md      |  2 +-
 src/libexpr/eval.cc                           | 10 ++--
 ...al-fail-bad-string-interpolation-1.err.exp |  2 +-
 ...al-fail-bad-string-interpolation-3.err.exp |  2 +-
 tests/unit/libexpr/error_traces.cc            | 28 +++++------
 6 files changed, 73 insertions(+), 21 deletions(-)
 create mode 100644 doc/manual/rl-next/print-value-in-coercion-error.md

diff --git a/doc/manual/rl-next/print-value-in-coercion-error.md b/doc/manual/rl-next/print-value-in-coercion-error.md
new file mode 100644
index 000000000..504ea67b9
--- /dev/null
+++ b/doc/manual/rl-next/print-value-in-coercion-error.md
@@ -0,0 +1,50 @@
+synopsis: Coercion errors include the failing value
+issues: #561
+prs: #9553
+description: {
+
+The `error: cannot coerce a <TYPE> to a string` message now includes the value which caused the error.
+
+Previously, a failed string coercion produced a confusing error message if the trace didn't show where the offending value was defined:
+
+```bash
+$ nix-instantiate --eval --expr '
+let x = { a = 1; }; in
+
+"${x}"
+'
+error:
+       … while evaluating a path segment
+
+         at «string»:4:2:
+
+            3|
+            4| "${x}"
+             |  ^
+            5|
+
+       error: cannot coerce a set to a string
+```
+
+Now, the error message includes the value itself:
+
+```bash
+$ nix-instantiate --eval --expr '
+let x = { a = 1; }; in
+
+"${x}"
+'
+error:
+       … while evaluating a path segment
+
+         at «string»:4:2:
+
+            3|
+            4| "${x}"
+             |  ^
+            5|
+
+       error: cannot coerce a set to a string: { a = 1; }
+```
+
+}
diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md
index e999b287b..6e28d2664 100644
--- a/doc/manual/src/language/string-interpolation.md
+++ b/doc/manual/src/language/string-interpolation.md
@@ -189,7 +189,7 @@ If neither is present, an error is thrown.
 > "${a}"
 > ```
 >
->     error: cannot coerce a set to a string
+>     error: cannot coerce a set to a string: { }
 >
 >            at «string»:4:2:
 >
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 7e68e6f9b..b52274b64 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -26,9 +26,9 @@
 #include <unistd.h>
 #include <sys/time.h>
 #include <sys/resource.h>
-#include <iostream>
 #include <fstream>
 #include <functional>
+#include <strstream>
 
 #include <sys/resource.h>
 #include <nlohmann/json.hpp>
@@ -2286,7 +2286,7 @@ BackedStringView EvalState::coerceToString(
             return std::move(*maybeString);
         auto i = v.attrs->find(sOutPath);
         if (i == v.attrs->end()) {
-            error("cannot coerce %1% to a string", showType(v))
+            error("cannot coerce %1% to a string: %2%", showType(v), printValue(*this, v))
                 .withTrace(pos, errorCtx)
                 .debugThrow<TypeError>();
         }
@@ -2332,7 +2332,7 @@ BackedStringView EvalState::coerceToString(
         }
     }
 
-    error("cannot coerce %1% to a string", showType(v))
+    error("cannot coerce %1% to a string: %2%", showType(v), printValue(*this, v))
         .withTrace(pos, errorCtx)
         .debugThrow<TypeError>();
 }
@@ -2691,8 +2691,10 @@ void EvalState::printStatistics()
 
 std::string ExternalValueBase::coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const
 {
+    std::strstream printed;
+    print(printed);
     throw TypeError({
-        .msg = hintfmt("cannot coerce %1% to a string", showType())
+        .msg = hintfmt("cannot coerce %1% to a string: %2%", showType(), printed.str())
     });
 }
 
diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp
index eb73e9a52..e54ecc6d1 100644
--- a/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp
+++ b/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp
@@ -7,4 +7,4 @@ error:
              |  ^
             2|
 
-       error: cannot coerce a function to a string
+       error: cannot coerce a function to a string: <LAMBDA>
diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp
index ac14f329b..6f0a96f78 100644
--- a/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp
+++ b/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp
@@ -7,4 +7,4 @@ error:
              |   ^
             2|
 
-       error: cannot coerce a function to a string
+       error: cannot coerce a function to a string: <LAMBDA>
diff --git a/tests/unit/libexpr/error_traces.cc b/tests/unit/libexpr/error_traces.cc
index 81498f65a..c2403bee9 100644
--- a/tests/unit/libexpr/error_traces.cc
+++ b/tests/unit/libexpr/error_traces.cc
@@ -295,7 +295,7 @@ namespace nix {
     TEST_F(ErrorTraceTest, toPath) {
         ASSERT_TRACE2("toPath []",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string", "a list"),
+                      hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"),
                       hintfmt("while evaluating the first argument passed to builtins.toPath"));
 
         ASSERT_TRACE2("toPath \"foo\"",
@@ -309,7 +309,7 @@ namespace nix {
     TEST_F(ErrorTraceTest, storePath) {
         ASSERT_TRACE2("storePath true",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string", "a Boolean"),
+                      hintfmt("cannot coerce %s to a string: %s", "a Boolean", "true"),
                       hintfmt("while evaluating the first argument passed to 'builtins.storePath'"));
 
     }
@@ -318,7 +318,7 @@ namespace nix {
     TEST_F(ErrorTraceTest, pathExists) {
         ASSERT_TRACE2("pathExists []",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string", "a list"),
+                      hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"),
                       hintfmt("while realising the context of a path"));
 
         ASSERT_TRACE2("pathExists \"zorglub\"",
@@ -332,7 +332,7 @@ namespace nix {
     TEST_F(ErrorTraceTest, baseNameOf) {
         ASSERT_TRACE2("baseNameOf []",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string", "a list"),
+                      hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"),
                       hintfmt("while evaluating the first argument passed to builtins.baseNameOf"));
 
     }
@@ -377,7 +377,7 @@ namespace nix {
     TEST_F(ErrorTraceTest, filterSource) {
         ASSERT_TRACE2("filterSource [] []",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string", "a list"),
+                      hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"),
                       hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"));
 
         ASSERT_TRACE2("filterSource [] \"foo\"",
@@ -1038,7 +1038,7 @@ namespace nix {
     TEST_F(ErrorTraceTest, toString) {
         ASSERT_TRACE2("toString { a = 1; }",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string", "a set"),
+                      hintfmt("cannot coerce %s to a string: %s", "a set", "{ a = 1; }"),
                       hintfmt("while evaluating the first argument passed to builtins.toString"));
 
     }
@@ -1057,7 +1057,7 @@ namespace nix {
 
         ASSERT_TRACE2("substring 0 3 {}",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string", "a set"),
+                      hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"),
                       hintfmt("while evaluating the third argument (the string) passed to builtins.substring"));
 
         ASSERT_TRACE1("substring (-3) 3 \"sometext\"",
@@ -1070,7 +1070,7 @@ namespace nix {
     TEST_F(ErrorTraceTest, stringLength) {
         ASSERT_TRACE2("stringLength {} # TODO: context is missing ???",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string", "a set"),
+                      hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"),
                       hintfmt("while evaluating the argument passed to builtins.stringLength"));
 
     }
@@ -1143,7 +1143,7 @@ namespace nix {
 
         ASSERT_TRACE2("concatStringsSep \"foo\" [ 1 2 {} ] # TODO: coerce to string is buggy",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string", "an integer"),
+                      hintfmt("cannot coerce %s to a string: %s", "an integer", "1"),
                       hintfmt("while evaluating one element of the list of strings to concat passed to builtins.concatStringsSep"));
 
     }
@@ -1229,12 +1229,12 @@ namespace nix {
 
         ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = {}; }",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string", "a set"),
+                      hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"),
                       hintfmt("while evaluating the attribute 'system' of derivation 'foo'"));
 
         ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = {}; }",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string", "a set"),
+                      hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"),
                       hintfmt("while evaluating the attribute 'outputs' of derivation 'foo'"));
 
         ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"drv\"; }",
@@ -1279,17 +1279,17 @@ namespace nix {
 
         ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ {} ]; }",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string", "a set"),
+                      hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"),
                       hintfmt("while evaluating an element of the argument list"));
 
         ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ \"a\" {} ]; }",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string", "a set"),
+                      hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"),
                       hintfmt("while evaluating an element of the argument list"));
 
         ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; FOO = {}; }",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string", "a set"),
+                      hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"),
                       hintfmt("while evaluating the attribute 'FOO' of derivation 'foo'"));
 
     }

From f9ee1bedcf98334d8bc015c2e04e30fbba958a3e Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 8 Dec 2023 13:18:52 -0500
Subject: [PATCH 280/294] Avoid `std::strstream`, fix the clang build

According https://en.cppreference.com/w/cpp/io/strstream, it has been
deprecated since C++98! The Clang + Linux build systems to not have it
at all, or at least be hiding it.

We can just use `std::stringstream` instead, I think.
---
 src/libexpr/eval.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index b52274b64..5d627224f 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -28,7 +28,7 @@
 #include <sys/resource.h>
 #include <fstream>
 #include <functional>
-#include <strstream>
+#include <sstream>
 
 #include <sys/resource.h>
 #include <nlohmann/json.hpp>
@@ -2691,7 +2691,7 @@ void EvalState::printStatistics()
 
 std::string ExternalValueBase::coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const
 {
-    std::strstream printed;
+    std::stringstream printed;
     print(printed);
     throw TypeError({
         .msg = hintfmt("cannot coerce %1% to a string: %2%", showType(), printed.str())

From ce4ca574d24abe233b717babc679e4c9228ba94b Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 6 Nov 2023 09:04:50 -0500
Subject: [PATCH 281/294] Clarify `SourceAccessor` methods should never
 implicitly follow symlinks

The code has already been fixed (yay!) so what is left of this commit is
just updating the API docs.

Co-authored-by: Cole Helbling <cole.e.helbling@outlook.com>
---
 src/libutil/source-accessor.hh | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh
index 3ca12d624..4f4ff09c1 100644
--- a/src/libutil/source-accessor.hh
+++ b/src/libutil/source-accessor.hh
@@ -26,6 +26,13 @@ struct SourceAccessor
 
     /**
      * Return the contents of a file as a string.
+     *
+     * @note Unlike Unix, this method should *not* follow symlinks. Nix
+     * by default wants to manipulate symlinks explicitly, and not
+     * implictly follow them, as they are frequently untrusted user data
+     * and thus may point to arbitrary locations. Acting on the targets
+     * targets of symlinks should only occasionally be done, and only
+     * with care.
      */
     virtual std::string readFile(const CanonPath & path);
 
@@ -34,7 +41,10 @@ struct SourceAccessor
      * called with the size of the file before any data is written to
      * the sink.
      *
-     * Note: subclasses of `SourceAccessor` need to implement at least
+     * @note Like the other `readFile`, this method should *not* follow
+     * symlinks.
+     *
+     * @note subclasses of `SourceAccessor` need to implement at least
      * one of the `readFile()` variants.
      */
     virtual void readFile(
@@ -87,6 +97,9 @@ struct SourceAccessor
 
     typedef std::map<std::string, DirEntry> DirEntries;
 
+    /**
+     * @note Like `readFile`, this method should *not* follow symlinks.
+     */
     virtual DirEntries readDirectory(const CanonPath & path) = 0;
 
     virtual std::string readLink(const CanonPath & path) = 0;

From 9b7b7a7561b24d48452627709e6872d9c610428b Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sat, 9 Dec 2023 02:13:32 +0100
Subject: [PATCH 282/294] Revert "Print the value in `error: cannot coerce`
 messages (#9553)"

This reverts commit f0ac2a35d5e9dfb3a53e6cc810e871fe119cbf4b.

The request from the sibling PR, which also applies here, was not addressed.
https://github.com/NixOS/nix/pull/9554#issuecomment-1845095735
---
 .../rl-next/print-value-in-coercion-error.md  | 50 -------------------
 .../src/language/string-interpolation.md      |  2 +-
 src/libexpr/eval.cc                           | 10 ++--
 ...al-fail-bad-string-interpolation-1.err.exp |  2 +-
 ...al-fail-bad-string-interpolation-3.err.exp |  2 +-
 tests/unit/libexpr/error_traces.cc            | 28 +++++------
 6 files changed, 21 insertions(+), 73 deletions(-)
 delete mode 100644 doc/manual/rl-next/print-value-in-coercion-error.md

diff --git a/doc/manual/rl-next/print-value-in-coercion-error.md b/doc/manual/rl-next/print-value-in-coercion-error.md
deleted file mode 100644
index 504ea67b9..000000000
--- a/doc/manual/rl-next/print-value-in-coercion-error.md
+++ /dev/null
@@ -1,50 +0,0 @@
-synopsis: Coercion errors include the failing value
-issues: #561
-prs: #9553
-description: {
-
-The `error: cannot coerce a <TYPE> to a string` message now includes the value which caused the error.
-
-Previously, a failed string coercion produced a confusing error message if the trace didn't show where the offending value was defined:
-
-```bash
-$ nix-instantiate --eval --expr '
-let x = { a = 1; }; in
-
-"${x}"
-'
-error:
-       … while evaluating a path segment
-
-         at «string»:4:2:
-
-            3|
-            4| "${x}"
-             |  ^
-            5|
-
-       error: cannot coerce a set to a string
-```
-
-Now, the error message includes the value itself:
-
-```bash
-$ nix-instantiate --eval --expr '
-let x = { a = 1; }; in
-
-"${x}"
-'
-error:
-       … while evaluating a path segment
-
-         at «string»:4:2:
-
-            3|
-            4| "${x}"
-             |  ^
-            5|
-
-       error: cannot coerce a set to a string: { a = 1; }
-```
-
-}
diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md
index 6e28d2664..e999b287b 100644
--- a/doc/manual/src/language/string-interpolation.md
+++ b/doc/manual/src/language/string-interpolation.md
@@ -189,7 +189,7 @@ If neither is present, an error is thrown.
 > "${a}"
 > ```
 >
->     error: cannot coerce a set to a string: { }
+>     error: cannot coerce a set to a string
 >
 >            at «string»:4:2:
 >
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index c04e2d53d..841c223cd 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -27,9 +27,9 @@
 #include <unistd.h>
 #include <sys/time.h>
 #include <sys/resource.h>
+#include <iostream>
 #include <fstream>
 #include <functional>
-#include <sstream>
 
 #include <sys/resource.h>
 #include <nlohmann/json.hpp>
@@ -2230,7 +2230,7 @@ BackedStringView EvalState::coerceToString(
             return std::move(*maybeString);
         auto i = v.attrs->find(sOutPath);
         if (i == v.attrs->end()) {
-            error("cannot coerce %1% to a string: %2%", showType(v), printValue(*this, v))
+            error("cannot coerce %1% to a string", showType(v))
                 .withTrace(pos, errorCtx)
                 .debugThrow<TypeError>();
         }
@@ -2276,7 +2276,7 @@ BackedStringView EvalState::coerceToString(
         }
     }
 
-    error("cannot coerce %1% to a string: %2%", showType(v), printValue(*this, v))
+    error("cannot coerce %1% to a string", showType(v))
         .withTrace(pos, errorCtx)
         .debugThrow<TypeError>();
 }
@@ -2635,10 +2635,8 @@ void EvalState::printStatistics()
 
 std::string ExternalValueBase::coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const
 {
-    std::stringstream printed;
-    print(printed);
     throw TypeError({
-        .msg = hintfmt("cannot coerce %1% to a string: %2%", showType(), printed.str())
+        .msg = hintfmt("cannot coerce %1% to a string", showType())
     });
 }
 
diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp
index e54ecc6d1..eb73e9a52 100644
--- a/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp
+++ b/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp
@@ -7,4 +7,4 @@ error:
              |  ^
             2|
 
-       error: cannot coerce a function to a string: <LAMBDA>
+       error: cannot coerce a function to a string
diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp
index 6f0a96f78..ac14f329b 100644
--- a/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp
+++ b/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp
@@ -7,4 +7,4 @@ error:
              |   ^
             2|
 
-       error: cannot coerce a function to a string: <LAMBDA>
+       error: cannot coerce a function to a string
diff --git a/tests/unit/libexpr/error_traces.cc b/tests/unit/libexpr/error_traces.cc
index c2403bee9..81498f65a 100644
--- a/tests/unit/libexpr/error_traces.cc
+++ b/tests/unit/libexpr/error_traces.cc
@@ -295,7 +295,7 @@ namespace nix {
     TEST_F(ErrorTraceTest, toPath) {
         ASSERT_TRACE2("toPath []",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"),
+                      hintfmt("cannot coerce %s to a string", "a list"),
                       hintfmt("while evaluating the first argument passed to builtins.toPath"));
 
         ASSERT_TRACE2("toPath \"foo\"",
@@ -309,7 +309,7 @@ namespace nix {
     TEST_F(ErrorTraceTest, storePath) {
         ASSERT_TRACE2("storePath true",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string: %s", "a Boolean", "true"),
+                      hintfmt("cannot coerce %s to a string", "a Boolean"),
                       hintfmt("while evaluating the first argument passed to 'builtins.storePath'"));
 
     }
@@ -318,7 +318,7 @@ namespace nix {
     TEST_F(ErrorTraceTest, pathExists) {
         ASSERT_TRACE2("pathExists []",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"),
+                      hintfmt("cannot coerce %s to a string", "a list"),
                       hintfmt("while realising the context of a path"));
 
         ASSERT_TRACE2("pathExists \"zorglub\"",
@@ -332,7 +332,7 @@ namespace nix {
     TEST_F(ErrorTraceTest, baseNameOf) {
         ASSERT_TRACE2("baseNameOf []",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"),
+                      hintfmt("cannot coerce %s to a string", "a list"),
                       hintfmt("while evaluating the first argument passed to builtins.baseNameOf"));
 
     }
@@ -377,7 +377,7 @@ namespace nix {
     TEST_F(ErrorTraceTest, filterSource) {
         ASSERT_TRACE2("filterSource [] []",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"),
+                      hintfmt("cannot coerce %s to a string", "a list"),
                       hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"));
 
         ASSERT_TRACE2("filterSource [] \"foo\"",
@@ -1038,7 +1038,7 @@ namespace nix {
     TEST_F(ErrorTraceTest, toString) {
         ASSERT_TRACE2("toString { a = 1; }",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string: %s", "a set", "{ a = 1; }"),
+                      hintfmt("cannot coerce %s to a string", "a set"),
                       hintfmt("while evaluating the first argument passed to builtins.toString"));
 
     }
@@ -1057,7 +1057,7 @@ namespace nix {
 
         ASSERT_TRACE2("substring 0 3 {}",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"),
+                      hintfmt("cannot coerce %s to a string", "a set"),
                       hintfmt("while evaluating the third argument (the string) passed to builtins.substring"));
 
         ASSERT_TRACE1("substring (-3) 3 \"sometext\"",
@@ -1070,7 +1070,7 @@ namespace nix {
     TEST_F(ErrorTraceTest, stringLength) {
         ASSERT_TRACE2("stringLength {} # TODO: context is missing ???",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"),
+                      hintfmt("cannot coerce %s to a string", "a set"),
                       hintfmt("while evaluating the argument passed to builtins.stringLength"));
 
     }
@@ -1143,7 +1143,7 @@ namespace nix {
 
         ASSERT_TRACE2("concatStringsSep \"foo\" [ 1 2 {} ] # TODO: coerce to string is buggy",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string: %s", "an integer", "1"),
+                      hintfmt("cannot coerce %s to a string", "an integer"),
                       hintfmt("while evaluating one element of the list of strings to concat passed to builtins.concatStringsSep"));
 
     }
@@ -1229,12 +1229,12 @@ namespace nix {
 
         ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = {}; }",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"),
+                      hintfmt("cannot coerce %s to a string", "a set"),
                       hintfmt("while evaluating the attribute 'system' of derivation 'foo'"));
 
         ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = {}; }",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"),
+                      hintfmt("cannot coerce %s to a string", "a set"),
                       hintfmt("while evaluating the attribute 'outputs' of derivation 'foo'"));
 
         ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"drv\"; }",
@@ -1279,17 +1279,17 @@ namespace nix {
 
         ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ {} ]; }",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"),
+                      hintfmt("cannot coerce %s to a string", "a set"),
                       hintfmt("while evaluating an element of the argument list"));
 
         ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ \"a\" {} ]; }",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"),
+                      hintfmt("cannot coerce %s to a string", "a set"),
                       hintfmt("while evaluating an element of the argument list"));
 
         ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; FOO = {}; }",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"),
+                      hintfmt("cannot coerce %s to a string", "a set"),
                       hintfmt("while evaluating the attribute 'FOO' of derivation 'foo'"));
 
     }

From b9980b377ede0aca542b2baeeef9e4538dec20db Mon Sep 17 00:00:00 2001
From: Robert Hensing <roberth@users.noreply.github.com>
Date: Sat, 9 Dec 2023 02:36:33 +0100
Subject: [PATCH 283/294] Update rl-next/source-positions-in-errors for Nix
 2.19+

---
 doc/manual/rl-next/source-positions-in-errors.md | 2 --
 1 file changed, 2 deletions(-)

diff --git a/doc/manual/rl-next/source-positions-in-errors.md b/doc/manual/rl-next/source-positions-in-errors.md
index 00f0b27e8..15df884ea 100644
--- a/doc/manual/rl-next/source-positions-in-errors.md
+++ b/doc/manual/rl-next/source-positions-in-errors.md
@@ -21,8 +21,6 @@ it:
 error:
        … while evaluating an attribute name
 
-         at «none»:0: (source not available)
-
        error: value is a set while a string was expected
 ```
 

From 6e8d5983143ae576e3f4b1d2954a5267f2943a49 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sat, 9 Dec 2023 02:17:36 +0100
Subject: [PATCH 284/294] tests/lang/eval-fail-bad-string-interpolation-4: init

---
 .../lang/eval-fail-bad-string-interpolation-4.err.exp | 11 +++++++++++
 .../lang/eval-fail-bad-string-interpolation-4.nix     |  9 +++++++++
 2 files changed, 20 insertions(+)
 create mode 100644 tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp
 create mode 100644 tests/functional/lang/eval-fail-bad-string-interpolation-4.nix

diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp
new file mode 100644
index 000000000..07843a480
--- /dev/null
+++ b/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp
@@ -0,0 +1,11 @@
+error:
+       … while evaluating a path segment
+
+         at /pwd/lang/eval-fail-bad-string-interpolation-4.nix:9:3:
+
+            8| # The error message should not be too long.
+            9| ''${pkgs}''
+             |   ^
+           10|
+
+       error: cannot coerce a set to a string
diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-4.nix b/tests/functional/lang/eval-fail-bad-string-interpolation-4.nix
new file mode 100644
index 000000000..457b5f06a
--- /dev/null
+++ b/tests/functional/lang/eval-fail-bad-string-interpolation-4.nix
@@ -0,0 +1,9 @@
+let
+  # Basically a "billion laughs" attack, but toned down to simulated `pkgs`.
+  ha = x: y: { a = x y; b = x y; c = x y; d = x y; e = x y; f = x y; g = x y; h = x y; j = x y; };
+  has = ha (ha (ha (ha (x: x)))) "ha";
+  # A large structure that has already been evaluated.
+  pkgs = builtins.deepSeq has has;
+in
+# The error message should not be too long.
+''${pkgs}''

From 5417990e313272a5f1129ac39228b111e8dac857 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 8 Dec 2023 14:32:22 -0500
Subject: [PATCH 285/294] Create `ServeProto::BuildOptions` and a serializer
 for it

More tests, and more serializers for Hydra reuse.
---
 src/libstore/legacy-ssh-store.cc              |  22 +++-----
 src/libstore/serve-protocol.cc                |  36 +++++++++++++
 src/libstore/serve-protocol.hh                |  25 +++++++++
 src/nix-store/nix-store.cc                    |  34 ++++++++-----
 .../data/serve-protocol/build-options-2.1.bin | Bin 0 -> 16 bytes
 .../data/serve-protocol/build-options-2.2.bin | Bin 0 -> 24 bytes
 .../data/serve-protocol/build-options-2.3.bin | Bin 0 -> 40 bytes
 .../data/serve-protocol/build-options-2.7.bin | Bin 0 -> 48 bytes
 tests/unit/libstore/serve-protocol.cc         |  48 ++++++++++++++++++
 9 files changed, 137 insertions(+), 28 deletions(-)
 create mode 100644 tests/unit/libstore/data/serve-protocol/build-options-2.1.bin
 create mode 100644 tests/unit/libstore/data/serve-protocol/build-options-2.2.bin
 create mode 100644 tests/unit/libstore/data/serve-protocol/build-options-2.3.bin
 create mode 100644 tests/unit/libstore/data/serve-protocol/build-options-2.7.bin

diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 277445ee6..8ef2daa7b 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -275,20 +275,14 @@ private:
 
     void putBuildSettings(Connection & conn)
     {
-        conn.to
-            << settings.maxSilentTime
-            << settings.buildTimeout;
-        if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 2)
-            conn.to
-                << settings.maxLogSize;
-        if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3)
-            conn.to
-                << 0 // buildRepeat hasn't worked for ages anyway
-                << 0;
-
-        if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 7) {
-            conn.to << ((int) settings.keepFailed);
-        }
+        ServeProto::write(*this, conn, ServeProto::BuildOptions {
+            .maxSilentTime = settings.maxSilentTime,
+            .buildTimeout = settings.buildTimeout,
+            .maxLogSize = settings.maxLogSize,
+            .nrRepeats = 0, // buildRepeat hasn't worked for ages anyway
+            .enforceDeterminism = 0,
+            .keepFailed = settings.keepFailed,
+        });
     }
 
 public:
diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc
index c37b3095c..08bfad9e4 100644
--- a/src/libstore/serve-protocol.cc
+++ b/src/libstore/serve-protocol.cc
@@ -98,4 +98,40 @@ void ServeProto::Serialise<UnkeyedValidPathInfo>::write(const StoreDirConfig & s
             << info.sigs;
 }
 
+
+ServeProto::BuildOptions ServeProto::Serialise<ServeProto::BuildOptions>::read(const StoreDirConfig & store, ReadConn conn)
+{
+    BuildOptions options;
+    options.maxSilentTime = readInt(conn.from);
+    options.buildTimeout = readInt(conn.from);
+    if (GET_PROTOCOL_MINOR(conn.version) >= 2)
+        options.maxLogSize = readNum<unsigned long>(conn.from);
+    if (GET_PROTOCOL_MINOR(conn.version) >= 3) {
+        options.nrRepeats = readInt(conn.from);
+        options.enforceDeterminism = readInt(conn.from);
+    }
+    if (GET_PROTOCOL_MINOR(conn.version) >= 7) {
+        options.keepFailed = (bool) readInt(conn.from);
+    }
+    return options;
+}
+
+void ServeProto::Serialise<ServeProto::BuildOptions>::write(const StoreDirConfig & store, WriteConn conn, const ServeProto::BuildOptions & options)
+{
+    conn.to
+        << options.maxSilentTime
+        << options.buildTimeout;
+    if (GET_PROTOCOL_MINOR(conn.version) >= 2)
+        conn.to
+            << options.maxLogSize;
+    if (GET_PROTOCOL_MINOR(conn.version) >= 3)
+        conn.to
+            << options.nrRepeats
+            << options.enforceDeterminism;
+
+    if (GET_PROTOCOL_MINOR(conn.version) >= 7) {
+        conn.to << ((int) options.keepFailed);
+    }
+}
+
 }
diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh
index ada67a149..1665b935f 100644
--- a/src/libstore/serve-protocol.hh
+++ b/src/libstore/serve-protocol.hh
@@ -87,6 +87,13 @@ struct ServeProto
     {
         ServeProto::Serialise<T>::write(store, conn, t);
     }
+
+    /**
+     * Options for building shared between
+     * `ServeProto::Command::BuildPaths` and
+     * `ServeProto::Command::BuildDerivation`.
+     */
+    struct BuildOptions;
 };
 
 enum struct ServeProto::Command : uint64_t
@@ -102,6 +109,22 @@ enum struct ServeProto::Command : uint64_t
     AddToStoreNar = 9,
 };
 
+
+struct ServeProto::BuildOptions {
+    /**
+     * Default value in this and every other field is so tests pass when
+     * testing older deserialisers which do not set all the fields.
+     */
+    time_t maxSilentTime = -1;
+    time_t buildTimeout = -1;
+    size_t maxLogSize = -1;
+    size_t nrRepeats = -1;
+    bool enforceDeterminism = -1;
+    bool keepFailed = -1;
+
+    bool operator == (const ServeProto::BuildOptions &) const = default;
+};
+
 /**
  * Convenience for sending operation codes.
  *
@@ -144,6 +167,8 @@ template<>
 DECLARE_SERVE_SERIALISER(BuildResult);
 template<>
 DECLARE_SERVE_SERIALISER(UnkeyedValidPathInfo);
+template<>
+DECLARE_SERVE_SERIALISER(ServeProto::BuildOptions);
 
 template<typename T>
 DECLARE_SERVE_SERIALISER(std::vector<T>);
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 45af7879c..d361dc0ac 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -835,27 +835,33 @@ static void opServe(Strings opFlags, Strings opArgs)
         verbosity = lvlError;
         settings.keepLog = false;
         settings.useSubstitutes = false;
-        settings.maxSilentTime = readInt(in);
-        settings.buildTimeout = readInt(in);
+
+        auto options = ServeProto::Serialise<ServeProto::BuildOptions>::read(*store, rconn);
+
+        // Only certain feilds get initialized based on the protocol
+        // version. This is why not all the code below is unconditional.
+        // See how the serialization logic in
+        // `ServeProto::Serialise<ServeProto::BuildOptions>` matches
+        // these conditions.
+        settings.maxSilentTime = options.maxSilentTime;
+        settings.buildTimeout = options.buildTimeout;
         if (GET_PROTOCOL_MINOR(clientVersion) >= 2)
-            settings.maxLogSize = readNum<unsigned long>(in);
+            settings.maxLogSize = options.maxLogSize;
         if (GET_PROTOCOL_MINOR(clientVersion) >= 3) {
-            auto nrRepeats = readInt(in);
-            if (nrRepeats != 0) {
+            if (options.nrRepeats != 0) {
                 throw Error("client requested repeating builds, but this is not currently implemented");
             }
-            // Ignore 'enforceDeterminism'. It used to be true by
-            // default, but also only never had any effect when
-            // `nrRepeats == 0`.  We have already asserted that
-            // `nrRepeats` in fact is 0, so we can safely ignore this
-            // without doing something other than what the client
-            // asked for.
-            readInt(in);
-
+            // Ignore 'options.enforceDeterminism'.
+            //
+            // It used to be true by default, but also only never had
+            // any effect when `nrRepeats == 0`.  We have already
+            // checked that `nrRepeats` in fact is 0, so we can safely
+            // ignore this without doing something other than what the
+            // client asked for.
             settings.runDiffHook = true;
         }
         if (GET_PROTOCOL_MINOR(clientVersion) >= 7) {
-            settings.keepFailed = (bool) readInt(in);
+            settings.keepFailed = options.keepFailed;
         }
     };
 
diff --git a/tests/unit/libstore/data/serve-protocol/build-options-2.1.bin b/tests/unit/libstore/data/serve-protocol/build-options-2.1.bin
new file mode 100644
index 0000000000000000000000000000000000000000..61e1d97286139e43918505b1b953128360d27853
GIT binary patch
literal 16
NcmZQ&fB-fq4FCX;01N;C

literal 0
HcmV?d00001

diff --git a/tests/unit/libstore/data/serve-protocol/build-options-2.2.bin b/tests/unit/libstore/data/serve-protocol/build-options-2.2.bin
new file mode 100644
index 0000000000000000000000000000000000000000..045c2ff2b54ba708bc1d411f0e8786207c4e660a
GIT binary patch
literal 24
PcmZQ&fB-fq%?_mj0Vn_y

literal 0
HcmV?d00001

diff --git a/tests/unit/libstore/data/serve-protocol/build-options-2.3.bin b/tests/unit/libstore/data/serve-protocol/build-options-2.3.bin
new file mode 100644
index 0000000000000000000000000000000000000000..5c53458831dca70d5303363919f46f20f88993a2
GIT binary patch
literal 40
VcmZQ&fB-fq%?_nGpfn?t1^@!!02}}S

literal 0
HcmV?d00001

diff --git a/tests/unit/libstore/data/serve-protocol/build-options-2.7.bin b/tests/unit/libstore/data/serve-protocol/build-options-2.7.bin
new file mode 100644
index 0000000000000000000000000000000000000000..1bc7b02db38f5f751c2610de84ff937e630567c9
GIT binary patch
literal 48
WcmZQ&fB-fq%?_nGpfrqPgfajFxBwgg

literal 0
HcmV?d00001

diff --git a/tests/unit/libstore/serve-protocol.cc b/tests/unit/libstore/serve-protocol.cc
index c2298c6db..8f256d1e6 100644
--- a/tests/unit/libstore/serve-protocol.cc
+++ b/tests/unit/libstore/serve-protocol.cc
@@ -302,6 +302,54 @@ VERSIONED_CHARACTERIZATION_TEST(
         }),
     }))
 
+VERSIONED_CHARACTERIZATION_TEST(
+    ServeProtoTest,
+    build_options_2_1,
+    "build-options-2.1",
+    2 << 8 | 1,
+    (ServeProto::BuildOptions {
+        .maxSilentTime = 5,
+        .buildTimeout = 6,
+    }))
+
+VERSIONED_CHARACTERIZATION_TEST(
+    ServeProtoTest,
+    build_options_2_2,
+    "build-options-2.2",
+    2 << 8 | 2,
+    (ServeProto::BuildOptions {
+        .maxSilentTime = 5,
+        .buildTimeout = 6,
+        .maxLogSize = 7,
+    }))
+
+VERSIONED_CHARACTERIZATION_TEST(
+    ServeProtoTest,
+    build_options_2_3,
+    "build-options-2.3",
+    2 << 8 | 3,
+    (ServeProto::BuildOptions {
+        .maxSilentTime = 5,
+        .buildTimeout = 6,
+        .maxLogSize = 7,
+        .nrRepeats = 8,
+        .enforceDeterminism = true,
+    }))
+
+VERSIONED_CHARACTERIZATION_TEST(
+    ServeProtoTest,
+    build_options_2_7,
+    "build-options-2.7",
+    2 << 8 | 7,
+    (ServeProto::BuildOptions {
+        .maxSilentTime = 5,
+        .buildTimeout = 6,
+        .maxLogSize = 7,
+        .nrRepeats = 8,
+        .enforceDeterminism = false,
+        .keepFailed = true,
+    }))
+
 VERSIONED_CHARACTERIZATION_TEST(
     ServeProtoTest,
     vector,

From 360f3b3a9e0a74eb8b7d5a1744ad58f4cd487ca0 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sat, 9 Dec 2023 19:50:33 +0100
Subject: [PATCH 286/294] changelog-d: Use roberth fork with markdown
 frontmatter support

---
 flake.nix                  |  2 +-
 misc/changelog-d.cabal.nix | 31 +++++++++++++++++++++++++++++++
 misc/changelog-d.nix       | 31 +++++++++++++++++++++++++++++++
 3 files changed, 63 insertions(+), 1 deletion(-)
 create mode 100644 misc/changelog-d.cabal.nix
 create mode 100644 misc/changelog-d.nix

diff --git a/flake.nix b/flake.nix
index dbd45f053..90203e7d5 100644
--- a/flake.nix
+++ b/flake.nix
@@ -182,7 +182,7 @@
           "--enable-internal-api-docs"
         ];
 
-        changelog-d = pkgs.buildPackages.changelog-d;
+        changelog-d = pkgs.buildPackages.callPackage ./misc/changelog-d.nix { };
 
         nativeBuildDeps =
           [
diff --git a/misc/changelog-d.cabal.nix b/misc/changelog-d.cabal.nix
new file mode 100644
index 000000000..76f9353cd
--- /dev/null
+++ b/misc/changelog-d.cabal.nix
@@ -0,0 +1,31 @@
+{ mkDerivation, aeson, base, bytestring, cabal-install-parsers
+, Cabal-syntax, containers, directory, filepath, frontmatter
+, generic-lens-lite, lib, mtl, optparse-applicative, parsec, pretty
+, regex-applicative, text, pkgs
+}:
+let rev = "f30f6969e9cd8b56242309639d58acea21c99d06";
+in
+mkDerivation {
+  pname = "changelog-d";
+  version = "0.1";
+  src = pkgs.fetchurl {
+    name = "changelog-d-${rev}.tar.gz";
+    url = "https://codeberg.org/roberth/changelog-d/archive/${rev}.tar.gz";
+    hash = "sha256-8a2+i5u7YoszAgd5OIEW0eYUcP8yfhtoOIhLJkylYJ4=";
+  } // { inherit rev; };
+  isLibrary = false;
+  isExecutable = true;
+  libraryHaskellDepends = [
+    aeson base bytestring cabal-install-parsers Cabal-syntax containers
+    directory filepath frontmatter generic-lens-lite mtl parsec pretty
+    regex-applicative text
+  ];
+  executableHaskellDepends = [
+    base bytestring Cabal-syntax directory filepath
+    optparse-applicative
+  ];
+  doHaddock = false;
+  description = "Concatenate changelog entries into a single one";
+  license = lib.licenses.gpl3Plus;
+  mainProgram = "changelog-d";
+}
diff --git a/misc/changelog-d.nix b/misc/changelog-d.nix
new file mode 100644
index 000000000..1b20f4596
--- /dev/null
+++ b/misc/changelog-d.nix
@@ -0,0 +1,31 @@
+# Taken temporarily from <nixpkgs/pkgs/by-name/ch/changelog-d/package.nix>
+{
+  callPackage,
+  lib,
+  haskell,
+  haskellPackages,
+}:
+
+let
+  hsPkg = haskellPackages.callPackage ./changelog-d.cabal.nix { };
+
+  addCompletions = haskellPackages.generateOptparseApplicativeCompletions ["changelog-d"];
+
+  haskellModifications =
+    lib.flip lib.pipe [
+      addCompletions
+      haskell.lib.justStaticExecutables
+    ];
+
+  mkDerivationOverrides = finalAttrs: oldAttrs: {
+
+    version = oldAttrs.version + "-git-${lib.strings.substring 0 7 oldAttrs.src.rev}";
+
+    meta = oldAttrs.meta // {
+      homepage = "https://codeberg.org/roberth/changelog-d";
+      maintainers = [ lib.maintainers.roberth ];
+    };
+
+  };
+in
+  (haskellModifications hsPkg).overrideAttrs mkDerivationOverrides

From 3811b334c646bc3b4bf8caef6d13c9f5027246f1 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sat, 9 Dec 2023 19:51:20 +0100
Subject: [PATCH 287/294] rl-next: Use markdown frontmatter syntax

The old syntax is still supported, as long as you don't use a {
in the description - the reason to migrate.
---
 doc/manual/rl-next/hash-format-nix32.md          |  5 +++--
 doc/manual/rl-next/mounted-ssh-store.md          |  9 ++++-----
 doc/manual/rl-next/nix-config-show.md            | 11 +++++------
 doc/manual/rl-next/nix-env-json-drv-path.md      |  9 +++------
 doc/manual/rl-next/nix-hash-convert.md           |  6 +++---
 doc/manual/rl-next/source-positions-in-errors.md |  9 ++++-----
 doc/manual/src/contributing/hacking.md           |  9 ++++-----
 7 files changed, 26 insertions(+), 32 deletions(-)

diff --git a/doc/manual/rl-next/hash-format-nix32.md b/doc/manual/rl-next/hash-format-nix32.md
index 20c557da9..73e6fbb24 100644
--- a/doc/manual/rl-next/hash-format-nix32.md
+++ b/doc/manual/rl-next/hash-format-nix32.md
@@ -1,6 +1,7 @@
+---
 synopsis: Rename hash format `base32` to `nix32`
-prs: #9452
-description: {
+prs: 9452
+---
 
 Hash format `base32` was renamed to `nix32` since it used a special nix-specific character set for 
 [Base32](https://en.wikipedia.org/wiki/Base32).
diff --git a/doc/manual/rl-next/mounted-ssh-store.md b/doc/manual/rl-next/mounted-ssh-store.md
index 39fac5283..6df44dbb6 100644
--- a/doc/manual/rl-next/mounted-ssh-store.md
+++ b/doc/manual/rl-next/mounted-ssh-store.md
@@ -1,9 +1,8 @@
+---
 synopsis: Mounted SSH Store
-issues: #7890
-prs: #7912
-description: {
+issues: 7890
+prs: 7912
+---
 
 Introduced the store [`mounted-ssh-ng://`](@docroot@/command-ref/new-cli/nix3-help-stores.md).
 This store allows full access to a Nix store on a remote machine and additionally requires that the store be mounted in the local filesystem.
-
-}
diff --git a/doc/manual/rl-next/nix-config-show.md b/doc/manual/rl-next/nix-config-show.md
index b2ad3c666..26b961b76 100644
--- a/doc/manual/rl-next/nix-config-show.md
+++ b/doc/manual/rl-next/nix-config-show.md
@@ -1,8 +1,7 @@
-synopsis: `nix config show`
-issues: #7672
-prs: #9477
-description: {
+---
+synopsis: Rename to `nix config show`
+issues: 7672
+prs: 9477
+---
 
 `nix show-config` was renamed to `nix config show`, and `nix doctor` was renamed to `nix config check`, to be more consistent with the rest of the command-line interface.
-
-}
diff --git a/doc/manual/rl-next/nix-env-json-drv-path.md b/doc/manual/rl-next/nix-env-json-drv-path.md
index fbe2b67d8..734cefd1b 100644
--- a/doc/manual/rl-next/nix-env-json-drv-path.md
+++ b/doc/manual/rl-next/nix-env-json-drv-path.md
@@ -1,9 +1,6 @@
+---
 synopsis: Fix `nix-env --query --drv-path --json`
-prs: #9257
-description: {
+prs: 9257
+---
 
 Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set.
-
-}
-
-
diff --git a/doc/manual/rl-next/nix-hash-convert.md b/doc/manual/rl-next/nix-hash-convert.md
index de4367c5b..2b718a66b 100644
--- a/doc/manual/rl-next/nix-hash-convert.md
+++ b/doc/manual/rl-next/nix-hash-convert.md
@@ -1,6 +1,7 @@
+---
 synopsis: Add `nix hash convert`
-prs: #9452
-description: {
+prs: 9452
+---
 
 New [`nix hash convert`](https://github.com/NixOS/nix/issues/8876) sub command with a fast track
 to stabilization! Examples:
@@ -44,4 +45,3 @@ The following commands are still available but will emit a deprecation warning.
 - `nix hash to-base64 $hash1 $hash2`: Use `nix hash convert --to base64 $hash1 $hash2` instead.
 - `nix hash to-sri $hash1 $hash2`: : Use `nix hash convert --to sri $hash1 $hash2`
   or even just `nix hash convert $hash1 $hash2` instead.
-}
diff --git a/doc/manual/rl-next/source-positions-in-errors.md b/doc/manual/rl-next/source-positions-in-errors.md
index 15df884ea..5b210289d 100644
--- a/doc/manual/rl-next/source-positions-in-errors.md
+++ b/doc/manual/rl-next/source-positions-in-errors.md
@@ -1,7 +1,8 @@
+---
 synopsis: Source locations are printed more consistently in errors
-issues: #561
-prs: #9555
-description: {
+issues: 561
+prs: 9555
+---
 
 Source location information is now included in error messages more
 consistently. Given this code:
@@ -39,5 +40,3 @@ error:
 
        error: value is a set while a string was expected
 ```
-
-}
diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md
index 9de5ad39b..237eff925 100644
--- a/doc/manual/src/contributing/hacking.md
+++ b/doc/manual/src/contributing/hacking.md
@@ -257,17 +257,16 @@ User-visible changes should come with a release note.
 Here's what a complete entry looks like. The file name is not incorporated in the document.
 
 ```
+---
 synopsis: Basically a title
-issues: #1234
-prs: #1238
-description: {
+issues: 1234
+prs: 1238
+---
 
 Here's one or more paragraphs that describe the change.
 
 - It's markdown
 - Add references to the manual using @docroot@
-
-}
 ```
 
 Significant changes should add the following header, which moves them to the top.

From a856f603ed5a124f7eb818dadab6c88da73570fb Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sat, 9 Dec 2023 19:55:47 +0100
Subject: [PATCH 288/294] Add checks.rl-next

---
 flake.nix | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/flake.nix b/flake.nix
index 90203e7d5..f499b0a9b 100644
--- a/flake.nix
+++ b/flake.nix
@@ -691,6 +691,11 @@
         perlBindings = self.hydraJobs.perlBindings.${system};
         installTests = self.hydraJobs.installTests.${system};
         nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system};
+        rl-next =
+          let pkgs = nixpkgsFor.${system}.native;
+          in pkgs.buildPackages.runCommand "test-rl-next-release-notes" { } ''
+          LANG=C.UTF-8 ${(commonDeps { inherit pkgs; }).changelog-d}/bin/changelog-d ${./doc/manual/rl-next} >$out
+        '';
       } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) {
         dockerImage = self.hydraJobs.dockerImage.${system};
       });

From a63be6578f7e17182fdec8e3d3fdbab19a814152 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sat, 9 Dec 2023 21:22:20 +0100
Subject: [PATCH 289/294] flake.nix: Cache shell inputs through hydra

---
 flake.nix | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/flake.nix b/flake.nix
index f499b0a9b..99480183a 100644
--- a/flake.nix
+++ b/flake.nix
@@ -540,6 +540,8 @@
         # Binary package for various platforms.
         build = forAllSystems (system: self.packages.${system}.nix);
 
+        shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation);
+
         buildStatic = lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static);
 
         buildCross = forAllCrossSystems (crossSystem:

From 3c200da242d8f0ccda447866028bb757e0b0bbd9 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Sun, 10 Dec 2023 06:16:32 +0100
Subject: [PATCH 290/294] document `fetchTree` (#9258)

* document `fetchTree`

* display experimental feature note at the top

we have to enable the new `fetchTree` experimental feature to render it
at all. this was a bug introduced when adding that new feature flag.

Co-authored-by: tomberek <tomberek@users.noreply.github.com>
Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
Co-authored-by: Silvan Mosberger <github@infinisil.com>
---
 doc/manual/generate-builtins.nix   |  13 +-
 doc/manual/generate-settings.nix   |   4 +-
 doc/manual/generate-store-info.nix |   4 +-
 src/libexpr/primops/fetchTree.cc   | 242 ++++++++++++++++++++++++-----
 src/nix/main.cc                    |   1 +
 5 files changed, 221 insertions(+), 43 deletions(-)

diff --git a/doc/manual/generate-builtins.nix b/doc/manual/generate-builtins.nix
index 05cae1c46..007b698f1 100644
--- a/doc/manual/generate-builtins.nix
+++ b/doc/manual/generate-builtins.nix
@@ -8,7 +8,15 @@ let
   showBuiltin = name: { doc, args, arity, experimental-feature }:
     let
       experimentalNotice = optionalString (experimental-feature != null) ''
-        This function is only available if the [${experimental-feature}](@docroot@/contributing/experimental-features.md#xp-feature-${experimental-feature}) experimental feature is enabled.
+        > **Note**
+        >
+        > This function is only available if the [`${experimental-feature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimental-feature}) is enabled.
+        >
+        > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md):
+        >
+        > ```
+        > extra-experimental-features = ${experimental-feature}
+        > ```
       '';
     in
     squash ''
@@ -17,10 +25,9 @@ let
       </dt>
       <dd>
 
-      ${doc}
-
       ${experimentalNotice}
 
+      ${doc}
       </dd>
     '';
   listArgs = args: concatStringsSep " " (map (s: "<var>${s}</var>") args);
diff --git a/doc/manual/generate-settings.nix b/doc/manual/generate-settings.nix
index 74446b70b..504cda362 100644
--- a/doc/manual/generate-settings.nix
+++ b/doc/manual/generate-settings.nix
@@ -20,10 +20,10 @@ let
         else "`${setting}`";
       # separate body to cleanly handle indentation
       body = ''
-          ${description}
-
           ${experimentalFeatureNote}
 
+          ${description}
+
           **Default:** ${showDefault documentDefault defaultValue}
 
           ${showAliases aliases}
diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix
index 57247a181..c311c3c39 100644
--- a/doc/manual/generate-store-info.nix
+++ b/doc/manual/generate-store-info.nix
@@ -19,10 +19,10 @@ let
       result = squash ''
         # ${name}
 
-        ${doc}
-
         ${experimentalFeatureNote}
 
+        ${doc}
+
         ## Settings
 
         ${showSettings { prefix = "store-${slug}"; inherit inlineHTML; } settings}
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 15f870a95..eb2df8626 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -187,45 +187,215 @@ static RegisterPrimOp primop_fetchTree({
     .name = "fetchTree",
     .args = {"input"},
     .doc = R"(
-      Fetch a source tree or a plain file using one of the supported backends.
-      *input* must be a [flake reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references), either in attribute set representation or in the URL-like syntax.
-      The input should be "locked", that is, it should contain a commit hash or content hash unless impure evaluation (`--impure`) is enabled.
+      Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with:
 
-      > **Note**
+      - the resulting fixed-output [store path](@docroot@/glossary.md#gloss-store-path)
+      - the corresponding [NAR](@docroot@/glossary.md#gloss-nar) hash
+      - backend-specific metadata (currently not documented). <!-- TODO: document output attributes -->
+
+      *input* must be an attribute set with the following attributes:
+
+      - `type` (String, required)
+
+        One of the [supported source types](#source-types).
+        This determines other required and allowed input attributes.
+
+      - `narHash` (String, optional)
+
+        The `narHash` parameter can be used to substitute the source of the tree.
+        It also allows for verification of tree contents that may not be provided by the underlying transfer mechanism.
+        If `narHash` is set, the source is first looked up is the Nix store and [substituters](@docroot@/command-ref/conf-file.md#conf-substituters), and only fetched if not available.
+
+      A subset of the output attributes of `fetchTree` can be re-used for subsequent calls to `fetchTree` to produce the same result again.
+      That is, `fetchTree` is idempotent.
+
+      Downloads are cached in `$XDG_CACHE_HOME/nix`.
+      The remote source will be fetched from the network if both are true:
+      - A NAR hash is supplied and the corresponding store path is not [valid](@docroot@/glossary.md#gloss-validity), that is, not available in the store
+
+        > **Note**
+        >
+        > [Substituters](@docroot@/command-ref/conf-file.md#conf-substituters) are not used in fetching.
+
+      - There is no cache entry or the cache entry is older than [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl)
+
+      ## Source types
+
+      The following source types and associated input attributes are supported.
+
+      <!-- TODO: It would be soooo much more predictable to work with (and
+      document) if `fetchTree` was a curried call with the first paramter for
+      `type` or an attribute like `builtins.fetchTree.git`! -->
+
+      - `"file"`
+
+        Place a plain file into the Nix store.
+        This is similar to [`builtins.fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl)
+
+        - `url` (String, required)
+
+          Supported protocols:
+
+          - `https`
+
+            > **Example**
+            >
+            > ```nix
+            > fetchTree {
+            >   type = "file";
+            >   url = "https://example.com/index.html";
+            > }
+            > ```
+
+          - `http`
+
+            Insecure HTTP transfer for legacy sources.
+
+            > **Warning**
+            >
+            > HTTP performs no encryption or authentication.
+            > Use a `narHash` known in advance to ensure the output has expected contents.
+
+          - `file`
+
+            A file on the local file system.
+
+            > **Example**
+            >
+            > ```nix
+            > fetchTree {
+            >   type = "file";
+            >   url = "file:///home/eelco/nix/README.md";
+            > }
+            > ```
+
+      - `"tarball"`
+
+        Download a tar archive and extract it into the Nix store.
+        This has the same underyling implementation as [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball)
+
+        - `url` (String, required)
+
+           > **Example**
+           >
+           > ```nix
+           > fetchTree {
+           >   type = "tarball";
+           >   url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11";
+           > }
+           > ```
+
+      - `"git"`
+
+        Fetch a Git tree and copy it to the Nix store.
+        This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit).
+
+        - `url` (String, required)
+
+          The URL formats supported are the same as for Git itself.
+
+          > **Example**
+          >
+          > ```nix
+          > fetchTree {
+          >   type = "git";
+          >   url = "git@github.com:NixOS/nixpkgs.git";
+          > }
+          > ```
+
+          > **Note**
+          >
+          > If the URL points to a local directory, and no `ref` or `rev` is given, Nix will only consider files added to the Git index, as listed by `git ls-files` but use the *current file contents* of the Git working directory.
+
+        - `ref` (String, optional)
+
+          A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name.
+
+          Default: `"HEAD"`
+
+        - `rev` (String, optional)
+
+          A Git revision; a commit hash.
+
+          Default: the tip of `ref`
+
+        - `shallow` (Bool, optional)
+
+          Make a shallow clone when fetching the Git tree.
+
+          Default: `false`
+
+        - `submodules` (Bool, optional)
+
+          Also fetch submodules if available.
+
+          Default: `false`
+
+        - `allRefs` (Bool, optional)
+
+          If set to `true`, always fetch the entire repository, even if the latest commit is still in the cache.
+          Otherwise, only the latest commit is fetched if it is not already cached.
+
+          Default: `false`
+
+        - `lastModified` (Integer, optional)
+
+          Unix timestamp of the fetched commit.
+
+          If set, pass through the value to the output attribute set.
+          Otherwise, generated from the fetched Git tree.
+
+        - `revCount` (Integer, optional)
+
+          Number of revisions in the history of the Git repository before the fetched commit.
+
+          If set, pass through the value to the output attribute set.
+          Otherwise, generated from the fetched Git tree.
+
+      The following input types are still subject to change:
+
+      - `"path"`
+      - `"github"`
+      - `"gitlab"`
+      - `"sourcehut"`
+      - `"mercurial"`
+
+     *input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references).
+     The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled.
+
+      > **Example**
       >
-      > The URL-like syntax requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled.
+      > Fetch a GitHub repository using the attribute set representation:
+      >
+      > ```nix
+      > builtins.fetchTree {
+      >   type = "github";
+      >   owner = "NixOS";
+      >   repo = "nixpkgs";
+      >   rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
+      > }
+      > ```
+      >
+      > This evaluates to the following attribute set:
+      >
+      > ```nix
+      > {
+      >   lastModified = 1686503798;
+      >   lastModifiedDate = "20230611171638";
+      >   narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc=";
+      >   outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source";
+      >   rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
+      >   shortRev = "ae2e6b3";
+      > }
+      > ```
 
-      Here are some examples of how to use `fetchTree`:
-
-      - Fetch a GitHub repository using the attribute set representation:
-
-          ```nix
-          builtins.fetchTree {
-            type = "github";
-            owner = "NixOS";
-            repo = "nixpkgs";
-            rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
-          }
-          ```
-
-        This evaluates to the following attribute set:
-
-          ```
-          {
-            lastModified = 1686503798;
-            lastModifiedDate = "20230611171638";
-            narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc=";
-            outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source";
-            rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
-            shortRev = "ae2e6b3";
-          }
-          ```
-
-      - Fetch the same GitHub repository using the URL-like syntax:
-
-          ```
-          builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd"
-          ```
+      > **Example**
+      >
+      > Fetch the same GitHub repository using the URL-like syntax:
+      >
+      >   ```nix
+      >   builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd"
+      >   ```
     )",
     .fun = prim_fetchTree,
     .experimentalFeature = Xp::FetchTree,
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 109d2cc04..39c04069b 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -373,6 +373,7 @@ void mainWrapped(int argc, char * * argv)
             Xp::Flakes,
             Xp::FetchClosure,
             Xp::DynamicDerivations,
+            Xp::FetchTree,
         };
         evalSettings.pureEval = false;
         EvalState state({}, openStore("dummy://"));

From deadb3bfe9cde3e78e8e89340e4c92499069461a Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 10 Dec 2023 14:28:14 -0500
Subject: [PATCH 291/294] Create header for `LegacySSHStore`

In https://github.com/NixOS/nix/pull/6134#issuecomment-1079199888,
@thuffschmitt proposed exposing `LegacySSHStore` in Nix for
deduplication with Hydra, at least temporarily. I think that is a good
idea.

Note that the diff will look bad unless one ignores whitespace! Also try
this locally:

```shell-session
git diff --ignore-all-space HEAD^:src/libstore/legacy-ssh-store.cc HEAD:src/libstore/legacy-ssh-store.cc
git diff --ignore-all-space HEAD^:src/libstore/legacy-ssh-store.cc HEAD:src/libstore/legacy-ssh-store.hh
```
---
 src/libstore/legacy-ssh-store.cc | 726 ++++++++++++++-----------------
 src/libstore/legacy-ssh-store.hh | 132 ++++++
 2 files changed, 466 insertions(+), 392 deletions(-)
 create mode 100644 src/libstore/legacy-ssh-store.hh

diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 8ef2daa7b..06bef9d08 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -1,3 +1,4 @@
+#include "legacy-ssh-store.hh"
 #include "ssh-store-config.hh"
 #include "archive.hh"
 #include "pool.hh"
@@ -13,414 +14,355 @@
 
 namespace nix {
 
-struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig
+std::string LegacySSHStoreConfig::doc()
 {
-    using CommonSSHStoreConfig::CommonSSHStoreConfig;
+    return
+      #include "legacy-ssh-store.md"
+      ;
+}
 
-    const Setting<Path> remoteProgram{this, "nix-store", "remote-program",
-        "Path to the `nix-store` executable on the remote machine."};
 
-    const Setting<int> maxConnections{this, 1, "max-connections",
-        "Maximum number of concurrent SSH connections."};
-
-    const std::string name() override { return "SSH Store"; }
-
-    std::string doc() override
-    {
-        return
-          #include "legacy-ssh-store.md"
-          ;
-    }
-};
-
-struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store
+struct LegacySSHStore::Connection
 {
-    // Hack for getting remote build log output.
-    // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in
-    // the documentation
-    const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
-
-    struct Connection
-    {
-        std::unique_ptr<SSHMaster::Connection> sshConn;
-        FdSink to;
-        FdSource from;
-        ServeProto::Version remoteVersion;
-        bool good = true;
-
-        /**
-         * Coercion to `ServeProto::ReadConn`. This makes it easy to use the
-         * factored out serve protocol searlizers with a
-         * `LegacySSHStore::Connection`.
-         *
-         * The serve protocol connection types are unidirectional, unlike
-         * this type.
-         */
-        operator ServeProto::ReadConn ()
-        {
-            return ServeProto::ReadConn {
-                .from = from,
-                .version = remoteVersion,
-            };
-        }
-
-        /*
-         * Coercion to `ServeProto::WriteConn`. This makes it easy to use the
-         * factored out serve protocol searlizers with a
-         * `LegacySSHStore::Connection`.
-         *
-         * The serve protocol connection types are unidirectional, unlike
-         * this type.
-         */
-        operator ServeProto::WriteConn ()
-        {
-            return ServeProto::WriteConn {
-                .to = to,
-                .version = remoteVersion,
-            };
-        }
-    };
-
-    std::string host;
-
-    ref<Pool<Connection>> connections;
-
-    SSHMaster master;
-
-    static std::set<std::string> uriSchemes() { return {"ssh"}; }
-
-    LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params)
-        : StoreConfig(params)
-        , CommonSSHStoreConfig(params)
-        , LegacySSHStoreConfig(params)
-        , Store(params)
-        , host(host)
-        , connections(make_ref<Pool<Connection>>(
-            std::max(1, (int) maxConnections),
-            [this]() { return openConnection(); },
-            [](const ref<Connection> & r) { return r->good; }
-            ))
-        , master(
-            host,
-            sshKey,
-            sshPublicHostKey,
-            // Use SSH master only if using more than 1 connection.
-            connections->capacity() > 1,
-            compress,
-            logFD)
-    {
-    }
-
-    ref<Connection> openConnection()
-    {
-        auto conn = make_ref<Connection>();
-        conn->sshConn = master.startCommand(
-            fmt("%s --serve --write", remoteProgram)
-            + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get())));
-        conn->to = FdSink(conn->sshConn->in.get());
-        conn->from = FdSource(conn->sshConn->out.get());
-
-        try {
-            conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION;
-            conn->to.flush();
-
-            StringSink saved;
-            try {
-                TeeSource tee(conn->from, saved);
-                unsigned int magic = readInt(tee);
-                if (magic != SERVE_MAGIC_2)
-                    throw Error("'nix-store --serve' protocol mismatch from '%s'", host);
-            } catch (SerialisationError & e) {
-                /* In case the other side is waiting for our input,
-                   close it. */
-                conn->sshConn->in.close();
-                auto msg = conn->from.drain();
-                throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'",
-                    host, chomp(saved.s + msg));
-            }
-            conn->remoteVersion = readInt(conn->from);
-            if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200)
-                throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host);
-
-        } catch (EndOfFile & e) {
-            throw Error("cannot connect to '%1%'", host);
-        }
-
-        return conn;
-    };
-
-    std::string getUri() override
-    {
-        return *uriSchemes().begin() + "://" + host;
-    }
-
-    void queryPathInfoUncached(const StorePath & path,
-        Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override
-    {
-        try {
-            auto conn(connections->get());
-
-            /* No longer support missing NAR hash */
-            assert(GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4);
-
-            debug("querying remote host '%s' for info on '%s'", host, printStorePath(path));
-
-            conn->to << ServeProto::Command::QueryPathInfos << PathSet{printStorePath(path)};
-            conn->to.flush();
-
-            auto p = readString(conn->from);
-            if (p.empty()) return callback(nullptr);
-            auto path2 = parseStorePath(p);
-            assert(path == path2);
-            auto info = std::make_shared<ValidPathInfo>(
-                path,
-                ServeProto::Serialise<UnkeyedValidPathInfo>::read(*this, *conn));
-
-            if (info->narHash == Hash::dummy)
-                throw Error("NAR hash is now mandatory");
-
-            auto s = readString(conn->from);
-            assert(s == "");
-
-            callback(std::move(info));
-        } catch (...) { callback.rethrow(); }
-    }
-
-    void addToStore(const ValidPathInfo & info, Source & source,
-        RepairFlag repair, CheckSigsFlag checkSigs) override
-    {
-        debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host);
-
-        auto conn(connections->get());
-
-        if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) {
-
-            conn->to
-                << ServeProto::Command::AddToStoreNar
-                << printStorePath(info.path)
-                << (info.deriver ? printStorePath(*info.deriver) : "")
-                << info.narHash.to_string(HashFormat::Base16, false);
-            ServeProto::write(*this, *conn, info.references);
-            conn->to
-                << info.registrationTime
-                << info.narSize
-                << info.ultimate
-                << info.sigs
-                << renderContentAddress(info.ca);
-            try {
-                copyNAR(source, conn->to);
-            } catch (...) {
-                conn->good = false;
-                throw;
-            }
-            conn->to.flush();
-
-        } else {
-
-            conn->to
-                << ServeProto::Command::ImportPaths
-                << 1;
-            try {
-                copyNAR(source, conn->to);
-            } catch (...) {
-                conn->good = false;
-                throw;
-            }
-            conn->to
-                << exportMagic
-                << printStorePath(info.path);
-            ServeProto::write(*this, *conn, info.references);
-            conn->to
-                << (info.deriver ? printStorePath(*info.deriver) : "")
-                << 0
-                << 0;
-            conn->to.flush();
-
-        }
-
-        if (readInt(conn->from) != 1)
-            throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host);
-    }
-
-    void narFromPath(const StorePath & path, Sink & sink) override
-    {
-        auto conn(connections->get());
-
-        conn->to << ServeProto::Command::DumpStorePath << printStorePath(path);
-        conn->to.flush();
-        copyNAR(conn->from, sink);
-    }
-
-    std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
-    { unsupported("queryPathFromHashPart"); }
-
-    StorePath addToStore(
-            std::string_view name,
-            const Path & srcPath,
-            FileIngestionMethod method,
-            HashAlgorithm hashAlgo,
-            PathFilter & filter,
-            RepairFlag repair,
-            const StorePathSet & references) override
-    { unsupported("addToStore"); }
-
-    StorePath addTextToStore(
-        std::string_view name,
-        std::string_view s,
-        const StorePathSet & references,
-        RepairFlag repair) override
-    { unsupported("addTextToStore"); }
-
-private:
-
-    void putBuildSettings(Connection & conn)
-    {
-        ServeProto::write(*this, conn, ServeProto::BuildOptions {
-            .maxSilentTime = settings.maxSilentTime,
-            .buildTimeout = settings.buildTimeout,
-            .maxLogSize = settings.maxLogSize,
-            .nrRepeats = 0, // buildRepeat hasn't worked for ages anyway
-            .enforceDeterminism = 0,
-            .keepFailed = settings.keepFailed,
-        });
-    }
-
-public:
-
-    BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
-        BuildMode buildMode) override
-    {
-        auto conn(connections->get());
-
-        conn->to
-            << ServeProto::Command::BuildDerivation
-            << printStorePath(drvPath);
-        writeDerivation(conn->to, *this, drv);
-
-        putBuildSettings(*conn);
-
-        conn->to.flush();
-
-        return ServeProto::Serialise<BuildResult>::read(*this, *conn);
-    }
-
-    void buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override
-    {
-        if (evalStore && evalStore.get() != this)
-            throw Error("building on an SSH store is incompatible with '--eval-store'");
-
-        auto conn(connections->get());
-
-        conn->to << ServeProto::Command::BuildPaths;
-        Strings ss;
-        for (auto & p : drvPaths) {
-            auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p);
-            std::visit(overloaded {
-                [&](const StorePathWithOutputs & s) {
-                    ss.push_back(s.to_string(*this));
-                },
-                [&](const StorePath & drvPath) {
-                    throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath));
-                },
-                [&](std::monostate) {
-                    throw Error("wanted build derivation that is itself a build product, but the legacy ssh protocol doesn't support that. Try using ssh-ng://");
-                },
-            }, sOrDrvPath);
-        }
-        conn->to << ss;
-
-        putBuildSettings(*conn);
-
-        conn->to.flush();
-
-        BuildResult result;
-        result.status = (BuildResult::Status) readInt(conn->from);
-
-        if (!result.success()) {
-            conn->from >> result.errorMsg;
-            throw Error(result.status, result.errorMsg);
-        }
-    }
-
-    void ensurePath(const StorePath & path) override
-    { unsupported("ensurePath"); }
-
-    virtual ref<SourceAccessor> getFSAccessor(bool requireValidPath) override
-    { unsupported("getFSAccessor"); }
+    std::unique_ptr<SSHMaster::Connection> sshConn;
+    FdSink to;
+    FdSource from;
+    ServeProto::Version remoteVersion;
+    bool good = true;
 
     /**
-     * The default instance would schedule the work on the client side, but
-     * for consistency with `buildPaths` and `buildDerivation` it should happen
-     * on the remote side.
+     * Coercion to `ServeProto::ReadConn`. This makes it easy to use the
+     * factored out serve protocol searlizers with a
+     * `LegacySSHStore::Connection`.
      *
-     * We make this fail for now so we can add implement this properly later
-     * without it being a breaking change.
+     * The serve protocol connection types are unidirectional, unlike
+     * this type.
      */
-    void repairPath(const StorePath & path) override
-    { unsupported("repairPath"); }
-
-    void computeFSClosure(const StorePathSet & paths,
-        StorePathSet & out, bool flipDirection = false,
-        bool includeOutputs = false, bool includeDerivers = false) override
+    operator ServeProto::ReadConn ()
     {
-        if (flipDirection || includeDerivers) {
-            Store::computeFSClosure(paths, out, flipDirection, includeOutputs, includeDerivers);
-            return;
-        }
-
-        auto conn(connections->get());
-
-        conn->to
-            << ServeProto::Command::QueryClosure
-            << includeOutputs;
-        ServeProto::write(*this, *conn, paths);
-        conn->to.flush();
-
-        for (auto & i : ServeProto::Serialise<StorePathSet>::read(*this, *conn))
-            out.insert(i);
+        return ServeProto::ReadConn {
+            .from = from,
+            .version = remoteVersion,
+        };
     }
 
-    StorePathSet queryValidPaths(const StorePathSet & paths,
-        SubstituteFlag maybeSubstitute = NoSubstitute) override
-    {
-        auto conn(connections->get());
-
-        conn->to
-            << ServeProto::Command::QueryValidPaths
-            << false // lock
-            << maybeSubstitute;
-        ServeProto::write(*this, *conn, paths);
-        conn->to.flush();
-
-        return ServeProto::Serialise<StorePathSet>::read(*this, *conn);
-    }
-
-    void connect() override
-    {
-        auto conn(connections->get());
-    }
-
-    unsigned int getProtocol() override
-    {
-        auto conn(connections->get());
-        return conn->remoteVersion;
-    }
-
-    /**
-     * The legacy ssh protocol doesn't support checking for trusted-user.
-     * Try using ssh-ng:// instead if you want to know.
+    /*
+     * Coercion to `ServeProto::WriteConn`. This makes it easy to use the
+     * factored out serve protocol searlizers with a
+     * `LegacySSHStore::Connection`.
+     *
+     * The serve protocol connection types are unidirectional, unlike
+     * this type.
      */
-    std::optional<TrustedFlag> isTrustedClient() override
+    operator ServeProto::WriteConn ()
     {
-        return std::nullopt;
+        return ServeProto::WriteConn {
+            .to = to,
+            .version = remoteVersion,
+        };
     }
-
-    void queryRealisationUncached(const DrvOutput &,
-        Callback<std::shared_ptr<const Realisation>> callback) noexcept override
-    // TODO: Implement
-    { unsupported("queryRealisation"); }
 };
 
+
+LegacySSHStore::LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params)
+    : StoreConfig(params)
+    , CommonSSHStoreConfig(params)
+    , LegacySSHStoreConfig(params)
+    , Store(params)
+    , host(host)
+    , connections(make_ref<Pool<Connection>>(
+        std::max(1, (int) maxConnections),
+        [this]() { return openConnection(); },
+        [](const ref<Connection> & r) { return r->good; }
+        ))
+    , master(
+        host,
+        sshKey,
+        sshPublicHostKey,
+        // Use SSH master only if using more than 1 connection.
+        connections->capacity() > 1,
+        compress,
+        logFD)
+{
+}
+
+
+ref<LegacySSHStore::Connection> LegacySSHStore::openConnection()
+{
+    auto conn = make_ref<Connection>();
+    conn->sshConn = master.startCommand(
+        fmt("%s --serve --write", remoteProgram)
+        + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get())));
+    conn->to = FdSink(conn->sshConn->in.get());
+    conn->from = FdSource(conn->sshConn->out.get());
+
+    try {
+        conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION;
+        conn->to.flush();
+
+        StringSink saved;
+        try {
+            TeeSource tee(conn->from, saved);
+            unsigned int magic = readInt(tee);
+            if (magic != SERVE_MAGIC_2)
+                throw Error("'nix-store --serve' protocol mismatch from '%s'", host);
+        } catch (SerialisationError & e) {
+            /* In case the other side is waiting for our input,
+               close it. */
+            conn->sshConn->in.close();
+            auto msg = conn->from.drain();
+            throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'",
+                host, chomp(saved.s + msg));
+        }
+        conn->remoteVersion = readInt(conn->from);
+        if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200)
+            throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host);
+
+    } catch (EndOfFile & e) {
+        throw Error("cannot connect to '%1%'", host);
+    }
+
+    return conn;
+};
+
+
+std::string LegacySSHStore::getUri()
+{
+    return *uriSchemes().begin() + "://" + host;
+}
+
+
+void LegacySSHStore::queryPathInfoUncached(const StorePath & path,
+    Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
+{
+    try {
+        auto conn(connections->get());
+
+        /* No longer support missing NAR hash */
+        assert(GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4);
+
+        debug("querying remote host '%s' for info on '%s'", host, printStorePath(path));
+
+        conn->to << ServeProto::Command::QueryPathInfos << PathSet{printStorePath(path)};
+        conn->to.flush();
+
+        auto p = readString(conn->from);
+        if (p.empty()) return callback(nullptr);
+        auto path2 = parseStorePath(p);
+        assert(path == path2);
+        auto info = std::make_shared<ValidPathInfo>(
+            path,
+            ServeProto::Serialise<UnkeyedValidPathInfo>::read(*this, *conn));
+
+        if (info->narHash == Hash::dummy)
+            throw Error("NAR hash is now mandatory");
+
+        auto s = readString(conn->from);
+        assert(s == "");
+
+        callback(std::move(info));
+    } catch (...) { callback.rethrow(); }
+}
+
+
+void LegacySSHStore::addToStore(const ValidPathInfo & info, Source & source,
+    RepairFlag repair, CheckSigsFlag checkSigs)
+{
+    debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host);
+
+    auto conn(connections->get());
+
+    if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) {
+
+        conn->to
+            << ServeProto::Command::AddToStoreNar
+            << printStorePath(info.path)
+            << (info.deriver ? printStorePath(*info.deriver) : "")
+            << info.narHash.to_string(HashFormat::Base16, false);
+        ServeProto::write(*this, *conn, info.references);
+        conn->to
+            << info.registrationTime
+            << info.narSize
+            << info.ultimate
+            << info.sigs
+            << renderContentAddress(info.ca);
+        try {
+            copyNAR(source, conn->to);
+        } catch (...) {
+            conn->good = false;
+            throw;
+        }
+        conn->to.flush();
+
+    } else {
+
+        conn->to
+            << ServeProto::Command::ImportPaths
+            << 1;
+        try {
+            copyNAR(source, conn->to);
+        } catch (...) {
+            conn->good = false;
+            throw;
+        }
+        conn->to
+            << exportMagic
+            << printStorePath(info.path);
+        ServeProto::write(*this, *conn, info.references);
+        conn->to
+            << (info.deriver ? printStorePath(*info.deriver) : "")
+            << 0
+            << 0;
+        conn->to.flush();
+
+    }
+
+    if (readInt(conn->from) != 1)
+        throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host);
+}
+
+
+void LegacySSHStore::narFromPath(const StorePath & path, Sink & sink)
+{
+    auto conn(connections->get());
+
+    conn->to << ServeProto::Command::DumpStorePath << printStorePath(path);
+    conn->to.flush();
+    copyNAR(conn->from, sink);
+}
+
+
+void LegacySSHStore::putBuildSettings(Connection & conn)
+{
+    ServeProto::write(*this, conn, ServeProto::BuildOptions {
+        .maxSilentTime = settings.maxSilentTime,
+        .buildTimeout = settings.buildTimeout,
+        .maxLogSize = settings.maxLogSize,
+        .nrRepeats = 0, // buildRepeat hasn't worked for ages anyway
+        .enforceDeterminism = 0,
+        .keepFailed = settings.keepFailed,
+    });
+}
+
+
+BuildResult LegacySSHStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
+    BuildMode buildMode)
+{
+    auto conn(connections->get());
+
+    conn->to
+        << ServeProto::Command::BuildDerivation
+        << printStorePath(drvPath);
+    writeDerivation(conn->to, *this, drv);
+
+    putBuildSettings(*conn);
+
+    conn->to.flush();
+
+    return ServeProto::Serialise<BuildResult>::read(*this, *conn);
+}
+
+
+void LegacySSHStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore)
+{
+    if (evalStore && evalStore.get() != this)
+        throw Error("building on an SSH store is incompatible with '--eval-store'");
+
+    auto conn(connections->get());
+
+    conn->to << ServeProto::Command::BuildPaths;
+    Strings ss;
+    for (auto & p : drvPaths) {
+        auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p);
+        std::visit(overloaded {
+            [&](const StorePathWithOutputs & s) {
+                ss.push_back(s.to_string(*this));
+            },
+            [&](const StorePath & drvPath) {
+                throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath));
+            },
+            [&](std::monostate) {
+                throw Error("wanted build derivation that is itself a build product, but the legacy ssh protocol doesn't support that. Try using ssh-ng://");
+            },
+        }, sOrDrvPath);
+    }
+    conn->to << ss;
+
+    putBuildSettings(*conn);
+
+    conn->to.flush();
+
+    BuildResult result;
+    result.status = (BuildResult::Status) readInt(conn->from);
+
+    if (!result.success()) {
+        conn->from >> result.errorMsg;
+        throw Error(result.status, result.errorMsg);
+    }
+}
+
+
+void LegacySSHStore::computeFSClosure(const StorePathSet & paths,
+    StorePathSet & out, bool flipDirection,
+    bool includeOutputs, bool includeDerivers)
+{
+    if (flipDirection || includeDerivers) {
+        Store::computeFSClosure(paths, out, flipDirection, includeOutputs, includeDerivers);
+        return;
+    }
+
+    auto conn(connections->get());
+
+    conn->to
+        << ServeProto::Command::QueryClosure
+        << includeOutputs;
+    ServeProto::write(*this, *conn, paths);
+    conn->to.flush();
+
+    for (auto & i : ServeProto::Serialise<StorePathSet>::read(*this, *conn))
+        out.insert(i);
+}
+
+
+StorePathSet LegacySSHStore::queryValidPaths(const StorePathSet & paths,
+    SubstituteFlag maybeSubstitute)
+{
+    auto conn(connections->get());
+
+    conn->to
+        << ServeProto::Command::QueryValidPaths
+        << false // lock
+        << maybeSubstitute;
+    ServeProto::write(*this, *conn, paths);
+    conn->to.flush();
+
+    return ServeProto::Serialise<StorePathSet>::read(*this, *conn);
+}
+
+
+void LegacySSHStore::connect()
+{
+    auto conn(connections->get());
+}
+
+
+unsigned int LegacySSHStore::getProtocol()
+{
+    auto conn(connections->get());
+    return conn->remoteVersion;
+}
+
+
+/**
+ * The legacy ssh protocol doesn't support checking for trusted-user.
+ * Try using ssh-ng:// instead if you want to know.
+ */
+std::optional<TrustedFlag> isTrustedClient()
+{
+    return std::nullopt;
+}
+
+
 static RegisterStoreImplementation<LegacySSHStore, LegacySSHStoreConfig> regLegacySSHStore;
 
 }
diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh
new file mode 100644
index 000000000..c40c256bb
--- /dev/null
+++ b/src/libstore/legacy-ssh-store.hh
@@ -0,0 +1,132 @@
+#pragma once
+///@file
+
+#include "ssh-store-config.hh"
+#include "store-api.hh"
+#include "ssh.hh"
+#include "callback.hh"
+#include "pool.hh"
+
+namespace nix {
+
+struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig
+{
+    using CommonSSHStoreConfig::CommonSSHStoreConfig;
+
+    const Setting<Path> remoteProgram{this, "nix-store", "remote-program",
+        "Path to the `nix-store` executable on the remote machine."};
+
+    const Setting<int> maxConnections{this, 1, "max-connections",
+        "Maximum number of concurrent SSH connections."};
+
+    const std::string name() override { return "SSH Store"; }
+
+    std::string doc() override;
+};
+
+struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store
+{
+    // Hack for getting remote build log output.
+    // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in
+    // the documentation
+    const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
+
+    struct Connection;
+
+    std::string host;
+
+    ref<Pool<Connection>> connections;
+
+    SSHMaster master;
+
+    static std::set<std::string> uriSchemes() { return {"ssh"}; }
+
+    LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params);
+
+    ref<Connection> openConnection();
+
+    std::string getUri() override;
+
+    void queryPathInfoUncached(const StorePath & path,
+        Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override;
+
+    void addToStore(const ValidPathInfo & info, Source & source,
+        RepairFlag repair, CheckSigsFlag checkSigs) override;
+
+    void narFromPath(const StorePath & path, Sink & sink) override;
+
+    std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
+    { unsupported("queryPathFromHashPart"); }
+
+    StorePath addToStore(
+            std::string_view name,
+            const Path & srcPath,
+            FileIngestionMethod method,
+            HashAlgorithm hashAlgo,
+            PathFilter & filter,
+            RepairFlag repair,
+            const StorePathSet & references) override
+    { unsupported("addToStore"); }
+
+    StorePath addTextToStore(
+        std::string_view name,
+        std::string_view s,
+        const StorePathSet & references,
+        RepairFlag repair) override
+    { unsupported("addTextToStore"); }
+
+private:
+
+    void putBuildSettings(Connection & conn);
+
+public:
+
+    BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
+        BuildMode buildMode) override;
+
+    void buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override;
+
+    void ensurePath(const StorePath & path) override
+    { unsupported("ensurePath"); }
+
+    virtual ref<SourceAccessor> getFSAccessor(bool requireValidPath) override
+    { unsupported("getFSAccessor"); }
+
+    /**
+     * The default instance would schedule the work on the client side, but
+     * for consistency with `buildPaths` and `buildDerivation` it should happen
+     * on the remote side.
+     *
+     * We make this fail for now so we can add implement this properly later
+     * without it being a breaking change.
+     */
+    void repairPath(const StorePath & path) override
+    { unsupported("repairPath"); }
+
+    void computeFSClosure(const StorePathSet & paths,
+        StorePathSet & out, bool flipDirection = false,
+        bool includeOutputs = false, bool includeDerivers = false) override;
+
+    StorePathSet queryValidPaths(const StorePathSet & paths,
+        SubstituteFlag maybeSubstitute = NoSubstitute) override;
+
+    void connect() override;
+
+    unsigned int getProtocol() override;
+
+    /**
+     * The legacy ssh protocol doesn't support checking for trusted-user.
+     * Try using ssh-ng:// instead if you want to know.
+     */
+    std::optional<TrustedFlag> isTrustedClient() override
+    {
+        return std::nullopt;
+    }
+
+    void queryRealisationUncached(const DrvOutput &,
+        Callback<std::shared_ptr<const Realisation>> callback) noexcept override
+    // TODO: Implement
+    { unsupported("queryRealisation"); }
+};
+
+}

From e43bb655feaa23977322d68278c88ac075eb2c41 Mon Sep 17 00:00:00 2001
From: Adam Joseph <adam@westernsemico.com>
Date: Sun, 10 Dec 2023 13:58:35 -0800
Subject: [PATCH 292/294] libstore/daemon.cc: note trust model difference in
 readDerivation()s

Below the comment added by this commit is a much longer comment
followed by a trust check, both of which have confused me on at
least two occasions.  I figured it out once, forgot it, then had to
ask @Ericson2314 to explain it, at which point I understood it
again.  I think this might confuse other people too, or maybe I will
just forget it a third time.  So let's add a comment.

Farther down in the function is the following check:

```
if (!(drvType.isCA() || trusted))
  throw Error("you are not privileged to build input-addressed derivations");
```

This seems really strange at first.  A key property of Nix is that
you can compute the outpath of a derivation using the derivation
(and its references-closure) without trusting anybody!

The missing insight is that at this point in the code the builder
doesn't necessarily have the references-closure of the derivation
being built, and therefore needs to trust that the derivation's
outPath is honest.  It's incredibly easy to overlook this, because
the only difference between these two cases is which of these
identically-named functions we used:

- `readDerivation(Source,Store)`
- `Store::readDerivation()`

These functions have different trust models (except in the special
case where the first function is used on the local store).  We
should call the reader's attention to this fact.

Co-authored-by: Cole Helbling <cole.e.helbling@outlook.com>
---
 src/libstore/daemon.cc | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 530b1a178..a112d6d31 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -574,6 +574,15 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case WorkerProto::Op::BuildDerivation: {
         auto drvPath = store->parseStorePath(readString(from));
         BasicDerivation drv;
+        /*
+         * Note: unlike wopEnsurePath, this operation reads a
+         * derivation-to-be-realized from the client with
+         * readDerivation(Source,Store) rather than reading it from
+         * the local store with Store::readDerivation().  Since the
+         * derivation-to-be-realized is not registered in the store
+         * it cannot be trusted that its outPath was calculated
+         * correctly.
+         */
         readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath));
         BuildMode buildMode = (BuildMode) readInt(from);
         logger->startWork();

From 89cf53648ca98434a40b0c0cef51fa64f6e0fa37 Mon Sep 17 00:00:00 2001
From: Robert Hensing <roberth@users.noreply.github.com>
Date: Mon, 11 Dec 2023 12:26:31 +0100
Subject: [PATCH 293/294] Contributing branches and reverting (#9577)

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
---
 doc/manual/src/contributing/hacking.md | 42 ++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md
index 237eff925..4d3d66397 100644
--- a/doc/manual/src/contributing/hacking.md
+++ b/doc/manual/src/contributing/hacking.md
@@ -282,3 +282,45 @@ See also the [format documentation](https://github.com/haskell/cabal/blob/master
 
 Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`.
 Set `buildUnreleasedNotes = true;` in `flake.nix` to build the release notes on the fly.
+
+## Branches
+
+- [`master`](https://github.com/NixOS/nix/commits/master)
+
+  The main development branch. All changes are approved and merged here.
+  When developing a change, create a branch based on the latest `master`.
+
+  Maintainers try to [keep it in a release-worthy state](#reverting).
+
+- [`maintenance-*.*`](https://github.com/NixOS/nix/branches/all?query=maintenance)
+
+  These branches are the subject of backports only, and are
+  also [kept](#reverting) in a release-worthy state.
+
+  See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md)
+
+- [`latest-release`](https://github.com/NixOS/nix/tree/latest-release)
+
+  The latest patch release of the latest minor version.
+
+  See [`maintainers/release-process.md`](https://github.com/NixOS/nix/blob/master/maintainers/release-process.md)
+
+- [`backport-*-to-*`](https://github.com/NixOS/nix/branches/all?query=backport)
+
+  Generally branches created by the backport action.
+
+  See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md)
+
+- [_other_](https://github.com/NixOS/nix/branches/all)
+
+  Branches that do not conform to the above patterns should be feature branches.
+
+## Reverting
+
+If a change turns out to be merged by mistake, or contain a regression, it may be reverted.
+A revert is not a rejection of the contribution, but merely part of an effective development process.
+It makes sure that development keeps running smoothly, with minimal uncertainty, and less overhead.
+If maintainers have to worry too much about avoiding reverts, they would not be able to merge as much.
+By embracing reverts as a good part of the development process, everyone wins.
+
+However, taking a step back may be frustrating, so maintainers will be extra supportive on the next try.

From 5f30c8acc7e0cad08924cc53e350e811d097fae7 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 10 Dec 2023 18:51:23 -0500
Subject: [PATCH 294/294] Give `Store::queryDerivationOutputMap` and
 `evalStore` argument

Picking up where https://github.com/NixOS/nix/pull/9563 left off.
---
 src/libstore/store-api.cc | 4 ++--
 src/libstore/store-api.hh | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 800df7fa0..7f35e74af 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -547,8 +547,8 @@ std::map<std::string, std::optional<StorePath>> Store::queryPartialDerivationOut
     return outputs;
 }
 
-OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) {
-    auto resp = queryPartialDerivationOutputMap(path);
+OutputPathMap Store::queryDerivationOutputMap(const StorePath & path, Store * evalStore) {
+    auto resp = queryPartialDerivationOutputMap(path, evalStore);
     OutputPathMap result;
     for (auto & [outName, optOutPath] : resp) {
         if (!optOutPath)
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index ada6699d5..13e5a1446 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -369,7 +369,7 @@ public:
      * Query the mapping outputName=>outputPath for the given derivation.
      * Assume every output has a mapping and throw an exception otherwise.
      */
-    OutputPathMap queryDerivationOutputMap(const StorePath & path);
+    OutputPathMap queryDerivationOutputMap(const StorePath & path, Store * evalStore = nullptr);
 
     /**
      * Query the full store path given the hash part of a valid store