Merge remote-tracking branch 'origin/master' into detsys-main

doc/manual/generate-manpage.nix

@@ -116,9 +116,12 @@ let
             storeInfo = commandInfo.stores;
             inherit inlineHTML;
           };
+          hasInfix = infix: content:
+            builtins.stringLength content != builtins.stringLength (replaceStrings [ infix ] [ "" ] content);
         in
         optionalString (details ? doc) (
-          if match ".*@store-types@.*" details.doc != null
+          # An alternate implementation with builtins.match stack overflowed on some systems.
+          if hasInfix "@store-types@" details.doc
           then help-stores
           else details.doc
         );

doc/manual/rl-next/repl-doc-renders-doc-comments.md

@@ -0,0 +1,53 @@
+---
+synopsis: "`nix-repl`'s `:doc` shows documentation comments"
+significance: significant
+issues:
+- 3904
+- 10771
+prs:
+- 1652
+- 9054
+- 11072
+---
+
+`nix repl` has a `:doc` command that previously only rendered documentation for internally defined functions.
+This feature has been extended to also render function documentation comments, in accordance with [RFC 145].
+
+Example:
+
+```
+nix-repl> :doc lib.toFunction
+Function toFunction
+    … defined at /home/user/h/nixpkgs/lib/trivial.nix:1072:5
+
+    Turns any non-callable values into constant functions. Returns
+    callable values as is.
+
+Inputs
+
+    v
+
+      : Any value
+
+Examples
+
+    :::{.example}
+
+## lib.trivial.toFunction usage example
+
+      | nix-repl> lib.toFunction 1 2
+      | 1
+      | 
+      | nix-repl> lib.toFunction (x: x + 1) 2
+      | 3
+
+    :::
+```
+
+Known limitations:
+- It does not render documentation for "formals", such as `{ /** the value to return */ x, ... }: x`.
+- Some extensions to markdown are not yet supported, as you can see in the example above.
+
+We'd like to acknowledge Yingchi Long for proposing a proof of concept for this functionality in [#9054](https://github.com/NixOS/nix/pull/9054), as well as @sternenseemann and Johannes Kirschbauer for their contributions, proposals, and their work on [RFC 145].
+
+[RFC 145]: https://github.com/NixOS/rfcs/pull/145

doc/manual/src/contributing/testing.md

@@ -91,7 +91,7 @@ A environment variables that Google Test accepts are also worth knowing:
 Putting the two together, one might run
 
 ```bash
-GTEST_BREIF=1 GTEST_FILTER='ErrorTraceTest.*' meson test nix-expr-tests -v
+GTEST_BRIEF=1 GTEST_FILTER='ErrorTraceTest.*' meson test nix-expr-tests -v
 ```
 
 for short but comprensive output.

maintainers/flake-module.nix

@@ -143,6 +143,7 @@
             ''^src/libstore/common-protocol-impl\.hh$''
             ''^src/libstore/common-protocol\.cc$''
             ''^src/libstore/common-protocol\.hh$''
+            ''^src/libstore/common-ssh-store-config\.hh$''
             ''^src/libstore/content-address\.cc$''
             ''^src/libstore/content-address\.hh$''
             ''^src/libstore/daemon\.cc$''
@@ -215,7 +216,6 @@
             ''^src/libstore/serve-protocol\.hh$''
             ''^src/libstore/sqlite\.cc$''
             ''^src/libstore/sqlite\.hh$''
-            ''^src/libstore/ssh-store-config\.hh$''
             ''^src/libstore/ssh-store\.cc$''
             ''^src/libstore/ssh\.cc$''
             ''^src/libstore/ssh\.hh$''
@@ -499,7 +499,6 @@
             ''^misc/bash/completion\.sh$''
             ''^misc/fish/completion\.fish$''
             ''^misc/zsh/completion\.zsh$''
-            ''^scripts/check-hydra-status\.sh$''
             ''^scripts/create-darwin-volume\.sh$''
             ''^scripts/install-darwin-multi-user\.sh$''
             ''^scripts/install-multi-user\.sh$''

packaging/dependencies.nix

@@ -11,11 +11,28 @@
   versionSuffix,
 }:
 
+let
+  prevStdenv = stdenv;
+in
+
 let
   inherit (pkgs) lib;
 
   root = ../.;
 
+  stdenv = if prevStdenv.isDarwin && prevStdenv.isx86_64
+    then darwinStdenv
+    else prevStdenv;
+
+  # Fix the following error with the default x86_64-darwin SDK:
+  #
+  #     error: aligned allocation function of type 'void *(std::size_t, std::align_val_t)' is only available on macOS 10.13 or newer
+  #
+  # Despite the use of the 10.13 deployment target here, the aligned
+  # allocation function Clang uses with this setting actually works
+  # all the way back to 10.6.
+  darwinStdenv = pkgs.overrideSDK prevStdenv { darwinMinVersion = "10.13"; };
+
   # Nixpkgs implements this by returning a subpath into the fetched Nix sources.
   resolvePath = p: p;
 

src/build-remote/build-remote.cc

@@ -16,6 +16,7 @@
 #include "serialise.hh"
 #include "build-result.hh"
 #include "store-api.hh"
+#include "strings.hh"
 #include "derivations.hh"
 #include "local-store.hh"
 #include "legacy.hh"

src/libcmd/built-path.cc

@@ -1,6 +1,7 @@
 #include "built-path.hh"
 #include "derivations.hh"
 #include "store-api.hh"
+#include "comparator.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -8,30 +9,24 @@
 
 namespace nix {
 
-#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \
-    bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \
-    { \
-        const MY_TYPE* me = this; \
-        auto fields1 = std::tie(*me->drvPath, me->FIELD); \
-        me = &other; \
-        auto fields2 = std::tie(*me->drvPath, me->FIELD); \
-        return fields1 COMPARATOR fields2; \
-    }
-#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \
-    CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \
-    CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \
-    CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <)
+// Custom implementation to avoid `ref` ptr equality
+GENERATE_CMP_EXT(
+    ,
+    std::strong_ordering,
+    SingleBuiltPathBuilt,
+    *me->drvPath,
+    me->output);
 
-#define FIELD_TYPE std::pair<std::string, StorePath>
-CMP(SingleBuiltPath, SingleBuiltPathBuilt, output)
-#undef FIELD_TYPE
+// Custom implementation to avoid `ref` ptr equality
 
-#define FIELD_TYPE std::map<std::string, StorePath>
-CMP(SingleBuiltPath, BuiltPathBuilt, outputs)
-#undef FIELD_TYPE
-
-#undef CMP
-#undef CMP_ONE
+// TODO no `GENERATE_CMP_EXT` because no `std::set::operator<=>` on
+// Darwin, per header.
+GENERATE_EQUAL(
+    ,
+    BuiltPathBuilt ::,
+    BuiltPathBuilt,
+    *me->drvPath,
+    me->outputs);
 
 StorePath SingleBuiltPath::outPath() const
 {

src/libcmd/built-path.hh

@@ -18,7 +18,8 @@ struct SingleBuiltPathBuilt {
     static SingleBuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view);
     nlohmann::json toJSON(const StoreDirConfig & store) const;
 
-    DECLARE_CMP(SingleBuiltPathBuilt);
+    bool operator ==(const SingleBuiltPathBuilt &) const noexcept;
+    std::strong_ordering operator <=>(const SingleBuiltPathBuilt &) const noexcept;
 };
 
 using _SingleBuiltPathRaw = std::variant<
@@ -33,6 +34,9 @@ struct SingleBuiltPath : _SingleBuiltPathRaw {
     using Opaque = DerivedPathOpaque;
     using Built = SingleBuiltPathBuilt;
 
+    bool operator == (const SingleBuiltPath &) const = default;
+    auto operator <=> (const SingleBuiltPath &) const = default;
+
     inline const Raw & raw() const {
         return static_cast<const Raw &>(*this);
     }
@@ -59,11 +63,13 @@ struct BuiltPathBuilt {
     ref<SingleBuiltPath> drvPath;
     std::map<std::string, StorePath> outputs;
 
+    bool operator == (const BuiltPathBuilt &) const noexcept;
+    // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet.
+    //std::strong_ordering operator <=> (const BuiltPathBuilt &) const noexcept;
+
     std::string to_string(const StoreDirConfig & store) const;
     static BuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view);
     nlohmann::json toJSON(const StoreDirConfig & store) const;
-
-    DECLARE_CMP(BuiltPathBuilt);
 };
 
 using _BuiltPathRaw = std::variant<
@@ -82,6 +88,10 @@ struct BuiltPath : _BuiltPathRaw {
     using Opaque = DerivedPathOpaque;
     using Built = BuiltPathBuilt;
 
+    bool operator == (const BuiltPath &) const = default;
+    // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet.
+    //auto operator <=> (const BuiltPath &) const = default;
+
     inline const Raw & raw() const {
         return static_cast<const Raw &>(*this);
     }

src/libcmd/command.cc

@@ -1,3 +1,5 @@
+#include <nlohmann/json.hpp>
+
 #include "command.hh"
 #include "markdown.hh"
 #include "store-api.hh"
@@ -6,8 +8,7 @@
 #include "nixexpr.hh"
 #include "profiles.hh"
 #include "repl.hh"
-
-#include <nlohmann/json.hpp>
+#include "strings.hh"
 
 extern char * * environ __attribute__((weak));
 
@@ -132,7 +133,7 @@ ref<EvalState> EvalCommand::getEvalState()
             #else
             std::make_shared<EvalState>(
             #endif
-                lookupPath, getEvalStore(), evalSettings, getStore())
+                lookupPath, getEvalStore(), fetchSettings, evalSettings, getStore())
             ;
 
         evalState->repair = repair;

src/libcmd/common-eval-args.cc

@@ -1,3 +1,4 @@
+#include "fetch-settings.hh"
 #include "eval-settings.hh"
 #include "common-eval-args.hh"
 #include "shared.hh"
@@ -7,6 +8,7 @@
 #include "fetchers.hh"
 #include "registry.hh"
 #include "flake/flakeref.hh"
+#include "flake/settings.hh"
 #include "store-api.hh"
 #include "command.hh"
 #include "tarball.hh"
@@ -16,6 +18,10 @@
 
 namespace nix {
 
+fetchers::Settings fetchSettings;
+
+static GlobalConfig::Register rFetchSettings(&fetchSettings);
+
 EvalSettings evalSettings {
     settings.readOnlyMode,
     {
@@ -23,7 +29,7 @@ EvalSettings evalSettings {
             "flake",
             [](ref<Store> store, std::string_view rest) {
                 // FIXME `parseFlakeRef` should take a `std::string_view`.
-                auto flakeRef = parseFlakeRef(std::string { rest }, {}, true, false);
+                auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false);
                 debug("fetching flake search path element '%s''", rest);
                 auto storePath = flakeRef.resolve(store).fetchTree(store).first;
                 return store->toRealPath(storePath);
@@ -34,6 +40,12 @@ EvalSettings evalSettings {
 
 static GlobalConfig::Register rEvalSettings(&evalSettings);
 
+
+flake::Settings flakeSettings;
+
+static GlobalConfig::Register rFlakeSettings(&flakeSettings);
+
+
 CompatibilitySettings compatibilitySettings {};
 
 static GlobalConfig::Register rCompatibilitySettings(&compatibilitySettings);
@@ -170,8 +182,8 @@ MixEvalArgs::MixEvalArgs()
         .category = category,
         .labels = {"original-ref", "resolved-ref"},
         .handler = {[&](std::string _from, std::string _to) {
-            auto from = parseFlakeRef(_from, absPath("."));
-            auto to = parseFlakeRef(_to, absPath("."));
+            auto from = parseFlakeRef(fetchSettings, _from, absPath("."));
+            auto to = parseFlakeRef(fetchSettings, _to, absPath("."));
             fetchers::Attrs extraAttrs;
             if (to.subdir != "") extraAttrs["dir"] = to.subdir;
             fetchers::overrideRegistry(from.input, to.input, extraAttrs);
@@ -228,7 +240,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas
     }
 
     else if (hasPrefix(s, "flake:")) {
-        auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false);
+        auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false);
         auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first;
         return state.rootPath(CanonPath(state.store->toRealPath(storePath)));
     }

src/libcmd/common-eval-args.hh

@@ -11,17 +11,32 @@
 namespace nix {
 
 class Store;
+
+namespace fetchers { struct Settings; }
+
 class EvalState;
 struct EvalSettings;
 struct CompatibilitySettings;
 class Bindings;
 struct SourcePath;
 
+namespace flake { struct Settings; }
+
+/**
+ * @todo Get rid of global setttings variables
+ */
+extern fetchers::Settings fetchSettings;
+
 /**
  * @todo Get rid of global setttings variables
  */
 extern EvalSettings evalSettings;
 
+/**
+ * @todo Get rid of global setttings variables
+ */
+extern flake::Settings flakeSettings;
+
 /**
  * Settings that control behaviors that have changed since Nix 2.3.
  */

src/libcmd/installable-flake.cc

@@ -196,7 +196,8 @@ std::shared_ptr<flake::LockedFlake> InstallableFlake::getLockedFlake() const
         flake::LockFlags lockFlagsApplyConfig = lockFlags;
         // FIXME why this side effect?
         lockFlagsApplyConfig.applyNixConfig = true;
-        _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlagsApplyConfig));
+        _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(
+            flakeSettings, *state, flakeRef, lockFlagsApplyConfig));
     }
     return _lockedFlake;
 }

src/libcmd/installable-flake.hh

@@ -1,6 +1,7 @@
 #pragma once
 ///@file
 
+#include "common-eval-args.hh"
 #include "installable-value.hh"
 
 namespace nix {
@@ -78,7 +79,7 @@ struct InstallableFlake : InstallableValue
  */
 static inline FlakeRef defaultNixpkgsFlakeRef()
 {
-    return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}});
+    return FlakeRef::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", "nixpkgs"}});
 }
 
 ref<eval_cache::EvalCache> openEvalCache(

src/libcmd/installables.cc

@@ -27,6 +27,8 @@
 
 #include <nlohmann/json.hpp>
 
+#include "strings-inline.hh"
+
 namespace nix {
 
 void completeFlakeInputPath(
@@ -129,7 +131,7 @@ MixFlakeOptions::MixFlakeOptions()
             lockFlags.writeLockFile = false;
             lockFlags.inputOverrides.insert_or_assign(
                 flake::parseInputPath(inputPath),
-                parseFlakeRef(flakeRef, absPath(getCommandBaseDir()), true));
+                parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()), true));
         }},
         .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) {
             if (n == 0) {
@@ -170,14 +172,15 @@ MixFlakeOptions::MixFlakeOptions()
         .handler = {[&](std::string flakeRef) {
             auto evalState = getEvalState();
             auto flake = flake::lockFlake(
+                flakeSettings,
                 *evalState,
-                parseFlakeRef(flakeRef, absPath(getCommandBaseDir())),
+                parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir())),
                 { .writeLockFile = false });
             for (auto & [inputName, input] : flake.lockFile.root->inputs) {
                 auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes
                 if (auto input3 = std::dynamic_pointer_cast<const flake::LockedNode>(input2)) {
                     overrideRegistry(
-                        fetchers::Input::fromAttrs({{"type","indirect"}, {"id", inputName}}),
+                        fetchers::Input::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", inputName}}),
                         input3->lockedRef.input,
                         {});
                 }
@@ -289,10 +292,10 @@ void SourceExprCommand::completeInstallable(AddCompletions & completions, std::s
 
             if (v2.type() == nAttrs) {
                 for (auto & i : *v2.attrs()) {
-                    std::string name = state->symbols[i.name];
+                    std::string_view name = state->symbols[i.name];
                     if (name.find(searchWord) == 0) {
                         if (prefix_ == "")
-                            completions.add(name);
+                            completions.add(std::string(name));
                         else
                             completions.add(prefix_ + "." + name);
                     }
@@ -338,10 +341,11 @@ void completeFlakeRefWithFragment(
             auto flakeRefS = std::string(prefix.substr(0, hash));
 
             // TODO: ideally this would use the command base directory instead of assuming ".".
-            auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath("."));
+            auto flakeRef = parseFlakeRef(fetchSettings, expandTilde(flakeRefS), absPath("."));
 
             auto evalCache = openEvalCache(*evalState,
-                std::make_shared<flake::LockedFlake>(lockFlake(*evalState, flakeRef, lockFlags)));
+                std::make_shared<flake::LockedFlake>(lockFlake(
+                    flakeSettings, *evalState, flakeRef, lockFlags)));
 
             auto root = evalCache->getRoot();
 
@@ -372,6 +376,7 @@ void completeFlakeRefWithFragment(
                         auto attrPath2 = (*attr)->getAttrPath(attr2);
                         /* Strip the attrpath prefix. */
                         attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size());
+                        // FIXME: handle names with dots
                         completions.add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2)));
                     }
                 }
@@ -400,7 +405,7 @@ void completeFlakeRef(AddCompletions & completions, ref<Store> store, std::strin
     Args::completeDir(completions, 0, prefix);
 
     /* Look for registry entries that match the prefix. */
-    for (auto & registry : fetchers::getRegistries(store)) {
+    for (auto & registry : fetchers::getRegistries(fetchSettings, store)) {
         for (auto & entry : registry->entries) {
             auto from = entry.from.to_string();
             if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) {
@@ -526,7 +531,8 @@ Installables SourceExprCommand::parseInstallables(
             }
 
             try {
-                auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(getCommandBaseDir()));
+                auto [flakeRef, fragment] = parseFlakeRefWithFragment(
+                    fetchSettings, std::string { prefix }, absPath(getCommandBaseDir()));
                 result.push_back(make_ref<InstallableFlake>(
                         this,
                         getEvalState(),
@@ -843,6 +849,7 @@ std::vector<FlakeRef> RawInstallablesCommand::getFlakeRefsForCompletion()
     std::vector<FlakeRef> res;
     for (auto i : rawInstallables)
         res.push_back(parseFlakeRefWithFragment(
+            fetchSettings,
             expandTilde(i),
             absPath(getCommandBaseDir())).first);
     return res;
@@ -865,6 +872,7 @@ std::vector<FlakeRef> InstallableCommand::getFlakeRefsForCompletion()
 {
     return {
         parseFlakeRefWithFragment(
+            fetchSettings,
             expandTilde(_installable),
             absPath(getCommandBaseDir())).first
     };

src/libcmd/repl-interacter.cc

@@ -73,7 +73,7 @@ static int listPossibleCallback(char * s, char *** avp)
 {
     auto possible = curRepl->completePrefix(s);
 
-    if (possible.size() > (INT_MAX / sizeof(char *)))
+    if (possible.size() > (std::numeric_limits<int>::max() / sizeof(char *)))
         throw Error("too many completions");
 
     int ac = 0;

src/libcmd/repl.cc

@@ -1,7 +1,6 @@
 #include <iostream>
 #include <cstdlib>
 #include <cstring>
-#include <climits>
 
 #include "repl-interacter.hh"
 #include "repl.hh"
@@ -10,8 +9,6 @@
 #include "shared.hh"
 #include "config-global.hh"
 #include "eval.hh"
-#include "eval-cache.hh"
-#include "eval-inline.hh"
 #include "eval-settings.hh"
 #include "attr-path.hh"
 #include "signals.hh"
@@ -29,12 +26,16 @@
 #include "markdown.hh"
 #include "local-fs-store.hh"
 #include "print.hh"
+#include "ref.hh"
+#include "value.hh"
 
 #if HAVE_BOEHMGC
 #define GC_INCLUDE_NEW
 #include <gc/gc_cpp.h>
 #endif
 
+#include "strings.hh"
+
 namespace nix {
 
 /**
@@ -616,6 +617,38 @@ ProcessLineResult NixRepl::processLine(std::string line)
 
     else if (command == ":doc") {
         Value v;
+
+        auto expr = parseString(arg);
+        std::string fallbackName;
+        PosIdx fallbackPos;
+        DocComment fallbackDoc;
+        if (auto select = dynamic_cast<ExprSelect *>(expr)) {
+            Value vAttrs;
+            auto name = select->evalExceptFinalSelect(*state, *env, vAttrs);
+            fallbackName = state->symbols[name];
+
+            state->forceAttrs(vAttrs, noPos, "while evaluating an attribute set to look for documentation");
+            auto attrs = vAttrs.attrs();
+            assert(attrs);
+            auto attr = attrs->get(name);
+            if (!attr) {
+                // When missing, trigger the normal exception
+                // e.g. :doc builtins.foo
+                // behaves like
+                // nix-repl> builtins.foo      
+                // error: attribute 'foo' missing
+                evalString(arg, v);
+                assert(false);
+            }
+            if (attr->pos) {
+                fallbackPos = attr->pos;
+                fallbackDoc = state->getDocCommentForPos(fallbackPos);
+            }
+
+        } else {
+            evalString(arg, v);
+        }
+
         evalString(arg, v);
         if (auto doc = state->getDoc(v)) {
             std::string markdown;
@@ -633,6 +666,19 @@ ProcessLineResult NixRepl::processLine(std::string line)
             markdown += stripIndentation(doc->doc);
 
             logger->cout(trim(renderMarkdownToTerminal(markdown)));
+        } else if (fallbackPos) {
+            std::stringstream ss;
+            ss << "Attribute `" << fallbackName << "`\n\n";
+            ss << "  … defined at " << state->positions[fallbackPos] << "\n\n";
+            if (fallbackDoc) {
+                ss << fallbackDoc.getInnerText(state->positions);
+            } else {
+                ss << "No documentation found.\n\n";
+            }
+
+            auto markdown = ss.str();
+            logger->cout(trim(renderMarkdownToTerminal(markdown)));
+
         } else
             throw Error("value does not have documentation");
     }
@@ -690,14 +736,14 @@ void NixRepl::loadFlake(const std::string & flakeRefS)
     if (flakeRefS.empty())
         throw Error("cannot use ':load-flake' without a path specified. (Use '.' for the current working directory.)");
 
-    auto flakeRef = parseFlakeRef(flakeRefS, absPath("."), true);
+    auto flakeRef = parseFlakeRef(fetchSettings, flakeRefS, absPath("."), true);
     if (evalSettings.pureEval && !flakeRef.input.isLocked())
         throw Error("cannot use ':load-flake' on locked flake reference '%s' (use --impure to override)", flakeRefS);
 
     Value v;
 
     flake::callFlake(*state,
-        flake::lockFlake(*state, flakeRef,
+        flake::lockFlake(flakeSettings, *state, flakeRef,
             flake::LockFlags {
                 .updateLockFile = false,
                 .useRegistries = !evalSettings.pureEval,

src/libexpr-c/local.mk

@@ -15,7 +15,7 @@ libexprc_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libutilc) \
                      $(INCLUDE_libstore) $(INCLUDE_libstorec) \
                      $(INCLUDE_libexpr) $(INCLUDE_libexprc)
 
-libexprc_LIBS = libutil libutilc libstore libstorec libexpr
+libexprc_LIBS = libutil libutilc libstore libstorec libfetchers libexpr
 
 libexprc_LDFLAGS += $(THREAD_LDFLAGS)
 

src/libexpr-c/nix_api_expr.cc

@@ -1,12 +1,10 @@
 #include <cstring>
-#include <iostream>
 #include <stdexcept>
 #include <string>
 
-#include "config.hh"
 #include "eval.hh"
+#include "eval-gc.hh"
 #include "globals.hh"
-#include "util.hh"
 #include "eval-settings.hh"
 
 #include "nix_api_expr.h"
@@ -112,12 +110,14 @@ EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c
             static_cast<std::align_val_t>(alignof(EvalState)));
         auto * p2 = static_cast<EvalState *>(p);
         new (p) EvalState {
+            .fetchSettings = nix::fetchers::Settings{},
             .settings = nix::EvalSettings{
                 nix::settings.readOnlyMode,
             },
             .state = nix::EvalState(
                 nix::LookupPath::parse(lookupPath),
                 store->ptr,
+                p2->fetchSettings,
                 p2->settings),
         };
         loadConfFile(p2->settings);

src/libexpr-c/nix_api_expr_internal.h

@@ -1,6 +1,7 @@
 #ifndef NIX_API_EXPR_INTERNAL_H
 #define NIX_API_EXPR_INTERNAL_H
 
+#include "fetch-settings.hh"
 #include "eval.hh"
 #include "eval-settings.hh"
 #include "attr-set.hh"
@@ -8,6 +9,7 @@
 
 struct EvalState
 {
+    nix::fetchers::Settings fetchSettings;
     nix::EvalSettings settings;
     nix::EvalState state;
 };

src/libexpr-c/nix_api_external.cc

@@ -115,7 +115,7 @@ public:
     /**
      * Compare to another value of the same type.
      */
-    virtual bool operator==(const ExternalValueBase & b) const override
+    virtual bool operator==(const ExternalValueBase & b) const noexcept override
     {
         if (!desc.equal) {
             return false;

src/libexpr-c/nix_api_value.cc

@@ -383,7 +383,7 @@ nix_value * nix_get_attr_byidx(
     try {
         auto & v = check_value_in(value);
         const nix::Attr & a = (*v.attrs())[i];
-        *name = ((const std::string &) (state->state.symbols[a.name])).c_str();
+        *name = state->state.symbols[a.name].c_str();
         nix_gc_incref(nullptr, a.value);
         state->state.forceValue(*a.value, nix::noPos);
         return as_nix_value_ptr(a.value);
@@ -399,7 +399,7 @@ nix_get_attr_name_byidx(nix_c_context * context, const nix_value * value, EvalSt
     try {
         auto & v = check_value_in(value);
         const nix::Attr & a = (*v.attrs())[i];
-        return ((const std::string &) (state->state.symbols[a.name])).c_str();
+        return state->state.symbols[a.name].c_str();
     }
     NIXC_CATCH_ERRS_NULL
 }

src/libexpr/attr-path.cc

@@ -76,7 +76,7 @@ std::pair<Value *, PosIdx> findAlongAttrPath(EvalState & state, const std::strin
             if (!a) {
                 std::set<std::string> attrNames;
                 for (auto & attr : *v->attrs())
-                    attrNames.insert(state.symbols[attr.name]);
+                    attrNames.insert(std::string(state.symbols[attr.name]));
 
                 auto suggestions = Suggestions::bestMatches(attrNames, attr);
                 throw AttrPathNotFound(suggestions, "attribute '%1%' in selection path '%2%' not found", attr, attrPath);

src/libexpr/attr-set.hh

@@ -5,7 +5,6 @@
 #include "symbol-table.hh"
 
 #include <algorithm>
-#include <optional>
 
 namespace nix {
 
@@ -28,9 +27,9 @@ struct Attr
     Attr(Symbol name, Value * value, PosIdx pos = noPos)
         : name(name), pos(pos), value(value) { };
     Attr() { };
-    bool operator < (const Attr & a) const
+    auto operator <=> (const Attr & a) const
     {
-        return name < a.name;
+        return name <=> a.name;
     }
 };
 

src/libexpr/eval-cache.cc

@@ -225,7 +225,7 @@ struct AttrDb
                 (key.first)
                 (symbols[key.second])
                 (AttrType::ListOfStrings)
-                (concatStringsSep("\t", l)).exec();
+                (dropEmptyInitThenConcatStringsSep("\t", l)).exec();
 
             return state->db.getLastInsertedRowId();
         });
@@ -441,12 +441,12 @@ std::vector<Symbol> AttrCursor::getAttrPath(Symbol name) const
 
 std::string AttrCursor::getAttrPathStr() const
 {
-    return concatStringsSep(".", root->state.symbols.resolve(getAttrPath()));
+    return dropEmptyInitThenConcatStringsSep(".", root->state.symbols.resolve(getAttrPath()));
 }
 
 std::string AttrCursor::getAttrPathStr(Symbol name) const
 {
-    return concatStringsSep(".", root->state.symbols.resolve(getAttrPath(name)));
+    return dropEmptyInitThenConcatStringsSep(".", root->state.symbols.resolve(getAttrPath(name)));
 }
 
 Value & AttrCursor::forceValue()
@@ -490,7 +490,7 @@ Suggestions AttrCursor::getSuggestionsForAttr(Symbol name)
     auto attrNames = getAttrs();
     std::set<std::string> strAttrNames;
     for (auto & name : attrNames)
-        strAttrNames.insert(root->state.symbols[name]);
+        strAttrNames.insert(std::string(root->state.symbols[name]));
 
     return Suggestions::bestMatches(strAttrNames, root->state.symbols[name]);
 }

src/libexpr/eval-error.hh

@@ -1,7 +1,5 @@
 #pragma once
 
-#include <algorithm>
-
 #include "error.hh"
 #include "pos-idx.hh"
 

src/libexpr/eval-settings.cc

@@ -1,5 +1,4 @@
 #include "users.hh"
-#include "config-global.hh"
 #include "globals.hh"
 #include "profiles.hh"
 #include "eval.hh"

src/libexpr/eval-settings.hh

@@ -2,6 +2,7 @@
 ///@file
 
 #include "config.hh"
+#include "ref.hh"
 
 namespace nix {
 

src/libexpr/eval.cc

@@ -1,6 +1,6 @@
 #include "eval.hh"
+#include "eval-gc.hh"
 #include "eval-settings.hh"
-#include "hash.hh"
 #include "primops.hh"
 #include "print-options.hh"
 #include "exit.hh"
@@ -16,7 +16,6 @@
 #include "print.hh"
 #include "filtering-source-accessor.hh"
 #include "memory-source-accessor.hh"
-#include "signals.hh"
 #include "gc-small-vector.hh"
 #include "url.hh"
 #include "fetch-to-store.hh"
@@ -24,7 +23,6 @@
 #include "parser-tab.hh"
 
 #include <algorithm>
-#include <chrono>
 #include <iostream>
 #include <sstream>
 #include <cstring>
@@ -51,6 +49,8 @@
 
 #endif
 
+#include "strings-inline.hh"
+
 using json = nlohmann::json;
 
 namespace nix {
@@ -217,9 +217,11 @@ static constexpr size_t BASE_ENV_SIZE = 128;
 EvalState::EvalState(
     const LookupPath & _lookupPath,
     ref<Store> store,
+    const fetchers::Settings & fetchSettings,
     const EvalSettings & settings,
     std::shared_ptr<Store> buildStore)
-    : settings{settings}
+    : fetchSettings{fetchSettings}
+    , settings{settings}
     , sWith(symbols.create("<with>"))
     , sOutPath(symbols.create("outPath"))
     , sDrvPath(symbols.create("drvPath"))
@@ -557,6 +559,54 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
                 .doc = doc,
             };
     }
+    if (v.isLambda()) {
+        auto exprLambda = v.payload.lambda.fun;
+
+        std::stringstream s(std::ios_base::out);
+        std::string name;
+        auto pos = positions[exprLambda->getPos()];
+        std::string docStr;
+
+        if (exprLambda->name) {
+            name = symbols[exprLambda->name];
+        }
+
+        if (exprLambda->docComment) {
+            docStr = exprLambda->docComment.getInnerText(positions);
+        }
+
+        if (name.empty()) {
+            s << "Function ";
+        }
+        else {
+            s << "Function `" << name << "`";
+            if (pos)
+                s << "\\\n  … " ;
+            else
+                s << "\\\n";
+        }
+        if (pos) {
+            s << "defined at " << pos;
+        }
+        if (!docStr.empty()) {
+            s << "\n\n";
+        }
+
+        s << docStr;
+
+        s << '\0'; // for making a c string below
+        std::string ss = s.str();
+
+        return Doc {
+            .pos = pos,
+            .name = name,
+            .arity = 0, // FIXME: figure out how deep by syntax only? It's not semantically useful though...
+            .args = {},
+            .doc =
+                // FIXME: this leaks; make the field std::string?
+                strdup(ss.data()),
+        };
+    }
     return {};
 }
 
@@ -633,11 +683,11 @@ void mapStaticEnvBindings(const SymbolTable & st, const StaticEnv & se, const En
         if (se.isWith && !env.values[0]->isThunk()) {
             // add 'with' bindings.
             for (auto & j : *env.values[0]->attrs())
-                vm[st[j.name]] = j.value;
+                vm.insert_or_assign(std::string(st[j.name]), j.value);
         } else {
             // iterate through staticenv bindings and add them.
             for (auto & i : se.vars)
-                vm[st[i.first]] = env.values[i.second];
+                vm.insert_or_assign(std::string(st[i.first]), env.values[i.second]);
         }
     }
 }
@@ -1011,7 +1061,7 @@ void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial)
     if (!e)
         e = parseExprFromFile(resolvedPath);
 
-    fileParseCache[resolvedPath] = e;
+    fileParseCache.emplace(resolvedPath, e);
 
     try {
         auto dts = debugRepl
@@ -1034,8 +1084,8 @@ void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial)
         throw;
     }
 
-    fileEvalCache[resolvedPath] = v;
-    if (path != resolvedPath) fileEvalCache[path] = v;
+    fileEvalCache.emplace(resolvedPath, v);
+    if (path != resolvedPath) fileEvalCache.emplace(path, v);
 }
 
 
@@ -1338,7 +1388,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
                 if (!(j = vAttrs->attrs()->get(name))) {
                     std::set<std::string> allAttrNames;
                     for (auto & attr : *vAttrs->attrs())
-                        allAttrNames.insert(state.symbols[attr.name]);
+                        allAttrNames.insert(std::string(state.symbols[attr.name]));
                     auto suggestions = Suggestions::bestMatches(allAttrNames, state.symbols[name]);
                     state.error<EvalError>("attribute '%1%' missing", state.symbols[name])
                         .atPos(pos).withSuggestions(suggestions).withFrame(env, *this).debugThrow();
@@ -1365,6 +1415,22 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
     v = *vAttrs;
 }
 
+Symbol ExprSelect::evalExceptFinalSelect(EvalState & state, Env & env, Value & attrs)
+{
+    Value vTmp;
+    Symbol name = getName(attrPath[attrPath.size() - 1], state, env);
+
+    if (attrPath.size() == 1) {
+        e->eval(state, env, vTmp);
+    } else {
+        ExprSelect init(*this);
+        init.attrPath.pop_back();
+        init.eval(state, env, vTmp);
+    }
+    attrs = vTmp;
+    return name;
+}
+
 
 void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v)
 {
@@ -1496,7 +1562,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                         if (!lambda.formals->has(i.name)) {
                             std::set<std::string> formalNames;
                             for (auto & formal : lambda.formals->formals)
-                                formalNames.insert(symbols[formal.name]);
+                                formalNames.insert(std::string(symbols[formal.name]));
                             auto suggestions = Suggestions::bestMatches(formalNames, symbols[i.name]);
                             error<TypeError>("function '%1%' called with unexpected argument '%2%'",
                                              (lambda.name ? std::string(symbols[lambda.name]) : "anonymous lambda"),
@@ -2826,13 +2892,37 @@ Expr * EvalState::parse(
     const SourcePath & basePath,
     std::shared_ptr<StaticEnv> & staticEnv)
 {
-    auto result = parseExprFromBuf(text, length, origin, basePath, symbols, settings, positions, rootFS, exprSymbols);
+    DocCommentMap tmpDocComments; // Only used when not origin is not a SourcePath
+    DocCommentMap *docComments = &tmpDocComments;
+
+    if (auto sourcePath = std::get_if<SourcePath>(&origin)) {
+        auto [it, _] = positionToDocComment.try_emplace(*sourcePath);
+        docComments = &it->second;
+    }
+
+    auto result = parseExprFromBuf(text, length, origin, basePath, symbols, settings, positions, *docComments, rootFS, exprSymbols);
 
     result->bindVars(*this, staticEnv);
 
     return result;
 }
 
+DocComment EvalState::getDocCommentForPos(PosIdx pos)
+{
+    auto pos2 = positions[pos];
+    auto path = pos2.getSourcePath();
+    if (!path)
+        return {};
+
+    auto table = positionToDocComment.find(*path);
+    if (table == positionToDocComment.end())
+        return {};
+
+    auto it = table->second.find(pos);
+    if (it == table->second.end())
+        return {};
+    return it->second;
+}
 
 std::string ExternalValueBase::coerceToString(EvalState & state, const PosIdx & pos, NixStringContext & context, bool copyMore, bool copyToStore) const
 {
@@ -2842,7 +2932,7 @@ std::string ExternalValueBase::coerceToString(EvalState & state, const PosIdx &
 }
 
 
-bool ExternalValueBase::operator==(const ExternalValueBase & b) const
+bool ExternalValueBase::operator==(const ExternalValueBase & b) const noexcept
 {
     return false;
 }

src/libexpr/eval.hh

@@ -3,21 +3,21 @@
 
 #include "attr-set.hh"
 #include "eval-error.hh"
-#include "eval-gc.hh"
 #include "types.hh"
 #include "value.hh"
 #include "nixexpr.hh"
 #include "symbol-table.hh"
 #include "config.hh"
 #include "experimental-features.hh"
+#include "position.hh"
+#include "pos-table.hh"
 #include "source-accessor.hh"
 #include "search-path.hh"
 #include "repl-exit-status.hh"
+#include "ref.hh"
 
 #include <map>
 #include <optional>
-#include <unordered_map>
-#include <mutex>
 #include <functional>
 
 namespace nix {
@@ -30,6 +30,7 @@ namespace nix {
 constexpr size_t maxPrimOpArity = 8;
 
 class Store;
+namespace fetchers { struct Settings; }
 struct EvalSettings;
 class EvalState;
 class StorePath;
@@ -43,7 +44,7 @@ namespace eval_cache {
 /**
  * Function that implements a primop.
  */
-typedef void (* PrimOpFun) (EvalState & state, const PosIdx pos, Value * * args, Value & v);
+using PrimOpFun = void(EvalState & state, const PosIdx pos, Value * * args, Value & v);
 
 /**
  * Info about a primitive operation, and its implementation
@@ -84,7 +85,7 @@ struct PrimOp
     /**
      * Implementation of the primop.
      */
-    std::function<std::remove_pointer<PrimOpFun>::type> fun;
+    std::function<PrimOpFun> fun;
 
     /**
      * Optional experimental for this to be gated on.
@@ -129,6 +130,8 @@ struct Constant
     typedef std::map<std::string, Value *> ValMap;
 #endif
 
+typedef std::map<PosIdx, DocComment> DocCommentMap;
+
 struct Env
 {
     Env * up;
@@ -162,6 +165,7 @@ struct DebugTrace {
 class EvalState : public std::enable_shared_from_this<EvalState>
 {
 public:
+    const fetchers::Settings & fetchSettings;
     const EvalSettings & settings;
     SymbolTable symbols;
     PosTable positions;
@@ -305,15 +309,15 @@ private:
 
     /* Cache for calls to addToStore(); maps source paths to the store
        paths. */
-    Sync<std::map<SourcePath, StorePath>> srcToStore;
+    Sync<std::unordered_map<SourcePath, StorePath>> srcToStore;
 
     /**
      * A cache from path names to parse trees.
      */
 #if HAVE_BOEHMGC
-    typedef std::map<SourcePath, Expr *, std::less<SourcePath>, traceable_allocator<std::pair<const SourcePath, Expr *>>> FileParseCache;
+    typedef std::unordered_map<SourcePath, Expr *, std::hash<SourcePath>, std::equal_to<SourcePath>, traceable_allocator<std::pair<const SourcePath, Expr *>>> FileParseCache;
 #else
-    typedef std::map<SourcePath, Expr *> FileParseCache;
+    typedef std::unordered_map<SourcePath, Expr *> FileParseCache;
 #endif
     FileParseCache fileParseCache;
 
@@ -321,12 +325,18 @@ private:
      * A cache from path names to values.
      */
 #if HAVE_BOEHMGC
-    typedef std::map<SourcePath, Value, std::less<SourcePath>, traceable_allocator<std::pair<const SourcePath, Value>>> FileEvalCache;
+    typedef std::unordered_map<SourcePath, Value, std::hash<SourcePath>, std::equal_to<SourcePath>, traceable_allocator<std::pair<const SourcePath, Value>>> FileEvalCache;
 #else
-    typedef std::map<SourcePath, Value> FileEvalCache;
+    typedef std::unordered_map<SourcePath, Value> FileEvalCache;
 #endif
     FileEvalCache fileEvalCache;
 
+    /**
+     * Associate source positions of certain AST nodes with their preceding doc comment, if they have one.
+     * Grouped by file.
+     */
+    std::map<SourcePath, DocCommentMap> positionToDocComment;
+
     LookupPath lookupPath;
 
     std::map<std::string, std::optional<std::string>> lookupPathResolved;
@@ -353,6 +363,7 @@ public:
     EvalState(
         const LookupPath & _lookupPath,
         ref<Store> store,
+        const fetchers::Settings & fetchSettings,
         const EvalSettings & settings,
         std::shared_ptr<Store> buildStore = nullptr);
     ~EvalState();
@@ -768,6 +779,8 @@ public:
         std::string_view pathArg,
         PosIdx pos);
 
+    DocComment getDocCommentForPos(PosIdx pos);
+
 private:
 
     /**

src/libexpr/get-drvs.cc

@@ -342,9 +342,9 @@ std::optional<PackageInfo> getDerivation(EvalState & state, Value & v,
 }
 
 
-static std::string addToPath(const std::string & s1, const std::string & s2)
+static std::string addToPath(const std::string & s1, std::string_view s2)
 {
-    return s1.empty() ? s2 : s1 + "." + s2;
+    return s1.empty() ? std::string(s2) : s1 + "." + s2;
 }
 
 

src/libexpr/lexer-helpers.cc

@@ -0,0 +1,30 @@
+#include "lexer-tab.hh"
+#include "lexer-helpers.hh"
+#include "parser-tab.hh"
+
+void nix::lexer::internal::initLoc(YYLTYPE * loc)
+{
+    loc->beginOffset = loc->endOffset = 0;
+}
+
+void nix::lexer::internal::adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len)
+{
+    loc->stash();
+
+    LexerState & lexerState = *yyget_extra(yyscanner);
+
+    if (lexerState.docCommentDistance == 1) {
+        // Preceding token was a doc comment.
+        ParserLocation doc;
+        doc.beginOffset = lexerState.lastDocCommentLoc.beginOffset;
+        ParserLocation docEnd;
+        docEnd.beginOffset = lexerState.lastDocCommentLoc.endOffset;
+        DocComment docComment{lexerState.at(doc), lexerState.at(docEnd)};
+        PosIdx locPos = lexerState.at(*loc);
+        lexerState.positionToDocComment.emplace(locPos, docComment);
+    }
+    lexerState.docCommentDistance++;
+
+    loc->beginOffset = loc->endOffset;
+    loc->endOffset += len;
+}

src/libexpr/lexer-helpers.hh

@@ -0,0 +1,9 @@
+#pragma once
+
+namespace nix::lexer::internal {
+
+void initLoc(YYLTYPE * loc);
+
+void adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len);
+
+} // namespace nix::lexer

src/libexpr/lexer.l

@@ -5,7 +5,7 @@
 %option stack
 %option nodefault
 %option nounput noyy_top_state
-
+%option extra-type="::nix::LexerState *"
 
 %s DEFAULT
 %x STRING
@@ -14,6 +14,10 @@
 %x INPATH_SLASH
 %x PATH_START
 
+%top {
+#include "parser-tab.hh" // YYSTYPE
+#include "parser-state.hh"
+}
 
 %{
 #ifdef __clang__
@@ -22,28 +26,19 @@
 
 #include "nixexpr.hh"
 #include "parser-tab.hh"
+#include "lexer-helpers.hh"
+
+namespace nix {
+  struct LexerState;
+}
 
 using namespace nix;
+using namespace nix::lexer::internal;
 
 namespace nix {
 
 #define CUR_POS state->at(*yylloc)
 
-static void initLoc(YYLTYPE * loc)
-{
-    loc->first_line = loc->last_line = 0;
-    loc->first_column = loc->last_column = 0;
-}
-
-static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
-{
-    loc->stash();
-
-    loc->first_column = loc->last_column;
-    loc->last_column += len;
-}
-
-
 // we make use of the fact that the parser receives a private copy of the input
 // string and can munge around in it.
 static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length)
@@ -79,7 +74,7 @@ static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length)
 #pragma GCC diagnostic ignored "-Wimplicit-fallthrough"
 
 #define YY_USER_INIT initLoc(yylloc)
-#define YY_USER_ACTION adjustLoc(yylloc, yytext, yyleng);
+#define YY_USER_ACTION adjustLoc(yyscanner, yylloc, yytext, yyleng);
 
 #define PUSH_STATE(state) yy_push_state(state, yyscanner)
 #define POP_STATE() yy_pop_state(yyscanner)
@@ -279,9 +274,32 @@ or          { return OR_KW; }
 {SPATH}     { yylval->path = {yytext, (size_t) yyleng}; return SPATH; }
 {URI}       { yylval->uri = {yytext, (size_t) yyleng}; return URI; }
 
-[ \t\r\n]+    /* eat up whitespace */
-\#[^\r\n]*    /* single-line comments */
-\/\*([^*]|\*+[^*/])*\*+\/  /* long comments */
+%{
+// Doc comment rule
+//
+//   \/\*\*                            /**
+//         [^/*]                       reject /**/ (empty comment) and /***
+//              ([^*]|\*+[^*/])*\*+\/  same as the long comment rule
+//              (             )*       zero or more non-ending sequences
+//                              \*     end(1)
+//                                 \/  end(2)
+%}
+\/\*\*[^/*]([^*]|\*+[^*/])*\*+\/  /* doc comments */ {
+    LexerState & lexerState = *yyget_extra(yyscanner);
+    lexerState.docCommentDistance = 0;
+    lexerState.lastDocCommentLoc.beginOffset = yylloc->beginOffset;
+    lexerState.lastDocCommentLoc.endOffset = yylloc->endOffset;
+}
+
+
+%{
+// The following rules have docCommentDistance--
+// This compensates for the docCommentDistance++ which happens by default to
+// make all the other rules invalidate the doc comment.
+%}
+[ \t\r\n]+    /* eat up whitespace */ { yyget_extra(yyscanner)->docCommentDistance--; }
+\#[^\r\n]*    /* single-line comments */ { yyget_extra(yyscanner)->docCommentDistance--; }
+\/\*([^*]|\*+[^*/])*\*+\/  /* long comments */ { yyget_extra(yyscanner)->docCommentDistance--; }
 
 {ANY}       {
               /* Don't return a negative number, as this will cause

src/libexpr/meson.build

@@ -139,6 +139,7 @@ sources = files(
   'function-trace.cc',
   'get-drvs.cc',
   'json-to-value.cc',
+  'lexer-helpers.cc',
   'nixexpr.cc',
   'paths.cc',
   'primops.cc',
@@ -165,6 +166,7 @@ headers = [config_h] + files(
   'gc-small-vector.hh',
   'get-drvs.hh',
   'json-to-value.hh',
+  # internal: 'lexer-helpers.hh',
   'nixexpr.hh',
   'parser-state.hh',
   'pos-idx.hh',

src/libexpr/nixexpr.cc

@@ -1,5 +1,4 @@
 #include "nixexpr.hh"
-#include "derivations.hh"
 #include "eval.hh"
 #include "symbol-table.hh"
 #include "util.hh"
@@ -8,6 +7,8 @@
 #include <cstdlib>
 #include <sstream>
 
+#include "strings-inline.hh"
+
 namespace nix {
 
 unsigned long Expr::nrExprs = 0;
@@ -582,6 +583,22 @@ std::string ExprLambda::showNamePos(const EvalState & state) const
     return fmt("%1% at %2%", id, state.positions[pos]);
 }
 
+void ExprLambda::setDocComment(DocComment docComment) {
+    // RFC 145 specifies that the innermost doc comment wins.
+    // See https://github.com/NixOS/rfcs/blob/master/rfcs/0145-doc-strings.md#ambiguous-placement
+    if (!this->docComment) {
+        this->docComment = docComment;
+
+        // Curried functions are defined by putting a function directly
+        // in the body of another function. To render docs for those, we
+        // need to propagate the doc comment to the innermost function.
+        //
+        // If we have our own comment, we've already propagated it, so this
+        // belongs in the same conditional.
+        body->setDocComment(docComment);
+    }
+};
+
 
 
 /* Position table. */
@@ -626,4 +643,22 @@ size_t SymbolTable::totalSize() const
     return n;
 }
 
+std::string DocComment::getInnerText(const PosTable & positions) const {
+    auto beginPos = positions[begin];
+    auto endPos = positions[end];
+    auto docCommentStr = beginPos.getSnippetUpTo(endPos).value_or("");
+
+    // Strip "/**" and "*/"
+    constexpr size_t prefixLen = 3;
+    constexpr size_t suffixLen = 2;
+    std::string docStr = docCommentStr.substr(prefixLen, docCommentStr.size() - prefixLen - suffixLen);
+    if (docStr.empty())
+        return {};
+    // Turn the now missing "/**" into indentation
+    docStr = "   " + docStr;
+    // Strip indentation (for the whole, potentially multi-line string)
+    docStr = stripIndentation(docStr);
+    return docStr;
+}
+
 }

src/libexpr/nixexpr.hh

@@ -6,21 +6,58 @@
 
 #include "value.hh"
 #include "symbol-table.hh"
-#include "error.hh"
-#include "position.hh"
 #include "eval-error.hh"
 #include "pos-idx.hh"
-#include "pos-table.hh"
 
 namespace nix {
 
-
-struct Env;
-struct Value;
 class EvalState;
+class PosTable;
+struct Env;
 struct ExprWith;
 struct StaticEnv;
+struct Value;
 
+/**
+ * A documentation comment, in the sense of [RFC 145](https://github.com/NixOS/rfcs/blob/master/rfcs/0145-doc-strings.md)
+ *
+ * Note that this does not implement the following:
+ *  - argument attribute names ("formals"): TBD
+ *  - argument names: these are internal to the function and their names may not be optimal for documentation
+ *  - function arity (degree of currying or number of ':'s):
+ *      - Functions returning partially applied functions have a higher arity
+ *        than can be determined locally and without evaluation.
+ *        We do not want to present false data.
+ *      - Some functions should be thought of as transformations of other
+ *        functions. For instance `overlay -> overlay -> overlay` is the simplest
+ *        way to understand `composeExtensions`, but its implementation looks like
+ *        `f: g: final: prev: <...>`. The parameters `final` and `prev` are part
+ *        of the overlay concept, while distracting from the function's purpose.
+ */
+struct DocComment {
+
+    /**
+     * Start of the comment, including the opening, ie `/` and `**`.
+     */
+    PosIdx begin;
+
+    /**
+     * Position right after the final asterisk and `/` that terminate the comment.
+     */
+    PosIdx end;
+
+    /**
+     * Whether the comment is set.
+     *
+     * A `DocComment` is small enough that it makes sense to pass by value, and
+     * therefore baking optionality into it is also useful, to avoiding the memory
+     * overhead of `std::optional`.
+     */
+    operator bool() const { return static_cast<bool>(begin); }
+
+    std::string getInnerText(const PosTable & positions) const;
+
+};
 
 /**
  * An attribute path is a sequence of attribute names.
@@ -57,6 +94,7 @@ struct Expr
     virtual void eval(EvalState & state, Env & env, Value & v);
     virtual Value * maybeThunk(EvalState & state, Env & env);
     virtual void setName(Symbol name);
+    virtual void setDocComment(DocComment docComment) { };
     virtual PosIdx getPos() const { return noPos; }
 };
 
@@ -159,6 +197,17 @@ struct ExprSelect : Expr
     ExprSelect(const PosIdx & pos, Expr * e, AttrPath attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(std::move(attrPath)) { };
     ExprSelect(const PosIdx & pos, Expr * e, Symbol name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); };
     PosIdx getPos() const override { return pos; }
+
+    /**
+     * Evaluate the `a.b.c` part of `a.b.c.d`. This exists mostly for the purpose of :doc in the repl.
+     *
+     * @param[out] v The attribute set that should contain the last attribute name (if it exists).
+     * @return The last attribute name in `attrPath`
+     * 
+     * @note This does *not* evaluate the final attribute, and does not fail if that's the only attribute that does not exist.
+     */
+    Symbol evalExceptFinalSelect(EvalState & state, Env & env, Value & attrs);
+
     COMMON_METHODS
 };
 
@@ -281,6 +330,8 @@ struct ExprLambda : Expr
     Symbol arg;
     Formals * formals;
     Expr * body;
+    DocComment docComment;
+
     ExprLambda(PosIdx pos, Symbol arg, Formals * formals, Expr * body)
         : pos(pos), arg(arg), formals(formals), body(body)
     {
@@ -293,6 +344,7 @@ struct ExprLambda : Expr
     std::string showNamePos(const EvalState & state) const;
     inline bool hasFormals() const { return formals != nullptr; }
     PosIdx getPos() const override { return pos; }
+    virtual void setDocComment(DocComment docComment) override;
     COMMON_METHODS
 };
 

src/libexpr/parser-state.hh

@@ -1,6 +1,8 @@
 #pragma once
 ///@file
 
+#include <limits>
+
 #include "eval.hh"
 
 namespace nix {
@@ -20,25 +22,59 @@ struct StringToken
 
 struct ParserLocation
 {
-    int first_line, first_column;
-    int last_line, last_column;
+    int beginOffset;
+    int endOffset;
 
     // backup to recover from yyless(0)
-    int stashed_first_column, stashed_last_column;
+    int stashedBeginOffset, stashedEndOffset;
 
     void stash() {
-        stashed_first_column = first_column;
-        stashed_last_column = last_column;
+        stashedBeginOffset = beginOffset;
+        stashedEndOffset = endOffset;
     }
 
     void unstash() {
-        first_column = stashed_first_column;
-        last_column = stashed_last_column;
+        beginOffset = stashedBeginOffset;
+        endOffset = stashedEndOffset;
     }
+
+    /** Latest doc comment position, or 0. */
+    int doc_comment_first_column, doc_comment_last_column;
+};
+
+struct LexerState
+{
+    /**
+     * Tracks the distance to the last doc comment, in terms of lexer tokens.
+     *
+     * The lexer sets this to 0 when reading a doc comment, and increments it
+     * for every matched rule; see `lexer-helpers.cc`.
+     * Whitespace and comment rules decrement the distance, so that they result
+     * in a net 0 change in distance.
+     */
+    int docCommentDistance = std::numeric_limits<int>::max();
+
+    /**
+     * The location of the last doc comment.
+     *
+     * (stashing fields are not used)
+     */
+    ParserLocation lastDocCommentLoc;
+
+    /**
+     * @brief Maps some positions to a DocComment, where the comment is relevant to the location.
+     */
+    std::map<PosIdx, DocComment> & positionToDocComment;
+
+    PosTable & positions;
+    PosTable::Origin origin;
+
+    PosIdx at(const ParserLocation & loc);
 };
 
 struct ParserState
 {
+    const LexerState & lexerState;
     SymbolTable & symbols;
     PosTable & positions;
     Expr * result;
@@ -270,9 +306,14 @@ inline Expr * ParserState::stripIndentation(const PosIdx pos,
     return new ExprConcatStrings(pos, true, es2);
 }
 
+inline PosIdx LexerState::at(const ParserLocation & loc)
+{
+    return positions.add(origin, loc.beginOffset);
+}
+
 inline PosIdx ParserState::at(const ParserLocation & loc)
 {
-    return positions.add(origin, loc.first_column);
+    return positions.add(origin, loc.beginOffset);
 }
 
 }

src/libexpr/parser.y

@@ -31,8 +31,25 @@
 #define YY_DECL int yylex \
     (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParserState * state)
 
+// For efficiency, we only track offsets; not line,column coordinates
+# define YYLLOC_DEFAULT(Current, Rhs, N)                                \
+    do                                                                  \
+      if (N)                                                            \
+        {                                                               \
+          (Current).beginOffset = YYRHSLOC (Rhs, 1).beginOffset;        \
+          (Current).endOffset  = YYRHSLOC (Rhs, N).endOffset;           \
+        }                                                               \
+      else                                                              \
+        {                                                               \
+          (Current).beginOffset = (Current).endOffset =                 \
+            YYRHSLOC (Rhs, 0).endOffset;                                \
+        }                                                               \
+    while (0)
+
 namespace nix {
 
+typedef std::map<PosIdx, DocComment> DocCommentMap;
+
 Expr * parseExprFromBuf(
     char * text,
     size_t length,
@@ -41,6 +58,7 @@ Expr * parseExprFromBuf(
     SymbolTable & symbols,
     const EvalSettings & settings,
     PosTable & positions,
+    DocCommentMap & docComments,
     const ref<SourceAccessor> rootFS,
     const Expr::AstSymbols & astSymbols);
 
@@ -65,8 +83,7 @@ using namespace nix;
 void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * error)
 {
     if (std::string_view(error).starts_with("syntax error, unexpected end of file")) {
-        loc->first_column = loc->last_column;
-        loc->first_line = loc->last_line;
+        loc->beginOffset = loc->endOffset;
     }
     throw ParseError({
         .msg = HintFmt(error),
@@ -74,6 +91,14 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char *
     });
 }
 
+#define SET_DOC_POS(lambda, pos) setDocPosition(state->lexerState, lambda, state->at(pos))
+static void setDocPosition(const LexerState & lexerState, ExprLambda * lambda, PosIdx start) {
+    auto it = lexerState.positionToDocComment.find(start);
+    if (it != lexerState.positionToDocComment.end()) {
+        lambda->setDocComment(it->second);
+    }
+}
+
 
 %}
 
@@ -119,6 +144,7 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char *
 %token IND_STRING_OPEN IND_STRING_CLOSE
 %token ELLIPSIS
 
+
 %right IMPL
 %left OR
 %left AND
@@ -140,18 +166,28 @@ expr: expr_function;
 
 expr_function
   : ID ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, state->symbols.create($1), 0, $3); }
+    { auto me = new ExprLambda(CUR_POS, state->symbols.create($1), 0, $3);
+      $$ = me;
+      SET_DOC_POS(me, @1);
+    }
   | '{' formals '}' ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, state->validateFormals($2), $5); }
+    { auto me = new ExprLambda(CUR_POS, state->validateFormals($2), $5);
+      $$ = me;
+      SET_DOC_POS(me, @1);
+    }
   | '{' formals '}' '@' ID ':' expr_function
     {
       auto arg = state->symbols.create($5);
-      $$ = new ExprLambda(CUR_POS, arg, state->validateFormals($2, CUR_POS, arg), $7);
+      auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($2, CUR_POS, arg), $7);
+      $$ = me;
+      SET_DOC_POS(me, @1);
     }
   | ID '@' '{' formals '}' ':' expr_function
     {
       auto arg = state->symbols.create($1);
-      $$ = new ExprLambda(CUR_POS, arg, state->validateFormals($4, CUR_POS, arg), $7);
+      auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($4, CUR_POS, arg), $7);
+      $$ = me;
+      SET_DOC_POS(me, @1);
     }
   | ASSERT expr ';' expr_function
     { $$ = new ExprAssert(CUR_POS, $2, $4); }
@@ -312,7 +348,22 @@ ind_string_parts
   ;
 
 binds
-  : binds attrpath '=' expr ';' { $$ = $1; state->addAttr($$, std::move(*$2), $4, state->at(@2)); delete $2; }
+  : binds attrpath '=' expr ';' {
+      $$ = $1;
+
+      auto pos = state->at(@2);
+      auto exprPos = state->at(@4);
+      {
+        auto it = state->lexerState.positionToDocComment.find(pos);
+        if (it != state->lexerState.positionToDocComment.end()) {
+          $4->setDocComment(it->second);
+          state->lexerState.positionToDocComment.emplace(exprPos, it->second);
+        }
+      }
+
+      state->addAttr($$, std::move(*$2), $4, pos);
+      delete $2;
+    }
   | binds INHERIT attrs ';'
     { $$ = $1;
       for (auto & [i, iPos] : *$3) {
@@ -431,21 +482,28 @@ Expr * parseExprFromBuf(
     SymbolTable & symbols,
     const EvalSettings & settings,
     PosTable & positions,
+    DocCommentMap & docComments,
     const ref<SourceAccessor> rootFS,
     const Expr::AstSymbols & astSymbols)
 {
     yyscan_t scanner;
+    LexerState lexerState {
+        .positionToDocComment = docComments,
+        .positions = positions,
+        .origin = positions.addOrigin(origin, length),
+    };
     ParserState state {
+        .lexerState = lexerState,
         .symbols = symbols,
         .positions = positions,
         .basePath = basePath,
-        .origin = positions.addOrigin(origin, length),
+        .origin = lexerState.origin,
         .rootFS = rootFS,
         .s = astSymbols,
         .settings = settings,
     };
 
-    yylex_init(&scanner);
+    yylex_init_extra(&lexerState, &scanner);
     Finally _destroy([&] { yylex_destroy(scanner); });
 
     yy_scan_buffer(text, length, scanner);

src/libexpr/pos-idx.hh

@@ -28,20 +28,15 @@ public:
         return id > 0;
     }
 
-    bool operator<(const PosIdx other) const
+    auto operator<=>(const PosIdx other) const
     {
-        return id < other.id;
+        return id <=> other.id;
     }
 
     bool operator==(const PosIdx other) const
     {
         return id == other.id;
     }
-
-    bool operator!=(const PosIdx other) const
-    {
-        return id != other.id;
-    }
 };
 
 inline PosIdx noPos = {};

src/libexpr/pos-table.hh

@@ -1,10 +1,8 @@
 #pragma once
 
-#include <cinttypes>
-#include <numeric>
+#include <cstdint>
 #include <vector>
 
-#include "chunked-vector.hh"
 #include "pos-idx.hh"
 #include "position.hh"
 #include "sync.hh"

src/libexpr/primops.cc

@@ -1,4 +1,3 @@
-#include "archive.hh"
 #include "derivations.hh"
 #include "downstream-placeholder.hh"
 #include "eval-inline.hh"
@@ -1225,7 +1224,7 @@ static void derivationStrictInternal(
 
     for (auto & i : attrs->lexicographicOrder(state.symbols)) {
         if (i->name == state.sIgnoreNulls) continue;
-        const std::string & key = state.symbols[i->name];
+        auto key = state.symbols[i->name];
         vomit("processing attribute '%1%'", key);
 
         auto handleHashMode = [&](const std::string_view s) {
@@ -1309,7 +1308,7 @@ static void derivationStrictInternal(
 
                     if (i->name == state.sStructuredAttrs) continue;
 
-                    (*jsonObject)[key] = printValueAsJSON(state, true, *i->value, pos, context);
+                    jsonObject->emplace(key, printValueAsJSON(state, true, *i->value, pos, context));
 
                     if (i->name == state.sBuilder)
                         drv.builder = state.forceString(*i->value, context, pos, context_below);

src/libexpr/primops/fetchMercurial.cc

@@ -62,7 +62,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a
     attrs.insert_or_assign("name", std::string(name));
     if (ref) attrs.insert_or_assign("ref", *ref);
     if (rev) attrs.insert_or_assign("rev", rev->gitRev());
-    auto input = fetchers::Input::fromAttrs(std::move(attrs));
+    auto input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs));
 
     auto [storePath, input2] = input.fetchToStore(state.store);
 

src/libexpr/primops/fetchTree.cc

@@ -85,7 +85,7 @@ static void fetchTree(
     Value & v,
     const FetchTreeParams & params = FetchTreeParams{}
 ) {
-    fetchers::Input input;
+    fetchers::Input input { state.fetchSettings };
     NixStringContext context;
     std::optional<std::string> type;
     if (params.isFetchGit) type = "git";
@@ -148,7 +148,7 @@ static void fetchTree(
                     "attribute 'name' isn’t supported in call to 'fetchTree'"
                 ).atPos(pos).debugThrow();
 
-        input = fetchers::Input::fromAttrs(std::move(attrs));
+        input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs));
     } else {
         auto url = state.coerceToString(pos, *args[0], context,
                 "while evaluating the first argument passed to the fetcher",
@@ -161,9 +161,9 @@ static void fetchTree(
             if (!attrs.contains("exportIgnore") && (!attrs.contains("submodules") || !*fetchers::maybeGetBoolAttr(attrs, "submodules"))) {
                 attrs.emplace("exportIgnore", Explicit<bool>{true});
             }
-            input = fetchers::Input::fromAttrs(std::move(attrs));
+            input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs));
         } else {
-            input = fetchers::Input::fromURL(url);
+            input = fetchers::Input::fromURL(state.fetchSettings, url);
         }
     }
 

src/libexpr/symbol-table.hh

@@ -30,9 +30,9 @@ public:
         return *s == s2;
     }
 
-    operator const std::string & () const
+    const char * c_str() const
     {
-        return *s;
+        return s->c_str();
     }
 
     operator const std::string_view () const
@@ -41,6 +41,11 @@ public:
     }
 
     friend std::ostream & operator <<(std::ostream & os, const SymbolStr & symbol);
+
+    bool empty() const
+    {
+        return s->empty();
+    }
 };
 
 /**
@@ -62,9 +67,8 @@ public:
 
     explicit operator bool() const { return id > 0; }
 
-    bool operator<(const Symbol other) const { return id < other.id; }
+    auto operator<=>(const Symbol other) const { return id <=> other.id; }
     bool operator==(const Symbol other) const { return id == other.id; }
-    bool operator!=(const Symbol other) const { return id != other.id; }
 };
 
 /**

src/libexpr/value-to-json.cc

@@ -58,7 +58,7 @@ json printValueAsJSON(EvalState & state, bool strict,
                 out = json::object();
                 for (auto & a : v.attrs()->lexicographicOrder(state.symbols)) {
                     try {
-                        out[state.symbols[a->name]] = printValueAsJSON(state, strict, *a->value, a->pos, context, copyToStore);
+                        out.emplace(state.symbols[a->name], printValueAsJSON(state, strict, *a->value, a->pos, context, copyToStore));
                     } catch (Error & e) {
                         e.addTrace(state.positions[a->pos],
                             HintFmt("while evaluating attribute '%1%'", state.symbols[a->name]));

src/libexpr/value-to-xml.cc

@@ -9,7 +9,7 @@
 namespace nix {
 
 
-static XMLAttrs singletonAttrs(const std::string & name, const std::string & value)
+static XMLAttrs singletonAttrs(const std::string & name, std::string_view value)
 {
     XMLAttrs attrs;
     attrs[name] = value;

src/libexpr/value.hh

@@ -2,7 +2,6 @@
 ///@file
 
 #include <cassert>
-#include <climits>
 #include <span>
 
 #include "symbol-table.hh"
@@ -112,7 +111,7 @@ class ExternalValueBase
      * Compare to another value of the same type. Defaults to uncomparable,
      * i.e. always false.
      */
-    virtual bool operator ==(const ExternalValueBase & b) const;
+    virtual bool operator ==(const ExternalValueBase & b) const noexcept;
 
     /**
      * Print the value as JSON. Defaults to unconvertable, i.e. throws an error

src/libfetchers/fetch-settings.cc

@@ -1,14 +1,9 @@
 #include "fetch-settings.hh"
-#include "config-global.hh"
 
-namespace nix {
+namespace nix::fetchers {
 
-FetchSettings::FetchSettings()
+Settings::Settings()
 {
 }
 
-FetchSettings fetchSettings;
-
-static GlobalConfig::Register rFetchSettings(&fetchSettings);
-
 }

src/libfetchers/fetch-settings.hh

@@ -9,11 +9,11 @@
 
 #include <sys/types.h>
 
-namespace nix {
+namespace nix::fetchers {
 
-struct FetchSettings : public Config
+struct Settings : public Config
 {
-    FetchSettings();
+    Settings();
 
     Setting<StringMap> accessTokens{this, {}, "access-tokens",
         R"(
@@ -84,9 +84,14 @@ struct FetchSettings : public Config
           `narHash` attribute is specified,
           e.g. `github:NixOS/patchelf/7c2f768bf9601268a4e71c2ebe91e2011918a70f?narHash=sha256-PPXqKY2hJng4DBVE0I4xshv/vGLUskL7jl53roB8UdU%3D`.
         )"};
+
+    Setting<std::string> flakeRegistry{this, "https://channels.nixos.org/flake-registry.json", "flake-registry",
+        R"(
+          Path or URI of the global flake registry.
+
+          When empty, disables the global flake registry.
+        )",
+        {}, true};
 };
 
-// FIXME: don't use a global variable.
-extern FetchSettings fetchSettings;
-
 }

src/libfetchers/fetchers.cc

@@ -35,9 +35,11 @@ nlohmann::json dumpRegisterInputSchemeInfo() {
     return res;
 }
 
-Input Input::fromURL(const std::string & url, bool requireTree)
+Input Input::fromURL(
+    const Settings & settings,
+    const std::string & url, bool requireTree)
 {
-    return fromURL(parseURL(url), requireTree);
+    return fromURL(settings, parseURL(url), requireTree);
 }
 
 static void fixupInput(Input & input)
@@ -49,10 +51,12 @@ static void fixupInput(Input & input)
     input.getLastModified();
 }
 
-Input Input::fromURL(const ParsedURL & url, bool requireTree)
+Input Input::fromURL(
+    const Settings & settings,
+    const ParsedURL & url, bool requireTree)
 {
     for (auto & [_, inputScheme] : *inputSchemes) {
-        auto res = inputScheme->inputFromURL(url, requireTree);
+        auto res = inputScheme->inputFromURL(settings, url, requireTree);
         if (res) {
             experimentalFeatureSettings.require(inputScheme->experimentalFeature());
             res->scheme = inputScheme;
@@ -64,7 +68,7 @@ Input Input::fromURL(const ParsedURL & url, bool requireTree)
     throw Error("input '%s' is unsupported", url.url);
 }
 
-Input Input::fromAttrs(Attrs && attrs)
+Input Input::fromAttrs(const Settings & settings, Attrs && attrs)
 {
     auto schemeName = ({
         auto schemeNameOpt = maybeGetStrAttr(attrs, "type");
@@ -78,7 +82,7 @@ Input Input::fromAttrs(Attrs && attrs)
         // but not all of them. Doing this is to support those other
         // operations which are supposed to be robust on
         // unknown/uninterpretable inputs.
-        Input input;
+        Input input { settings };
         input.attrs = attrs;
         fixupInput(input);
         return input;
@@ -99,7 +103,7 @@ Input Input::fromAttrs(Attrs && attrs)
         if (name != "type" && allowedAttrs.count(name) == 0)
             throw Error("input attribute '%s' not supported by scheme '%s'", name, schemeName);
 
-    auto res = inputScheme->inputFromAttrs(attrs);
+    auto res = inputScheme->inputFromAttrs(settings, attrs);
     if (!res) return raw();
     res->scheme = inputScheme;
     fixupInput(*res);
@@ -146,7 +150,7 @@ Attrs Input::toAttrs() const
     return attrs;
 }
 
-bool Input::operator ==(const Input & other) const
+bool Input::operator ==(const Input & other) const noexcept
 {
     return attrs == other.attrs;
 }

src/libfetchers/fetchers.hh

@@ -11,12 +11,16 @@
 #include <memory>
 #include <nlohmann/json_fwd.hpp>
 
+#include "ref.hh"
+
 namespace nix { class Store; class StorePath; struct SourceAccessor; }
 
 namespace nix::fetchers {
 
 struct InputScheme;
 
+struct Settings;
+
 /**
  * The `Input` object is generated by a specific fetcher, based on
  * user-supplied information, and contains
@@ -28,6 +32,12 @@ struct Input
 {
     friend struct InputScheme;
 
+    const Settings * settings;
+
+    Input(const Settings & settings)
+        : settings{&settings}
+    { }
+
     std::shared_ptr<InputScheme> scheme; // note: can be null
     Attrs attrs;
 
@@ -42,16 +52,22 @@ public:
      *
      * The URL indicate which sort of fetcher, and provides information to that fetcher.
      */
-    static Input fromURL(const std::string & url, bool requireTree = true);
+    static Input fromURL(
+        const Settings & settings,
+        const std::string & url, bool requireTree = true);
 
-    static Input fromURL(const ParsedURL & url, bool requireTree = true);
+    static Input fromURL(
+        const Settings & settings,
+        const ParsedURL & url, bool requireTree = true);
 
     /**
      * Create an `Input` from a an `Attrs`.
      *
      * The URL indicate which sort of fetcher, and provides information to that fetcher.
      */
-    static Input fromAttrs(Attrs && attrs);
+    static Input fromAttrs(
+        const Settings & settings,
+        Attrs && attrs);
 
     ParsedURL toURL() const;
 
@@ -73,7 +89,7 @@ public:
      */
     bool isLocked() const;
 
-    bool operator ==(const Input & other) const;
+    bool operator ==(const Input & other) const noexcept;
 
     bool contains(const Input & other) const;
 
@@ -146,9 +162,13 @@ struct InputScheme
     virtual ~InputScheme()
     { }
 
-    virtual std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const = 0;
+    virtual std::optional<Input> inputFromURL(
+        const Settings & settings,
+        const ParsedURL & url, bool requireTree) const = 0;
 
-    virtual std::optional<Input> inputFromAttrs(const Attrs & attrs) const = 0;
+    virtual std::optional<Input> inputFromAttrs(
+        const Settings & settings,
+        const Attrs & attrs) const = 0;
 
     /**
      * What is the name of the scheme?

src/libfetchers/git-utils.cc

@@ -115,10 +115,10 @@ git_oid hashToOID(const Hash & hash)
     return oid;
 }
 
-Object lookupObject(git_repository * repo, const git_oid & oid)
+Object lookupObject(git_repository * repo, const git_oid & oid, git_object_t type = GIT_OBJECT_ANY)
 {
     Object obj;
-    if (git_object_lookup(Setter(obj), repo, &oid, GIT_OBJECT_ANY)) {
+    if (git_object_lookup(Setter(obj), repo, &oid, type)) {
         auto err = git_error_last();
         throw Error("getting Git object '%s': %s", oid, err->message);
     }
@@ -909,6 +909,61 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
         addToTree(*pathComponents.rbegin(), oid, GIT_FILEMODE_LINK);
     }
 
+    void createHardlink(const CanonPath & path, const CanonPath & target) override
+    {
+        std::vector<std::string> pathComponents;
+        for (auto & c : path)
+            pathComponents.emplace_back(c);
+
+        if (!prepareDirs(pathComponents, false)) return;
+
+        // We can't just look up the path from the start of the root, since
+        // some parent directories may not have finished yet, so we compute
+        // a relative path that helps us find the right git_tree_builder or object.
+        auto relTarget = CanonPath(path).parent()->makeRelative(target);
+
+        auto dir = pendingDirs.rbegin();
+
+        // For each ../ component at the start, go up one directory.
+        // CanonPath::makeRelative() always puts all .. elements at the start,
+        // so they're all handled by this loop:
+        std::string_view relTargetLeft(relTarget);
+        while (hasPrefix(relTargetLeft, "../")) {
+            if (dir == pendingDirs.rend())
+                throw Error("invalid hard link target '%s' for path '%s'", target, path);
+            ++dir;
+            relTargetLeft = relTargetLeft.substr(3);
+        }
+        if (dir == pendingDirs.rend())
+            throw Error("invalid hard link target '%s' for path '%s'", target, path);
+
+        // Look up the remainder of the target, starting at the
+        // top-most `git_treebuilder`.
+        std::variant<git_treebuilder *, git_oid> curDir{dir->builder.get()};
+        Object tree; // needed to keep `entry` alive
+        const git_tree_entry * entry = nullptr;
+
+        for (auto & c : CanonPath(relTargetLeft)) {
+            if (auto builder = std::get_if<git_treebuilder *>(&curDir)) {
+                assert(*builder);
+                if (!(entry = git_treebuilder_get(*builder, std::string(c).c_str())))
+                    throw Error("cannot find hard link target '%s' for path '%s'", target, path);
+                curDir = *git_tree_entry_id(entry);
+            } else if (auto oid = std::get_if<git_oid>(&curDir)) {
+                tree = lookupObject(*repo, *oid, GIT_OBJECT_TREE);
+                if (!(entry = git_tree_entry_byname((const git_tree *) &*tree, std::string(c).c_str())))
+                    throw Error("cannot find hard link target '%s' for path '%s'", target, path);
+                curDir = *git_tree_entry_id(entry);
+            }
+        }
+
+        assert(entry);
+
+        addToTree(*pathComponents.rbegin(),
+            *git_tree_entry_id(entry),
+            git_tree_entry_filemode(entry));
+    }
+
     Hash sync() override {
         updateBuilders({});
 

src/libfetchers/git-utils.hh

@@ -7,7 +7,7 @@ namespace nix {
 
 namespace fetchers { struct PublicKey; }
 
-struct GitFileSystemObjectSink : FileSystemObjectSink
+struct GitFileSystemObjectSink : ExtendedFileSystemObjectSink
 {
     /**
      * Flush builder and return a final Git hash.

src/libfetchers/git.cc

@@ -164,7 +164,9 @@ static const Hash nullRev{HashAlgorithm::SHA1};
 
 struct GitInputScheme : InputScheme
 {
-    std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override
+    std::optional<Input> inputFromURL(
+        const Settings & settings,
+        const ParsedURL & url, bool requireTree) const override
     {
         if (url.scheme != "git" &&
             url.scheme != "git+http" &&
@@ -190,7 +192,7 @@ struct GitInputScheme : InputScheme
 
         attrs.emplace("url", url2.to_string());
 
-        return inputFromAttrs(attrs);
+        return inputFromAttrs(settings, attrs);
     }
 
 
@@ -222,7 +224,9 @@ struct GitInputScheme : InputScheme
         };
     }
 
-    std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
+    std::optional<Input> inputFromAttrs(
+        const Settings & settings,
+        const Attrs & attrs) const override
     {
         for (auto & [name, _] : attrs)
             if (name == "verifyCommit"
@@ -238,7 +242,7 @@ struct GitInputScheme : InputScheme
                 throw BadURL("invalid Git branch/tag name '%s'", *ref);
         }
 
-        Input input;
+        Input input{settings};
         input.attrs = attrs;
         auto url = fixGitURL(getStrAttr(attrs, "url"));
         parseURL(url);
@@ -366,13 +370,13 @@ struct GitInputScheme : InputScheme
         /* URL of the repo, or its path if isLocal. Never a `file` URL. */
         std::string url;
 
-        void warnDirty() const
+        void warnDirty(const Settings & settings) const
         {
             if (workdirInfo.isDirty) {
-                if (!fetchSettings.allowDirty)
+                if (!settings.allowDirty)
                     throw Error("Git tree '%s' is dirty", url);
 
-                if (fetchSettings.warnDirty)
+                if (settings.warnDirty)
                     warn("Git tree '%s' is dirty", url);
             }
         }
@@ -653,7 +657,7 @@ struct GitInputScheme : InputScheme
                 attrs.insert_or_assign("exportIgnore", Explicit<bool>{ exportIgnore });
                 attrs.insert_or_assign("submodules", Explicit<bool>{ true });
                 attrs.insert_or_assign("allRefs", Explicit<bool>{ true });
-                auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs));
+                auto submoduleInput = fetchers::Input::fromAttrs(*input.settings, std::move(attrs));
                 auto [submoduleAccessor, submoduleInput2] =
                     submoduleInput.getAccessor(store);
                 submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»");
@@ -711,7 +715,7 @@ struct GitInputScheme : InputScheme
                 // TODO: fall back to getAccessorFromCommit-like fetch when submodules aren't checked out
                 // attrs.insert_or_assign("allRefs", Explicit<bool>{ true });
 
-                auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs));
+                auto submoduleInput = fetchers::Input::fromAttrs(*input.settings, std::move(attrs));
                 auto [submoduleAccessor, submoduleInput2] =
                     submoduleInput.getAccessor(store);
                 submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»");
@@ -743,7 +747,7 @@ struct GitInputScheme : InputScheme
 
             verifyCommit(input, repo);
         } else {
-            repoInfo.warnDirty();
+            repoInfo.warnDirty(*input.settings);
 
             if (repoInfo.workdirInfo.headRev) {
                 input.attrs.insert_or_assign("dirtyRev",

src/libfetchers/github.cc

@@ -31,7 +31,9 @@ struct GitArchiveInputScheme : InputScheme
 {
     virtual std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const = 0;
 
-    std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override
+    std::optional<Input> inputFromURL(
+        const fetchers::Settings & settings,
+        const ParsedURL & url, bool requireTree) const override
     {
         if (url.scheme != schemeName()) return {};
 
@@ -90,7 +92,7 @@ struct GitArchiveInputScheme : InputScheme
         if (ref && rev)
             throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url.url, *ref, rev->gitRev());
 
-        Input input;
+        Input input{settings};
         input.attrs.insert_or_assign("type", std::string { schemeName() });
         input.attrs.insert_or_assign("owner", path[0]);
         input.attrs.insert_or_assign("repo", path[1]);
@@ -119,12 +121,14 @@ struct GitArchiveInputScheme : InputScheme
         };
     }
 
-    std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
+    std::optional<Input> inputFromAttrs(
+        const fetchers::Settings & settings,
+        const Attrs & attrs) const override
     {
         getStrAttr(attrs, "owner");
         getStrAttr(attrs, "repo");
 
-        Input input;
+        Input input{settings};
         input.attrs = attrs;
         return input;
     }
@@ -168,18 +172,20 @@ struct GitArchiveInputScheme : InputScheme
         return input;
     }
 
-    std::optional<std::string> getAccessToken(const std::string & host) const
+    std::optional<std::string> getAccessToken(const fetchers::Settings & settings, const std::string & host) const
     {
-        auto tokens = fetchSettings.accessTokens.get();
+        auto tokens = settings.accessTokens.get();
         if (auto token = get(tokens, host))
             return *token;
         return {};
     }
 
-    Headers makeHeadersWithAuthTokens(const std::string & host) const
+    Headers makeHeadersWithAuthTokens(
+        const fetchers::Settings & settings,
+        const std::string & host) const
     {
         Headers headers;
-        auto accessToken = getAccessToken(host);
+        auto accessToken = getAccessToken(settings, host);
         if (accessToken) {
             auto hdr = accessHeaderFromToken(*accessToken);
             if (hdr)
@@ -295,7 +301,7 @@ struct GitArchiveInputScheme : InputScheme
            locking. FIXME: in the future, we may want to require a Git
            tree hash instead of a NAR hash. */
         return input.getRev().has_value()
-            && (fetchSettings.trustTarballsFromGitForges ||
+            && (input.settings->trustTarballsFromGitForges ||
                 input.getNarHash().has_value());
     }
 
@@ -347,7 +353,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
             : "https://%s/api/v3/repos/%s/%s/commits/%s",
             host, getOwner(input), getRepo(input), *input.getRef());
 
-        Headers headers = makeHeadersWithAuthTokens(host);
+        Headers headers = makeHeadersWithAuthTokens(*input.settings, host);
 
         auto json = nlohmann::json::parse(
             readFile(
@@ -364,7 +370,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
     {
         auto host = getHost(input);
 
-        Headers headers = makeHeadersWithAuthTokens(host);
+        Headers headers = makeHeadersWithAuthTokens(*input.settings, host);
 
         // If we have no auth headers then we default to the public archive
         // urls so we do not run into rate limits.
@@ -384,7 +390,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
     void clone(const Input & input, const Path & destDir) const override
     {
         auto host = getHost(input);
-        Input::fromURL(fmt("git+https://%s/%s/%s.git",
+        Input::fromURL(*input.settings, fmt("git+https://%s/%s/%s.git",
                 host, getOwner(input), getRepo(input)))
             .applyOverrides(input.getRef(), input.getRev())
             .clone(destDir);
@@ -421,7 +427,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
         auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s",
             host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
 
-        Headers headers = makeHeadersWithAuthTokens(host);
+        Headers headers = makeHeadersWithAuthTokens(*input.settings, host);
 
         auto json = nlohmann::json::parse(
             readFile(
@@ -451,7 +457,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
             host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
             input.getRev()->to_string(HashFormat::Base16, false));
 
-        Headers headers = makeHeadersWithAuthTokens(host);
+        Headers headers = makeHeadersWithAuthTokens(*input.settings, host);
         return DownloadUrl { url, headers };
     }
 
@@ -459,7 +465,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
     {
         auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
         // FIXME: get username somewhere
-        Input::fromURL(fmt("git+https://%s/%s/%s.git",
+        Input::fromURL(*input.settings, fmt("git+https://%s/%s/%s.git",
                 host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
             .applyOverrides(input.getRef(), input.getRev())
             .clone(destDir);
@@ -491,7 +497,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
         auto base_url = fmt("https://%s/%s/%s",
             host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"));
 
-        Headers headers = makeHeadersWithAuthTokens(host);
+        Headers headers = makeHeadersWithAuthTokens(*input.settings, host);
 
         std::string refUri;
         if (ref == "HEAD") {
@@ -538,14 +544,14 @@ struct SourceHutInputScheme : GitArchiveInputScheme
             host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
             input.getRev()->to_string(HashFormat::Base16, false));
 
-        Headers headers = makeHeadersWithAuthTokens(host);
+        Headers headers = makeHeadersWithAuthTokens(*input.settings, host);
         return DownloadUrl { url, headers };
     }
 
     void clone(const Input & input, const Path & destDir) const override
     {
         auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht");
-        Input::fromURL(fmt("git+https://%s/%s/%s",
+        Input::fromURL(*input.settings, fmt("git+https://%s/%s/%s",
                 host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
             .applyOverrides(input.getRef(), input.getRev())
             .clone(destDir);

src/libfetchers/indirect.cc

@@ -8,7 +8,9 @@ std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
 
 struct IndirectInputScheme : InputScheme
 {
-    std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override
+    std::optional<Input> inputFromURL(
+        const Settings & settings,
+        const ParsedURL & url, bool requireTree) const override
     {
         if (url.scheme != "flake") return {};
 
@@ -41,7 +43,7 @@ struct IndirectInputScheme : InputScheme
 
         // FIXME: forbid query params?
 
-        Input input;
+        Input input{settings};
         input.attrs.insert_or_assign("type", "indirect");
         input.attrs.insert_or_assign("id", id);
         if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
@@ -65,13 +67,15 @@ struct IndirectInputScheme : InputScheme
         };
     }
 
-    std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
+    std::optional<Input> inputFromAttrs(
+        const Settings & settings,
+        const Attrs & attrs) const override
     {
         auto id = getStrAttr(attrs, "id");
         if (!std::regex_match(id, flakeRegex))
             throw BadURL("'%s' is not a valid flake ID", id);
 
-        Input input;
+        Input input{settings};
         input.attrs = attrs;
         return input;
     }

src/libfetchers/mercurial.cc

@@ -45,7 +45,9 @@ static std::string runHg(const Strings & args, const std::optional<std::string>
 
 struct MercurialInputScheme : InputScheme
 {
-    std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override
+    std::optional<Input> inputFromURL(
+        const Settings & settings,
+        const ParsedURL & url, bool requireTree) const override
     {
         if (url.scheme != "hg+http" &&
             url.scheme != "hg+https" &&
@@ -68,7 +70,7 @@ struct MercurialInputScheme : InputScheme
 
         attrs.emplace("url", url2.to_string());
 
-        return inputFromAttrs(attrs);
+        return inputFromAttrs(settings, attrs);
     }
 
     std::string_view schemeName() const override
@@ -88,7 +90,9 @@ struct MercurialInputScheme : InputScheme
         };
     }
 
-    std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
+    std::optional<Input> inputFromAttrs(
+        const Settings & settings,
+        const Attrs & attrs) const override
     {
         parseURL(getStrAttr(attrs, "url"));
 
@@ -97,7 +101,7 @@ struct MercurialInputScheme : InputScheme
                 throw BadURL("invalid Mercurial branch/tag name '%s'", *ref);
         }
 
-        Input input;
+        Input input{settings};
         input.attrs = attrs;
         return input;
     }
@@ -182,10 +186,10 @@ struct MercurialInputScheme : InputScheme
                 /* This is an unclean working tree. So copy all tracked
                    files. */
 
-                if (!fetchSettings.allowDirty)
+                if (!input.settings->allowDirty)
                     throw Error("Mercurial tree '%s' is unclean", actualUrl);
 
-                if (fetchSettings.warnDirty)
+                if (input.settings->warnDirty)
                     warn("Mercurial tree '%s' is unclean", actualUrl);
 
                 input.attrs.insert_or_assign("ref", chomp(runHg({ "branch", "-R", actualUrl })));

src/libfetchers/path.cc

@@ -7,14 +7,16 @@ namespace nix::fetchers {
 
 struct PathInputScheme : InputScheme
 {
-    std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override
+    std::optional<Input> inputFromURL(
+        const Settings & settings,
+        const ParsedURL & url, bool requireTree) const override
     {
         if (url.scheme != "path") return {};
 
         if (url.authority && *url.authority != "")
             throw Error("path URL '%s' should not have an authority ('%s')", url.url, *url.authority);
 
-        Input input;
+        Input input{settings};
         input.attrs.insert_or_assign("type", "path");
         input.attrs.insert_or_assign("path", url.path);
 
@@ -54,11 +56,13 @@ struct PathInputScheme : InputScheme
         };
     }
 
-    std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
+    std::optional<Input> inputFromAttrs(
+        const Settings & settings,
+        const Attrs & attrs) const override
     {
         getStrAttr(attrs, "path");
 
-        Input input;
+        Input input{settings};
         input.attrs = attrs;
         return input;
     }

src/libfetchers/registry.cc

@@ -1,7 +1,7 @@
+#include "fetch-settings.hh"
 #include "registry.hh"
 #include "tarball.hh"
 #include "users.hh"
-#include "config-global.hh"
 #include "globals.hh"
 #include "store-api.hh"
 #include "local-fs-store.hh"
@@ -11,12 +11,13 @@
 namespace nix::fetchers {
 
 std::shared_ptr<Registry> Registry::read(
+    const Settings & settings,
     const Path & path, RegistryType type)
 {
-    auto registry = std::make_shared<Registry>(type);
+    auto registry = std::make_shared<Registry>(settings, type);
 
     if (!pathExists(path))
-        return std::make_shared<Registry>(type);
+        return std::make_shared<Registry>(settings, type);
 
     try {
 
@@ -36,8 +37,8 @@ std::shared_ptr<Registry> Registry::read(
                 auto exact = i.find("exact");
                 registry->entries.push_back(
                     Entry {
-                        .from = Input::fromAttrs(jsonToAttrs(i["from"])),
-                        .to = Input::fromAttrs(std::move(toAttrs)),
+                        .from = Input::fromAttrs(settings, jsonToAttrs(i["from"])),
+                        .to = Input::fromAttrs(settings, std::move(toAttrs)),
                         .extraAttrs = extraAttrs,
                         .exact = exact != i.end() && exact.value()
                     });
@@ -106,10 +107,10 @@ static Path getSystemRegistryPath()
     return settings.nixConfDir + "/registry.json";
 }
 
-static std::shared_ptr<Registry> getSystemRegistry()
+static std::shared_ptr<Registry> getSystemRegistry(const Settings & settings)
 {
     static auto systemRegistry =
-        Registry::read(getSystemRegistryPath(), Registry::System);
+        Registry::read(settings, getSystemRegistryPath(), Registry::System);
     return systemRegistry;
 }
 
@@ -118,25 +119,24 @@ Path getUserRegistryPath()
     return getConfigDir() + "/nix/registry.json";
 }
 
-std::shared_ptr<Registry> getUserRegistry()
+std::shared_ptr<Registry> getUserRegistry(const Settings & settings)
 {
     static auto userRegistry =
-        Registry::read(getUserRegistryPath(), Registry::User);
+        Registry::read(settings, getUserRegistryPath(), Registry::User);
     return userRegistry;
 }
 
-std::shared_ptr<Registry> getCustomRegistry(const Path & p)
+std::shared_ptr<Registry> getCustomRegistry(const Settings & settings, const Path & p)
 {
     static auto customRegistry =
-        Registry::read(p, Registry::Custom);
+        Registry::read(settings, p, Registry::Custom);
     return customRegistry;
 }
 
-static std::shared_ptr<Registry> flagRegistry =
-    std::make_shared<Registry>(Registry::Flag);
-
-std::shared_ptr<Registry> getFlagRegistry()
+std::shared_ptr<Registry> getFlagRegistry(const Settings & settings)
 {
+    static auto flagRegistry =
+        std::make_shared<Registry>(settings, Registry::Flag);
     return flagRegistry;
 }
 
@@ -145,30 +145,15 @@ void overrideRegistry(
     const Input & to,
     const Attrs & extraAttrs)
 {
-    flagRegistry->add(from, to, extraAttrs);
+    getFlagRegistry(*from.settings)->add(from, to, extraAttrs);
 }
 
-struct RegistrySettings : Config
-{
-    Setting<std::string> flakeRegistry{this, "https://channels.nixos.org/flake-registry.json", "flake-registry",
-        R"(
-          Path or URI of the global flake registry.
-
-          When empty, disables the global flake registry.
-        )",
-        {}, true};
-};
-
-RegistrySettings registrySettings;
-
-static GlobalConfig::Register rRegistrySettings(&registrySettings);
-
-static std::shared_ptr<Registry> getGlobalRegistry(ref<Store> store)
+static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, ref<Store> store)
 {
     static auto reg = [&]() {
-        auto path = registrySettings.flakeRegistry.get();
+        auto path = settings.flakeRegistry.get();
         if (path == "") {
-            return std::make_shared<Registry>(Registry::Global); // empty registry
+            return std::make_shared<Registry>(settings, Registry::Global); // empty registry
         }
 
         if (!hasPrefix(path, "/")) {
@@ -178,19 +163,19 @@ static std::shared_ptr<Registry> getGlobalRegistry(ref<Store> store)
             path = store->toRealPath(storePath);
         }
 
-        return Registry::read(path, Registry::Global);
+        return Registry::read(settings, path, Registry::Global);
     }();
 
     return reg;
 }
 
-Registries getRegistries(ref<Store> store)
+Registries getRegistries(const Settings & settings, ref<Store> store)
 {
     Registries registries;
-    registries.push_back(getFlagRegistry());
-    registries.push_back(getUserRegistry());
-    registries.push_back(getSystemRegistry());
-    registries.push_back(getGlobalRegistry(store));
+    registries.push_back(getFlagRegistry(settings));
+    registries.push_back(getUserRegistry(settings));
+    registries.push_back(getSystemRegistry(settings));
+    registries.push_back(getGlobalRegistry(settings, store));
     return registries;
 }
 
@@ -207,7 +192,7 @@ std::pair<Input, Attrs> lookupInRegistries(
     n++;
     if (n > 100) throw Error("cycle detected in flake registry for '%s'", input.to_string());
 
-    for (auto & registry : getRegistries(store)) {
+    for (auto & registry : getRegistries(*input.settings, store)) {
         // FIXME: O(n)
         for (auto & entry : registry->entries) {
             if (entry.exact) {

src/libfetchers/registry.hh

@@ -10,6 +10,8 @@ namespace nix::fetchers {
 
 struct Registry
 {
+    const Settings & settings;
+
     enum RegistryType {
         Flag = 0,
         User = 1,
@@ -29,11 +31,13 @@ struct Registry
 
     std::vector<Entry> entries;
 
-    Registry(RegistryType type)
-        : type(type)
+    Registry(const Settings & settings, RegistryType type)
+        : settings{settings}
+        , type{type}
     { }
 
     static std::shared_ptr<Registry> read(
+        const Settings & settings,
         const Path & path, RegistryType type);
 
     void write(const Path & path);
@@ -48,13 +52,13 @@ struct Registry
 
 typedef std::vector<std::shared_ptr<Registry>> Registries;
 
-std::shared_ptr<Registry> getUserRegistry();
+std::shared_ptr<Registry> getUserRegistry(const Settings & settings);
 
-std::shared_ptr<Registry> getCustomRegistry(const Path & p);
+std::shared_ptr<Registry> getCustomRegistry(const Settings & settings, const Path & p);
 
 Path getUserRegistryPath();
 
-Registries getRegistries(ref<Store> store);
+Registries getRegistries(const Settings & settings, ref<Store> store);
 
 void overrideRegistry(
     const Input & from,

src/libfetchers/tarball.cc

@@ -2,12 +2,10 @@
 #include "fetchers.hh"
 #include "cache.hh"
 #include "filetransfer.hh"
-#include "globals.hh"
 #include "store-api.hh"
 #include "archive.hh"
 #include "tarfile.hh"
 #include "types.hh"
-#include "split.hh"
 #include "store-path-accessor.hh"
 #include "store-api.hh"
 #include "git-utils.hh"
@@ -214,12 +212,14 @@ struct CurlInputScheme : InputScheme
 
     static const std::set<std::string> specialParams;
 
-    std::optional<Input> inputFromURL(const ParsedURL & _url, bool requireTree) const override
+    std::optional<Input> inputFromURL(
+        const Settings & settings,
+        const ParsedURL & _url, bool requireTree) const override
     {
         if (!isValidURL(_url, requireTree))
             return std::nullopt;
 
-        Input input;
+        Input input{settings};
 
         auto url = _url;
 
@@ -267,9 +267,11 @@ struct CurlInputScheme : InputScheme
         };
     }
 
-    std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
+    std::optional<Input> inputFromAttrs(
+        const Settings & settings,
+        const Attrs & attrs) const override
     {
-        Input input;
+        Input input{settings};
         input.attrs = attrs;
 
         //input.locked = (bool) maybeGetStrAttr(input.attrs, "hash");
@@ -349,7 +351,7 @@ struct TarballInputScheme : CurlInputScheme
         result.accessor->setPathDisplay("«" + input.to_string() + "»");
 
         if (result.immutableUrl) {
-            auto immutableInput = Input::fromURL(*result.immutableUrl);
+            auto immutableInput = Input::fromURL(*input.settings, *result.immutableUrl);
             // FIXME: would be nice to support arbitrary flakerefs
             // here, e.g. git flakes.
             if (immutableInput.getType() != "tarball")

src/libfetchers/tarball.hh

@@ -1,11 +1,12 @@
 #pragma once
 
-#include "types.hh"
-#include "path.hh"
-#include "hash.hh"
-
 #include <optional>
 
+#include "hash.hh"
+#include "path.hh"
+#include "ref.hh"
+#include "types.hh"
+
 namespace nix {
 class Store;
 struct SourceAccessor;

src/libflake/flake-settings.cc

@@ -1,12 +0,0 @@
-#include "flake-settings.hh"
-#include "config-global.hh"
-
-namespace nix {
-
-FlakeSettings::FlakeSettings() {}
-
-FlakeSettings flakeSettings;
-
-static GlobalConfig::Register rFlakeSettings(&flakeSettings);
-
-}

src/libflake/flake/config.cc

@@ -1,6 +1,6 @@
 #include "users.hh"
 #include "config-global.hh"
-#include "flake-settings.hh"
+#include "flake/settings.hh"
 #include "flake.hh"
 
 #include <nlohmann/json.hpp>
@@ -30,7 +30,7 @@ static void writeTrustedList(const TrustedList & trustedList)
     writeFile(path, nlohmann::json(trustedList).dump());
 }
 
-void ConfigFile::apply()
+void ConfigFile::apply(const Settings & flakeSettings)
 {
     std::set<std::string> whitelist{"bash-prompt", "bash-prompt-prefix", "bash-prompt-suffix", "flake-registry", "commit-lock-file-summary", "commit-lockfile-summary"};
 
@@ -47,11 +47,11 @@ void ConfigFile::apply()
         else if (auto* b = std::get_if<Explicit<bool>>(&value))
             valueS = b->t ? "true" : "false";
         else if (auto ss = std::get_if<std::vector<std::string>>(&value))
-            valueS = concatStringsSep(" ", *ss); // FIXME: evil
+            valueS = dropEmptyInitThenConcatStringsSep(" ", *ss); // FIXME: evil
         else
             assert(false);
 
-        if (!whitelist.count(baseName) && !nix::flakeSettings.acceptFlakeConfig) {
+        if (!whitelist.count(baseName) && !flakeSettings.acceptFlakeConfig) {
             bool trusted = false;
             auto trustedList = readTrustedList();
             auto tlname = get(trustedList, name);

src/libflake/flake/flake.cc

@@ -9,7 +9,7 @@
 #include "fetchers.hh"
 #include "finally.hh"
 #include "fetch-settings.hh"
-#include "flake-settings.hh"
+#include "flake/settings.hh"
 #include "value-to-json.hh"
 #include "local-fs-store.hh"
 
@@ -98,7 +98,7 @@ static std::map<FlakeId, FlakeInput> parseFlakeInputs(
     const std::optional<Path> & baseDir, InputPath lockRootPath);
 
 static FlakeInput parseFlakeInput(EvalState & state,
-    const std::string & inputName, Value * value, const PosIdx pos,
+    std::string_view inputName, Value * value, const PosIdx pos,
     const std::optional<Path> & baseDir, InputPath lockRootPath)
 {
     expectType(state, nAttrs, *value, pos);
@@ -164,7 +164,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
 
     if (attrs.count("type"))
         try {
-            input.ref = FlakeRef::fromAttrs(attrs);
+            input.ref = FlakeRef::fromAttrs(state.fetchSettings, attrs);
         } catch (Error & e) {
             e.addTrace(state.positions[pos], HintFmt("while evaluating flake input"));
             throw;
@@ -174,11 +174,11 @@ static FlakeInput parseFlakeInput(EvalState & state,
         if (!attrs.empty())
             throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, state.positions[pos]);
         if (url)
-            input.ref = parseFlakeRef(*url, baseDir, true, input.isFlake);
+            input.ref = parseFlakeRef(state.fetchSettings, *url, baseDir, true, input.isFlake);
     }
 
     if (!input.follows && !input.ref)
-        input.ref = FlakeRef::fromAttrs({{"type", "indirect"}, {"id", inputName}});
+        input.ref = FlakeRef::fromAttrs(state.fetchSettings, {{"type", "indirect"}, {"id", std::string(inputName)}});
 
     return input;
 }
@@ -244,7 +244,7 @@ Flake readFlake(
             for (auto & formal : outputs->value->payload.lambda.fun->formals->formals) {
                 if (formal.name != state.sSelf)
                     flake.inputs.emplace(state.symbols[formal.name], FlakeInput {
-                        .ref = parseFlakeRef(state.symbols[formal.name])
+                        .ref = parseFlakeRef(state.fetchSettings, std::string(state.symbols[formal.name]))
                     });
             }
         }
@@ -329,33 +329,37 @@ Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup
     return getFlake(state, originalRef, allowLookup, flakeCache);
 }
 
-static LockFile readLockFile(const SourcePath & lockFilePath)
+static LockFile readLockFile(
+    const fetchers::Settings & fetchSettings,
+    const SourcePath & lockFilePath)
 {
     return lockFilePath.pathExists()
-        ? LockFile(lockFilePath.readFile(), fmt("%s", lockFilePath))
+        ? LockFile(fetchSettings, lockFilePath.readFile(), fmt("%s", lockFilePath))
         : LockFile();
 }
 
 LockedFlake lockFlake(
+    const Settings & settings,
     EvalState & state,
     const FlakeRef & topRef,
     const LockFlags & lockFlags,
     Flake flake,
     FlakeCache & flakeCache)
 {
-    auto useRegistries = lockFlags.useRegistries.value_or(flakeSettings.useRegistries);
+    auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries);
 
     if (lockFlags.applyNixConfig) {
-        flake.config.apply();
+        flake.config.apply(settings);
         state.store->setOptions();
     }
 
     try {
-        if (!fetchSettings.allowDirty && lockFlags.referenceLockFilePath) {
+        if (!state.fetchSettings.allowDirty && lockFlags.referenceLockFilePath) {
             throw Error("reference lock file was provided, but the `allow-dirty` setting is set to false");
         }
 
         auto oldLockFile = readLockFile(
+            state.fetchSettings,
             lockFlags.referenceLockFilePath.value_or(
                 flake.lockFilePath()));
 
@@ -591,7 +595,7 @@ LockedFlake lockFlake(
                                 inputFlake.inputs, childNode, inputPath,
                                 oldLock
                                 ? std::dynamic_pointer_cast<const Node>(oldLock)
-                                : readLockFile(inputFlake.lockFilePath()).root.get_ptr(),
+                                : readLockFile(state.fetchSettings, inputFlake.lockFilePath()).root.get_ptr(),
                                 oldLock ? lockRootPath : inputPath,
                                 localPath,
                                 false);
@@ -654,7 +658,7 @@ LockedFlake lockFlake(
             if (lockFlags.writeLockFile) {
                 if (sourcePath || lockFlags.outputLockFilePath) {
                     if (auto unlockedInput = newLockFile.isUnlocked()) {
-                        if (fetchSettings.warnDirty)
+                        if (state.fetchSettings.warnDirty)
                             warn("will not write lock file of flake '%s' because it has an unlocked input ('%s')", topRef, *unlockedInput);
                     } else {
                         if (!lockFlags.updateLockFile)
@@ -686,7 +690,7 @@ LockedFlake lockFlake(
                             if (lockFlags.commitLockFile) {
                                 std::string cm;
 
-                                cm = flakeSettings.commitLockFileSummary.get();
+                                cm = settings.commitLockFileSummary.get();
 
                                 if (cm == "") {
                                     cm = fmt("%s: %s", relPath, lockFileExists ? "Update" : "Add");
@@ -735,25 +739,27 @@ LockedFlake lockFlake(
 }
 
 LockedFlake lockFlake(
+    const Settings & settings,
     EvalState & state,
     const FlakeRef & topRef,
     const LockFlags & lockFlags)
 {
     FlakeCache flakeCache;
 
-    auto useRegistries = lockFlags.useRegistries.value_or(flakeSettings.useRegistries);
+    auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries);
 
-    return lockFlake(state, topRef, lockFlags, getFlake(state, topRef, useRegistries, flakeCache), flakeCache);
+    return lockFlake(settings, state, topRef, lockFlags, getFlake(state, topRef, useRegistries, flakeCache), flakeCache);
 }
 
 LockedFlake lockFlake(
+    const Settings & settings,
     EvalState & state,
     const FlakeRef & topRef,
     const LockFlags & lockFlags,
     Flake flake)
 {
     FlakeCache flakeCache;
-    return lockFlake(state, topRef, lockFlags, std::move(flake), flakeCache);
+    return lockFlake(settings, state, topRef, lockFlags, std::move(flake), flakeCache);
 }
 
 void callFlake(EvalState & state,
@@ -814,45 +820,48 @@ void callFlake(EvalState & state,
     state.callFunction(*vTmp1, vOverrides, vRes, noPos);
 }
 
-static void prim_getFlake(EvalState & state, const PosIdx pos, Value * * args, Value & v)
+void initLib(const Settings & settings)
 {
-    std::string flakeRefS(state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.getFlake"));
-    auto flakeRef = parseFlakeRef(flakeRefS, {}, true);
-    if (state.settings.pureEval && !flakeRef.input.isLocked())
-        throw Error("cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)", flakeRefS, state.positions[pos]);
+    auto prim_getFlake = [&settings](EvalState & state, const PosIdx pos, Value * * args, Value & v)
+    {
+        std::string flakeRefS(state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.getFlake"));
+        auto flakeRef = parseFlakeRef(state.fetchSettings, flakeRefS, {}, true);
+        if (state.settings.pureEval && !flakeRef.input.isLocked())
+            throw Error("cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)", flakeRefS, state.positions[pos]);
 
-    callFlake(state,
-        lockFlake(state, flakeRef,
-            LockFlags {
-                .updateLockFile = false,
-                .writeLockFile = false,
-                .useRegistries = !state.settings.pureEval && flakeSettings.useRegistries,
-                .allowUnlocked = !state.settings.pureEval,
-            }),
-        v);
+        callFlake(state,
+            lockFlake(settings, state, flakeRef,
+                LockFlags {
+                    .updateLockFile = false,
+                    .writeLockFile = false,
+                    .useRegistries = !state.settings.pureEval && settings.useRegistries,
+                    .allowUnlocked = !state.settings.pureEval,
+                }),
+            v);
+    };
+
+    RegisterPrimOp::primOps->push_back({
+        .name =  "__getFlake",
+        .args = {"args"},
+        .doc = R"(
+          Fetch a flake from a flake reference, and return its output attributes and some metadata. For example:
+
+          ```nix
+          (builtins.getFlake "nix/55bc52401966fbffa525c574c14f67b00bc4fb3a").packages.x86_64-linux.nix
+          ```
+
+          Unless impure evaluation is allowed (`--impure`), the flake reference
+          must be "locked", e.g. contain a Git revision or content hash. An
+          example of an unlocked usage is:
+
+          ```nix
+          (builtins.getFlake "github:edolstra/dwarffs").rev
+          ```
+        )",
+        .fun = prim_getFlake,
+    });
 }
 
-static RegisterPrimOp r2({
-    .name =  "__getFlake",
-    .args = {"args"},
-    .doc = R"(
-      Fetch a flake from a flake reference, and return its output attributes and some metadata. For example:
-
-      ```nix
-      (builtins.getFlake "nix/55bc52401966fbffa525c574c14f67b00bc4fb3a").packages.x86_64-linux.nix
-      ```
-
-      Unless impure evaluation is allowed (`--impure`), the flake reference
-      must be "locked", e.g. contain a Git revision or content hash. An
-      example of an unlocked usage is:
-
-      ```nix
-      (builtins.getFlake "github:edolstra/dwarffs").rev
-      ```
-    )",
-    .fun = prim_getFlake,
-});
-
 static void prim_parseFlakeRef(
     EvalState & state,
     const PosIdx pos,
@@ -861,7 +870,7 @@ static void prim_parseFlakeRef(
 {
     std::string flakeRefS(state.forceStringNoCtx(*args[0], pos,
         "while evaluating the argument passed to builtins.parseFlakeRef"));
-    auto attrs = parseFlakeRef(flakeRefS, {}, true).toAttrs();
+    auto attrs = parseFlakeRef(state.fetchSettings, flakeRefS, {}, true).toAttrs();
     auto binds = state.buildBindings(attrs.size());
     for (const auto & [key, value] : attrs) {
         auto s = state.symbols.create(key);
@@ -925,7 +934,7 @@ static void prim_flakeRefToString(
                 showType(*attr.value)).debugThrow();
         }
     }
-    auto flakeRef = FlakeRef::fromAttrs(attrs);
+    auto flakeRef = FlakeRef::fromAttrs(state.fetchSettings, attrs);
     v.mkString(flakeRef.to_string());
 }
 

src/libflake/flake/flake.hh

@@ -12,6 +12,16 @@ class EvalState;
 
 namespace flake {
 
+struct Settings;
+
+/**
+ * Initialize `libnixflake`
+ *
+ * So far, this registers the `builtins.getFlake` primop, which depends
+ * on the choice of `flake:Settings`.
+ */
+void initLib(const Settings & settings);
+
 struct FlakeInput;
 
 typedef std::map<FlakeId, FlakeInput> FlakeInputs;
@@ -57,7 +67,7 @@ struct ConfigFile
 
     std::map<std::string, ConfigValue> settings;
 
-    void apply();
+    void apply(const Settings & settings);
 };
 
 /**
@@ -206,11 +216,13 @@ Flake readFlake(
  * and optionally write it to file, if the flake is writable.
  */
 LockedFlake lockFlake(
+    const Settings & settings,
     EvalState & state,
     const FlakeRef & flakeRef,
     const LockFlags & lockFlags);
 
 LockedFlake lockFlake(
+    const Settings & settings,
     EvalState & state,
     const FlakeRef & topRef,
     const LockFlags & lockFlags,

src/libflake/flake/flakeref.cc

@@ -36,11 +36,6 @@ std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef)
     return str;
 }
 
-bool FlakeRef::operator ==(const FlakeRef & other) const
-{
-    return input == other.input && subdir == other.subdir;
-}
-
 FlakeRef FlakeRef::resolve(ref<Store> store) const
 {
     auto [input2, extraAttrs] = lookupInRegistries(store, input);
@@ -48,28 +43,32 @@ FlakeRef FlakeRef::resolve(ref<Store> store) const
 }
 
 FlakeRef parseFlakeRef(
+    const fetchers::Settings & fetchSettings,
     const std::string & url,
     const std::optional<Path> & baseDir,
     bool allowMissing,
     bool isFlake)
 {
-    auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing, isFlake);
+    auto [flakeRef, fragment] = parseFlakeRefWithFragment(fetchSettings, url, baseDir, allowMissing, isFlake);
     if (fragment != "")
         throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url);
     return flakeRef;
 }
 
 std::optional<FlakeRef> maybeParseFlakeRef(
-    const std::string & url, const std::optional<Path> & baseDir)
+    const fetchers::Settings & fetchSettings,
+    const std::string & url,
+    const std::optional<Path> & baseDir)
 {
     try {
-        return parseFlakeRef(url, baseDir);
+        return parseFlakeRef(fetchSettings, url, baseDir);
     } catch (Error &) {
         return {};
     }
 }
 
 std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
+    const fetchers::Settings & fetchSettings,
     const std::string & url,
     const std::optional<Path> & baseDir,
     bool allowMissing,
@@ -166,7 +165,7 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
                         parsedURL.query.insert_or_assign("shallow", "1");
 
                     return std::make_pair(
-                        FlakeRef(fetchers::Input::fromURL(parsedURL), getOr(parsedURL.query, "dir", "")),
+                        FlakeRef(fetchers::Input::fromURL(fetchSettings, parsedURL), getOr(parsedURL.query, "dir", "")),
                         fragment);
                 }
 
@@ -185,13 +184,14 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
     attrs.insert_or_assign("type", "path");
     attrs.insert_or_assign("path", path);
 
-    return std::make_pair(FlakeRef(fetchers::Input::fromAttrs(std::move(attrs)), ""), fragment);
+    return std::make_pair(FlakeRef(fetchers::Input::fromAttrs(fetchSettings, std::move(attrs)), ""), fragment);
 };
 
 
 /* Check if 'url' is a flake ID. This is an abbreviated syntax for
     'flake:<flake-id>?ref=<ref>&rev=<rev>'. */
-std::optional<std::pair<FlakeRef, std::string>> parseFlakeIdRef(
+static std::optional<std::pair<FlakeRef, std::string>> parseFlakeIdRef(
+    const fetchers::Settings & fetchSettings,
     const std::string & url,
     bool isFlake
 )
@@ -213,7 +213,7 @@ std::optional<std::pair<FlakeRef, std::string>> parseFlakeIdRef(
         };
 
         return std::make_pair(
-            FlakeRef(fetchers::Input::fromURL(parsedURL, isFlake), ""),
+            FlakeRef(fetchers::Input::fromURL(fetchSettings, parsedURL, isFlake), ""),
             percentDecode(match.str(6)));
     }
 
@@ -221,6 +221,7 @@ std::optional<std::pair<FlakeRef, std::string>> parseFlakeIdRef(
 }
 
 std::optional<std::pair<FlakeRef, std::string>> parseURLFlakeRef(
+    const fetchers::Settings & fetchSettings,
     const std::string & url,
     const std::optional<Path> & baseDir,
     bool isFlake
@@ -236,7 +237,7 @@ std::optional<std::pair<FlakeRef, std::string>> parseURLFlakeRef(
     std::string fragment;
     std::swap(fragment, parsedURL.fragment);
 
-    auto input = fetchers::Input::fromURL(parsedURL, isFlake);
+    auto input = fetchers::Input::fromURL(fetchSettings, parsedURL, isFlake);
     input.parent = baseDir;
 
     return std::make_pair(
@@ -245,6 +246,7 @@ std::optional<std::pair<FlakeRef, std::string>> parseURLFlakeRef(
 }
 
 std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
+    const fetchers::Settings & fetchSettings,
     const std::string & url,
     const std::optional<Path> & baseDir,
     bool allowMissing,
@@ -254,31 +256,34 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
 
     std::smatch match;
 
-    if (auto res = parseFlakeIdRef(url, isFlake)) {
+    if (auto res = parseFlakeIdRef(fetchSettings, url, isFlake)) {
         return *res;
-    } else if (auto res = parseURLFlakeRef(url, baseDir, isFlake)) {
+    } else if (auto res = parseURLFlakeRef(fetchSettings, url, baseDir, isFlake)) {
         return *res;
     } else {
-        return parsePathFlakeRefWithFragment(url, baseDir, allowMissing, isFlake);
+        return parsePathFlakeRefWithFragment(fetchSettings, url, baseDir, allowMissing, isFlake);
     }
 }
 
 std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
+    const fetchers::Settings & fetchSettings,
     const std::string & url, const std::optional<Path> & baseDir)
 {
     try {
-        return parseFlakeRefWithFragment(url, baseDir);
+        return parseFlakeRefWithFragment(fetchSettings, url, baseDir);
     } catch (Error & e) {
         return {};
     }
 }
 
-FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs)
+FlakeRef FlakeRef::fromAttrs(
+    const fetchers::Settings & fetchSettings,
+    const fetchers::Attrs & attrs)
 {
     auto attrs2(attrs);
     attrs2.erase("dir");
     return FlakeRef(
-        fetchers::Input::fromAttrs(std::move(attrs2)),
+        fetchers::Input::fromAttrs(fetchSettings, std::move(attrs2)),
         fetchers::maybeGetStrAttr(attrs, "dir").value_or(""));
 }
 
@@ -289,13 +294,16 @@ std::pair<StorePath, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const
 }
 
 std::tuple<FlakeRef, std::string, ExtendedOutputsSpec> parseFlakeRefWithFragmentAndExtendedOutputsSpec(
+    const fetchers::Settings & fetchSettings,
     const std::string & url,
     const std::optional<Path> & baseDir,
     bool allowMissing,
     bool isFlake)
 {
     auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(url);
-    auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, baseDir, allowMissing, isFlake);
+    auto [flakeRef, fragment] = parseFlakeRefWithFragment(
+        fetchSettings,
+        std::string { prefix }, baseDir, allowMissing, isFlake);
     return {std::move(flakeRef), fragment, std::move(extendedOutputsSpec)};
 }
 

src/libflake/flake/flakeref.hh

@@ -1,14 +1,12 @@
 #pragma once
 ///@file
 
+#include <regex>
+
 #include "types.hh"
-#include "hash.hh"
 #include "fetchers.hh"
 #include "outputs-spec.hh"
 
-#include <regex>
-#include <variant>
-
 namespace nix {
 
 class Store;
@@ -48,7 +46,7 @@ struct FlakeRef
      */
     Path subdir;
 
-    bool operator==(const FlakeRef & other) const;
+    bool operator ==(const FlakeRef & other) const = default;
 
     FlakeRef(fetchers::Input && input, const Path & subdir)
         : input(std::move(input)), subdir(subdir)
@@ -61,7 +59,9 @@ struct FlakeRef
 
     FlakeRef resolve(ref<Store> store) const;
 
-    static FlakeRef fromAttrs(const fetchers::Attrs & attrs);
+    static FlakeRef fromAttrs(
+        const fetchers::Settings & fetchSettings,
+        const fetchers::Attrs & attrs);
 
     std::pair<StorePath, FlakeRef> fetchTree(ref<Store> store) const;
 };
@@ -72,6 +72,7 @@ std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef);
  * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory)
  */
 FlakeRef parseFlakeRef(
+    const fetchers::Settings & fetchSettings,
     const std::string & url,
     const std::optional<Path> & baseDir = {},
     bool allowMissing = false,
@@ -81,12 +82,15 @@ FlakeRef parseFlakeRef(
  * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory)
  */
 std::optional<FlakeRef> maybeParseFlake(
-    const std::string & url, const std::optional<Path> & baseDir = {});
+    const fetchers::Settings & fetchSettings,
+    const std::string & url,
+    const std::optional<Path> & baseDir = {});
 
 /**
  * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory)
  */
 std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
+    const fetchers::Settings & fetchSettings,
     const std::string & url,
     const std::optional<Path> & baseDir = {},
     bool allowMissing = false,
@@ -96,12 +100,15 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
  * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory)
  */
 std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
-    const std::string & url, const std::optional<Path> & baseDir = {});
+    const fetchers::Settings & fetchSettings,
+    const std::string & url,
+    const std::optional<Path> & baseDir = {});
 
 /**
  * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory)
  */
 std::tuple<FlakeRef, std::string, ExtendedOutputsSpec> parseFlakeRefWithFragmentAndExtendedOutputsSpec(
+    const fetchers::Settings & fetchSettings,
     const std::string & url,
     const std::optional<Path> & baseDir = {},
     bool allowMissing = false,

src/libflake/flake/lockfile.cc

@@ -1,6 +1,7 @@
+#include <unordered_set>
+
 #include "lockfile.hh"
 #include "store-api.hh"
-#include "url-parts.hh"
 
 #include <algorithm>
 #include <iomanip>
@@ -8,9 +9,12 @@
 #include <iterator>
 #include <nlohmann/json.hpp>
 
+#include "strings.hh"
+
 namespace nix::flake {
 
-FlakeRef getFlakeRef(
+static FlakeRef getFlakeRef(
+    const fetchers::Settings & fetchSettings,
     const nlohmann::json & json,
     const char * attr,
     const char * info)
@@ -26,15 +30,17 @@ FlakeRef getFlakeRef(
                     attrs.insert_or_assign(k.first, k.second);
             }
         }
-        return FlakeRef::fromAttrs(attrs);
+        return FlakeRef::fromAttrs(fetchSettings, attrs);
     }
 
     throw Error("attribute '%s' missing in lock file", attr);
 }
 
-LockedNode::LockedNode(const nlohmann::json & json)
-    : lockedRef(getFlakeRef(json, "locked", "info")) // FIXME: remove "info"
-    , originalRef(getFlakeRef(json, "original", nullptr))
+LockedNode::LockedNode(
+    const fetchers::Settings & fetchSettings,
+    const nlohmann::json & json)
+    : lockedRef(getFlakeRef(fetchSettings, json, "locked", "info")) // FIXME: remove "info"
+    , originalRef(getFlakeRef(fetchSettings, json, "original", nullptr))
     , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true)
 {
     if (!lockedRef.input.isLocked())
@@ -84,7 +90,9 @@ std::shared_ptr<Node> LockFile::findInput(const InputPath & path)
     return doFind(root, path, visited);
 }
 
-LockFile::LockFile(std::string_view contents, std::string_view path)
+LockFile::LockFile(
+    const fetchers::Settings & fetchSettings,
+    std::string_view contents, std::string_view path)
 {
     auto json = nlohmann::json::parse(contents);
 
@@ -113,7 +121,7 @@ LockFile::LockFile(std::string_view contents, std::string_view path)
                     auto jsonNode2 = nodes.find(inputKey);
                     if (jsonNode2 == nodes.end())
                         throw Error("lock file references missing node '%s'", inputKey);
-                    auto input = make_ref<LockedNode>(*jsonNode2);
+                    auto input = make_ref<LockedNode>(fetchSettings, *jsonNode2);
                     k = nodeMap.insert_or_assign(inputKey, input).first;
                     getInputs(*input, *jsonNode2);
                 }
@@ -243,11 +251,6 @@ bool LockFile::operator ==(const LockFile & other) const
     return toJSON().first == other.toJSON().first;
 }
 
-bool LockFile::operator !=(const LockFile & other) const
-{
-    return !(*this == other);
-}
-
 InputPath parseInputPath(std::string_view s)
 {
     InputPath path;

src/libflake/flake/lockfile.hh

@@ -45,7 +45,9 @@ struct LockedNode : Node
         : lockedRef(lockedRef), originalRef(originalRef), isFlake(isFlake)
     { }
 
-    LockedNode(const nlohmann::json & json);
+    LockedNode(
+        const fetchers::Settings & fetchSettings,
+        const nlohmann::json & json);
 
     StorePath computeStorePath(Store & store) const;
 };
@@ -55,7 +57,9 @@ struct LockFile
     ref<Node> root = make_ref<Node>();
 
     LockFile() {};
-    LockFile(std::string_view contents, std::string_view path);
+    LockFile(
+        const fetchers::Settings & fetchSettings,
+        std::string_view contents, std::string_view path);
 
     typedef std::map<ref<const Node>, std::string> KeyMap;
 
@@ -70,9 +74,6 @@ struct LockFile
     std::optional<FlakeRef> isUnlocked() const;
 
     bool operator ==(const LockFile & other) const;
-    // Needed for old gcc versions that don't synthesize it (like gcc 8.2.2
-    // that is still the default on aarch64-linux)
-    bool operator !=(const LockFile & other) const;
 
     std::shared_ptr<Node> findInput(const InputPath & path);
 

src/libflake/flake/settings.cc

@@ -0,0 +1,7 @@
+#include "flake/settings.hh"
+
+namespace nix::flake {
+
+Settings::Settings() {}
+
+}

src/libflake/flake/settings.hh

@@ -10,11 +10,11 @@
 
 #include <sys/types.h>
 
-namespace nix {
+namespace nix::flake {
 
-struct FlakeSettings : public Config
+struct Settings : public Config
 {
-    FlakeSettings();
+    Settings();
 
     Setting<bool> useRegistries{
         this, true, "use-registries", "Whether to use flake registries to resolve flake references.", {}, true};
@@ -39,7 +39,4 @@ struct FlakeSettings : public Config
         true};
 };
 
-// TODO: don't use a global variable.
-extern FlakeSettings flakeSettings;
-
 }

src/libflake/meson.build

@@ -42,21 +42,21 @@ add_project_arguments(
 subdir('build-utils-meson/diagnostics')
 
 sources = files(
-  'flake-settings.cc',
   'flake/config.cc',
   'flake/flake.cc',
   'flake/flakeref.cc',
-  'flake/url-name.cc',
   'flake/lockfile.cc',
+  'flake/settings.cc',
+  'flake/url-name.cc',
 )
 
 include_dirs = [include_directories('.')]
 
 headers = files(
-  'flake-settings.hh',
   'flake/flake.hh',
   'flake/flakeref.hh',
   'flake/lockfile.hh',
+  'flake/settings.hh',
   'flake/url-name.hh',
 )
 

src/libmain/shared.cc

@@ -8,7 +8,6 @@
 #include "signals.hh"
 
 #include <algorithm>
-#include <cctype>
 #include <exception>
 #include <iostream>
 
@@ -23,6 +22,8 @@
 
 #include <openssl/crypto.h>
 
+#include "exit.hh"
+#include "strings.hh"
 
 namespace nix {
 

src/libmain/shared.hh

@@ -8,13 +8,9 @@
 #include "common-args.hh"
 #include "path.hh"
 #include "derived-path.hh"
-#include "exit.hh"
 
 #include <signal.h>
 
-#include <locale>
-
-
 namespace nix {
 
 int handleExceptions(const std::string & programName, std::function<void()> fun);

src/libstore/build-result.cc

@@ -2,17 +2,7 @@
 
 namespace nix {
 
-GENERATE_CMP_EXT(
-    ,
-    BuildResult,
-    me->status,
-    me->errorMsg,
-    me->timesBuilt,
-    me->isNonDeterministic,
-    me->builtOutputs,
-    me->startTime,
-    me->stopTime,
-    me->cpuUser,
-    me->cpuSystem);
+bool BuildResult::operator==(const BuildResult &) const noexcept = default;
+std::strong_ordering BuildResult::operator<=>(const BuildResult &) const noexcept = default;
 
 }

src/libstore/build-result.hh

@@ -3,7 +3,6 @@
 
 #include "realisation.hh"
 #include "derived-path.hh"
-#include "comparator.hh"
 
 #include <string>
 #include <chrono>
@@ -101,7 +100,8 @@ struct BuildResult
      */
     std::optional<std::chrono::microseconds> cpuUser, cpuSystem;
 
-    DECLARE_CMP(BuildResult);
+    bool operator ==(const BuildResult &) const noexcept;
+    std::strong_ordering operator <=>(const BuildResult &) const noexcept;
 
     bool success()
     {

src/libstore/build/derivation-goal.cc

@@ -32,8 +32,18 @@
 
 #include <nlohmann/json.hpp>
 
+#include "strings.hh"
+
 namespace nix {
 
+Goal::Co DerivationGoal::init() {
+    if (useDerivation) {
+        co_return getDerivation();
+    } else {
+        co_return haveDerivation();
+    }
+}
+
 DerivationGoal::DerivationGoal(const StorePath & drvPath,
     const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode)
     : Goal(worker, DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = wantedOutputs })
@@ -42,7 +52,6 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath,
     , wantedOutputs(wantedOutputs)
     , buildMode(buildMode)
 {
-    state = &DerivationGoal::getDerivation;
     name = fmt(
         "building of '%s' from .drv file",
         DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store));
@@ -63,7 +72,6 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation
 {
     this->drv = std::make_unique<Derivation>(drv);
 
-    state = &DerivationGoal::haveDerivation;
     name = fmt(
         "building of '%s' from in-memory derivation",
         DerivedPath::Built { makeConstantStorePathRef(drvPath), drv.outputNames() }.to_string(worker.store));
@@ -107,13 +115,9 @@ void DerivationGoal::killChild()
 void DerivationGoal::timedOut(Error && ex)
 {
     killChild();
-    done(BuildResult::TimedOut, {}, std::move(ex));
-}
-
-
-void DerivationGoal::work()
-{
-    (this->*state)();
+    // We're not inside a coroutine, hence we can't use co_return here.
+    // Thus we ignore the return value.
+    [[maybe_unused]] Done _ = done(BuildResult::TimedOut, {}, std::move(ex));
 }
 
 void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs)
@@ -137,7 +141,7 @@ void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs)
 }
 
 
-void DerivationGoal::getDerivation()
+Goal::Co DerivationGoal::getDerivation()
 {
     trace("init");
 
@@ -145,23 +149,22 @@ void DerivationGoal::getDerivation()
        exists.  If it doesn't, it may be created through a
        substitute. */
     if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) {
-        loadDerivation();
-        return;
+        co_return loadDerivation();
     }
 
     addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath)));
 
-    state = &DerivationGoal::loadDerivation;
+    co_await Suspend{};
+    co_return loadDerivation();
 }
 
 
-void DerivationGoal::loadDerivation()
+Goal::Co DerivationGoal::loadDerivation()
 {
     trace("loading derivation");
 
     if (nrFailed != 0) {
-        done(BuildResult::MiscFailure, {}, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)));
-        return;
+        co_return done(BuildResult::MiscFailure, {}, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)));
     }
 
     /* `drvPath' should already be a root, but let's be on the safe
@@ -183,11 +186,11 @@ void DerivationGoal::loadDerivation()
     }
     assert(drv);
 
-    haveDerivation();
+    co_return haveDerivation();
 }
 
 
-void DerivationGoal::haveDerivation()
+Goal::Co DerivationGoal::haveDerivation()
 {
     trace("have derivation");
 
@@ -215,8 +218,7 @@ void DerivationGoal::haveDerivation()
             });
         }
 
-        gaveUpOnSubstitution();
-        return;
+        co_return gaveUpOnSubstitution();
     }
 
     for (auto & i : drv->outputsAndOptPaths(worker.store))
@@ -238,8 +240,7 @@ void DerivationGoal::haveDerivation()
 
     /* If they are all valid, then we're done. */
     if (allValid && buildMode == bmNormal) {
-        done(BuildResult::AlreadyValid, std::move(validOutputs));
-        return;
+        co_return done(BuildResult::AlreadyValid, std::move(validOutputs));
     }
 
     /* We are first going to try to create the invalid output paths
@@ -266,24 +267,21 @@ void DerivationGoal::haveDerivation()
             }
         }
 
-    if (waitees.empty()) /* to prevent hang (no wake-up event) */
-        outputsSubstitutionTried();
-    else
-        state = &DerivationGoal::outputsSubstitutionTried;
+    if (!waitees.empty()) co_await Suspend{}; /* to prevent hang (no wake-up event) */
+    co_return outputsSubstitutionTried();
 }
 
 
-void DerivationGoal::outputsSubstitutionTried()
+Goal::Co DerivationGoal::outputsSubstitutionTried()
 {
     trace("all outputs substituted (maybe)");
 
     assert(!drv->type().isImpure());
 
     if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) {
-        done(BuildResult::TransientFailure, {},
+        co_return done(BuildResult::TransientFailure, {},
             Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
                 worker.store.printStorePath(drvPath)));
-        return;
     }
 
     /*  If the substitutes form an incomplete closure, then we should
@@ -317,32 +315,29 @@ void DerivationGoal::outputsSubstitutionTried()
 
     if (needRestart == NeedRestartForMoreOutputs::OutputsAddedDoNeed) {
         needRestart = NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed;
-        haveDerivation();
-        return;
+        co_return haveDerivation();
     }
 
     auto [allValid, validOutputs] = checkPathValidity();
 
     if (buildMode == bmNormal && allValid) {
-        done(BuildResult::Substituted, std::move(validOutputs));
-        return;
+        co_return done(BuildResult::Substituted, std::move(validOutputs));
     }
     if (buildMode == bmRepair && allValid) {
-        repairClosure();
-        return;
+        co_return repairClosure();
     }
     if (buildMode == bmCheck && !allValid)
         throw Error("some outputs of '%s' are not valid, so checking is not possible",
             worker.store.printStorePath(drvPath));
 
     /* Nothing to wait for; tail call */
-    gaveUpOnSubstitution();
+    co_return gaveUpOnSubstitution();
 }
 
 
 /* At least one of the output paths could not be
    produced using a substitute.  So we have to build instead. */
-void DerivationGoal::gaveUpOnSubstitution()
+Goal::Co DerivationGoal::gaveUpOnSubstitution()
 {
     /* At this point we are building all outputs, so if more are wanted there
        is no need to restart. */
@@ -403,14 +398,12 @@ void DerivationGoal::gaveUpOnSubstitution()
         addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i)));
     }
 
-    if (waitees.empty()) /* to prevent hang (no wake-up event) */
-        inputsRealised();
-    else
-        state = &DerivationGoal::inputsRealised;
+    if (!waitees.empty()) co_await Suspend{}; /* to prevent hang (no wake-up event) */
+    co_return inputsRealised();
 }
 
 
-void DerivationGoal::repairClosure()
+Goal::Co DerivationGoal::repairClosure()
 {
     assert(!drv->type().isImpure());
 
@@ -464,41 +457,39 @@ void DerivationGoal::repairClosure()
     }
 
     if (waitees.empty()) {
-        done(BuildResult::AlreadyValid, assertPathValidity());
-        return;
+        co_return done(BuildResult::AlreadyValid, assertPathValidity());
+    } else {
+        co_await Suspend{};
+        co_return closureRepaired();
     }
-
-    state = &DerivationGoal::closureRepaired;
 }
 
 
-void DerivationGoal::closureRepaired()
+Goal::Co DerivationGoal::closureRepaired()
 {
     trace("closure repaired");
     if (nrFailed > 0)
         throw Error("some paths in the output closure of derivation '%s' could not be repaired",
             worker.store.printStorePath(drvPath));
-    done(BuildResult::AlreadyValid, assertPathValidity());
+    co_return done(BuildResult::AlreadyValid, assertPathValidity());
 }
 
 
-void DerivationGoal::inputsRealised()
+Goal::Co DerivationGoal::inputsRealised()
 {
     trace("all inputs realised");
 
     if (nrFailed != 0) {
         if (!useDerivation)
             throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath));
-        done(BuildResult::DependencyFailed, {}, Error(
+        co_return done(BuildResult::DependencyFailed, {}, Error(
                 "%s dependencies of derivation '%s' failed to build",
                 nrFailed, worker.store.printStorePath(drvPath)));
-        return;
     }
 
     if (retrySubstitution == RetrySubstitution::YesNeed) {
         retrySubstitution = RetrySubstitution::AlreadyRetried;
-        haveDerivation();
-        return;
+        co_return haveDerivation();
     }
 
     /* Gather information necessary for computing the closure and/or
@@ -564,8 +555,8 @@ void DerivationGoal::inputsRealised()
                 pathResolved, wantedOutputs, buildMode);
             addWaitee(resolvedDrvGoal);
 
-            state = &DerivationGoal::resolvedFinished;
-            return;
+            co_await Suspend{};
+            co_return resolvedFinished();
         }
 
         std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumInputPaths;
@@ -629,8 +620,9 @@ void DerivationGoal::inputsRealised()
     /* Okay, try to build.  Note that here we don't wait for a build
        slot to become available, since we don't need one if there is a
        build hook. */
-    state = &DerivationGoal::tryToBuild;
     worker.wakeUp(shared_from_this());
+    co_await Suspend{};
+    co_return tryToBuild();
 }
 
 void DerivationGoal::started()
@@ -655,7 +647,7 @@ void DerivationGoal::started()
     worker.updateProgress();
 }
 
-void DerivationGoal::tryToBuild()
+Goal::Co DerivationGoal::tryToBuild()
 {
     trace("trying to build");
 
@@ -691,7 +683,8 @@ void DerivationGoal::tryToBuild()
             actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
                 fmt("waiting for lock on %s", Magenta(showPaths(lockFiles))));
         worker.waitForAWhile(shared_from_this());
-        return;
+        co_await Suspend{};
+        co_return tryToBuild();
     }
 
     actLock.reset();
@@ -708,8 +701,7 @@ void DerivationGoal::tryToBuild()
     if (buildMode != bmCheck && allValid) {
         debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath));
         outputLocks.setDeletion(true);
-        done(BuildResult::AlreadyValid, std::move(validOutputs));
-        return;
+        co_return done(BuildResult::AlreadyValid, std::move(validOutputs));
     }
 
     /* If any of the outputs already exist but are not valid, delete
@@ -735,9 +727,9 @@ void DerivationGoal::tryToBuild()
                    EOF from the hook. */
                 actLock.reset();
                 buildResult.startTime = time(0); // inexact
-                state = &DerivationGoal::buildDone;
                 started();
-                return;
+                co_await Suspend{};
+                co_return buildDone();
             case rpPostpone:
                 /* Not now; wait until at least one child finishes or
                    the wake-up timeout expires. */
@@ -746,7 +738,8 @@ void DerivationGoal::tryToBuild()
                         fmt("waiting for a machine to build '%s'", Magenta(worker.store.printStorePath(drvPath))));
                 worker.waitForAWhile(shared_from_this());
                 outputLocks.unlock();
-                return;
+                co_await Suspend{};
+                co_return tryToBuild();
             case rpDecline:
                 /* We should do it ourselves. */
                 break;
@@ -755,11 +748,12 @@ void DerivationGoal::tryToBuild()
 
     actLock.reset();
 
-    state = &DerivationGoal::tryLocalBuild;
     worker.wakeUp(shared_from_this());
+    co_await Suspend{};
+    co_return tryLocalBuild();
 }
 
-void DerivationGoal::tryLocalBuild() {
+Goal::Co DerivationGoal::tryLocalBuild() {
     throw Error(
         R"(
         Unable to build with a primary store that isn't a local store;
@@ -936,7 +930,7 @@ void runPostBuildHook(
     });
 }
 
-void DerivationGoal::buildDone()
+Goal::Co DerivationGoal::buildDone()
 {
     trace("build done");
 
@@ -1031,7 +1025,7 @@ void DerivationGoal::buildDone()
         outputLocks.setDeletion(true);
         outputLocks.unlock();
 
-        done(BuildResult::Built, std::move(builtOutputs));
+        co_return done(BuildResult::Built, std::move(builtOutputs));
 
     } catch (BuildError & e) {
         outputLocks.unlock();
@@ -1056,12 +1050,11 @@ void DerivationGoal::buildDone()
                 BuildResult::PermanentFailure;
         }
 
-        done(st, {}, std::move(e));
-        return;
+        co_return done(st, {}, std::move(e));
     }
 }
 
-void DerivationGoal::resolvedFinished()
+Goal::Co DerivationGoal::resolvedFinished()
 {
     trace("resolved derivation finished");
 
@@ -1129,7 +1122,7 @@ void DerivationGoal::resolvedFinished()
     if (status == BuildResult::AlreadyValid)
         status = BuildResult::ResolvesToAlreadyValid;
 
-    done(status, std::move(builtOutputs));
+    co_return done(status, std::move(builtOutputs));
 }
 
 HookReply DerivationGoal::tryBuildHook()
@@ -1323,7 +1316,9 @@ void DerivationGoal::handleChildOutput(Descriptor fd, std::string_view data)
         logSize += data.size();
         if (settings.maxLogSize && logSize > settings.maxLogSize) {
             killChild();
-            done(
+            // We're not inside a coroutine, hence we can't use co_return here.
+            // Thus we ignore the return value.
+            [[maybe_unused]] Done _ = done(
                 BuildResult::LogLimitExceeded, {},
                 Error("%s killed after writing more than %d bytes of log output",
                     getName(), settings.maxLogSize));
@@ -1529,7 +1524,7 @@ SingleDrvOutputs DerivationGoal::assertPathValidity()
 }
 
 
-void DerivationGoal::done(
+Goal::Done DerivationGoal::done(
     BuildResult::Status status,
     SingleDrvOutputs builtOutputs,
     std::optional<Error> ex)
@@ -1566,7 +1561,7 @@ void DerivationGoal::done(
         fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl;
     }
 
-    amDone(buildResult.success() ? ecSuccess : ecFailed, std::move(ex));
+    return amDone(buildResult.success() ? ecSuccess : ecFailed, std::move(ex));
 }
 
 

src/libstore/build/derivation-goal.hh

@@ -194,9 +194,6 @@ struct DerivationGoal : public Goal
      */
     std::optional<DerivationType> derivationType;
 
-    typedef void (DerivationGoal::*GoalState)();
-    GoalState state;
-
     BuildMode buildMode;
 
     std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
@@ -227,8 +224,6 @@ struct DerivationGoal : public Goal
 
     std::string key() override;
 
-    void work() override;
-
     /**
      * Add wanted outputs to an already existing derivation goal.
      */
@@ -237,18 +232,19 @@ struct DerivationGoal : public Goal
     /**
      * The states.
      */
-    void getDerivation();
-    void loadDerivation();
-    void haveDerivation();
-    void outputsSubstitutionTried();
-    void gaveUpOnSubstitution();
-    void closureRepaired();
-    void inputsRealised();
-    void tryToBuild();
-    virtual void tryLocalBuild();
-    void buildDone();
+    Co init() override;
+    Co getDerivation();
+    Co loadDerivation();
+    Co haveDerivation();
+    Co outputsSubstitutionTried();
+    Co gaveUpOnSubstitution();
+    Co closureRepaired();
+    Co inputsRealised();
+    Co tryToBuild();
+    virtual Co tryLocalBuild();
+    Co buildDone();
 
-    void resolvedFinished();
+    Co resolvedFinished();
 
     /**
      * Is the build hook willing to perform the build?
@@ -329,11 +325,11 @@ struct DerivationGoal : public Goal
      */
     virtual void killChild();
 
-    void repairClosure();
+    Co repairClosure();
 
     void started();
 
-    void done(
+    Done done(
         BuildResult::Status status,
         SingleDrvOutputs builtOutputs = {},
         std::optional<Error> ex = {});

src/libstore/build/drv-output-substitution-goal.cc

@@ -14,146 +14,135 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(
     : Goal(worker, DerivedPath::Opaque { StorePath::dummy })
     , id(id)
 {
-    state = &DrvOutputSubstitutionGoal::init;
     name = fmt("substitution of '%s'", id.to_string());
     trace("created");
 }
 
 
-void DrvOutputSubstitutionGoal::init()
+Goal::Co DrvOutputSubstitutionGoal::init()
 {
     trace("init");
 
     /* If the derivation already exists, we’re done */
     if (worker.store.queryRealisation(id)) {
-        amDone(ecSuccess);
-        return;
+        co_return amDone(ecSuccess);
     }
 
-    subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
-    tryNext();
-}
+    auto subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
 
-void DrvOutputSubstitutionGoal::tryNext()
-{
-    trace("trying next substituter");
+    bool substituterFailed = false;
 
-    if (subs.size() == 0) {
-        /* None left.  Terminate this goal and let someone else deal
-           with it. */
-        debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string());
+    for (auto sub : subs) {
+        trace("trying next substituter");
 
-        /* Hack: don't indicate failure if there were no substituters.
-           In that case the calling derivation should just do a
-           build. */
-        amDone(substituterFailed ? ecFailed : ecNoSubstituters);
+        /* The callback of the curl download below can outlive `this` (if
+           some other error occurs), so it must not touch `this`. So put
+           the shared state in a separate refcounted object. */
+        auto outPipe = std::make_shared<MuxablePipe>();
+    #ifndef _WIN32
+        outPipe->create();
+    #else
+        outPipe->createAsyncPipe(worker.ioport.get());
+    #endif
 
-        if (substituterFailed) {
-            worker.failedSubstitutions++;
-            worker.updateProgress();
+        auto promise = std::make_shared<std::promise<std::shared_ptr<const Realisation>>>();
+
+        sub->queryRealisation(
+            id,
+            { [outPipe(outPipe), promise(promise)](std::future<std::shared_ptr<const Realisation>> res) {
+                try {
+                    Finally updateStats([&]() { outPipe->writeSide.close(); });
+                    promise->set_value(res.get());
+                } catch (...) {
+                    promise->set_exception(std::current_exception());
+                }
+            } });
+
+        worker.childStarted(shared_from_this(), {
+    #ifndef _WIN32
+            outPipe->readSide.get()
+    #else
+            &outPipe
+    #endif
+        }, true, false);
+
+        co_await Suspend{};
+
+        worker.childTerminated(this);
+
+        /*
+         * The realisation corresponding to the given output id.
+         * Will be filled once we can get it.
+         */
+        std::shared_ptr<const Realisation> outputInfo;
+
+        try {
+            outputInfo = promise->get_future().get();
+        } catch (std::exception & e) {
+            printError(e.what());
+            substituterFailed = true;
         }
 
-        return;
+        if (!outputInfo) continue;
+
+        bool failed = false;
+
+        for (const auto & [depId, depPath] : outputInfo->dependentRealisations) {
+            if (depId != id) {
+                if (auto localOutputInfo = worker.store.queryRealisation(depId);
+                    localOutputInfo && localOutputInfo->outPath != depPath) {
+                    warn(
+                        "substituter '%s' has an incompatible realisation for '%s', ignoring.\n"
+                        "Local:  %s\n"
+                        "Remote: %s",
+                        sub->getUri(),
+                        depId.to_string(),
+                        worker.store.printStorePath(localOutputInfo->outPath),
+                        worker.store.printStorePath(depPath)
+                    );
+                    failed = true;
+                    break;
+                }
+                addWaitee(worker.makeDrvOutputSubstitutionGoal(depId));
+            }
+        }
+
+        if (failed) continue;
+
+        co_return realisationFetched(outputInfo, sub);
     }
 
-    sub = subs.front();
-    subs.pop_front();
+    /* None left.  Terminate this goal and let someone else deal
+       with it. */
+    debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string());
 
-    // FIXME: Make async
-    // outputInfo = sub->queryRealisation(id);
+    if (substituterFailed) {
+        worker.failedSubstitutions++;
+        worker.updateProgress();
+    }
 
-    /* The callback of the curl download below can outlive `this` (if
-       some other error occurs), so it must not touch `this`. So put
-       the shared state in a separate refcounted object. */
-    downloadState = std::make_shared<DownloadState>();
-#ifndef _WIN32
-    downloadState->outPipe.create();
-#else
-    downloadState->outPipe.createAsyncPipe(worker.ioport.get());
-#endif
-
-    sub->queryRealisation(
-        id,
-        { [downloadState(downloadState)](std::future<std::shared_ptr<const Realisation>> res) {
-            try {
-                Finally updateStats([&]() { downloadState->outPipe.writeSide.close(); });
-                downloadState->promise.set_value(res.get());
-            } catch (...) {
-                downloadState->promise.set_exception(std::current_exception());
-            }
-        } });
-
-    worker.childStarted(shared_from_this(), {
-#ifndef _WIN32
-        downloadState->outPipe.readSide.get()
-#else
-        &downloadState->outPipe
-#endif
-    }, true, false);
-
-    state = &DrvOutputSubstitutionGoal::realisationFetched;
+    /* Hack: don't indicate failure if there were no substituters.
+       In that case the calling derivation should just do a
+       build. */
+    co_return amDone(substituterFailed ? ecFailed : ecNoSubstituters);
 }
 
-void DrvOutputSubstitutionGoal::realisationFetched()
-{
-    worker.childTerminated(this);
-
-    try {
-        outputInfo = downloadState->promise.get_future().get();
-    } catch (std::exception & e) {
-        printError(e.what());
-        substituterFailed = true;
-    }
-
-    if (!outputInfo) {
-        return tryNext();
-    }
-
-    for (const auto & [depId, depPath] : outputInfo->dependentRealisations) {
-        if (depId != id) {
-            if (auto localOutputInfo = worker.store.queryRealisation(depId);
-                localOutputInfo && localOutputInfo->outPath != depPath) {
-                warn(
-                    "substituter '%s' has an incompatible realisation for '%s', ignoring.\n"
-                    "Local:  %s\n"
-                    "Remote: %s",
-                    sub->getUri(),
-                    depId.to_string(),
-                    worker.store.printStorePath(localOutputInfo->outPath),
-                    worker.store.printStorePath(depPath)
-                );
-                tryNext();
-                return;
-            }
-            addWaitee(worker.makeDrvOutputSubstitutionGoal(depId));
-        }
-    }
-
+Goal::Co DrvOutputSubstitutionGoal::realisationFetched(std::shared_ptr<const Realisation> outputInfo, nix::ref<nix::Store> sub) {
     addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath));
 
-    if (waitees.empty()) outPathValid();
-    else state = &DrvOutputSubstitutionGoal::outPathValid;
-}
+    if (!waitees.empty()) co_await Suspend{};
 
-void DrvOutputSubstitutionGoal::outPathValid()
-{
-    assert(outputInfo);
     trace("output path substituted");
 
     if (nrFailed > 0) {
         debug("The output path of the derivation output '%s' could not be substituted", id.to_string());
-        amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed);
-        return;
+        co_return amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed);
     }
 
     worker.store.registerDrvOutput(*outputInfo);
-    finished();
-}
 
-void DrvOutputSubstitutionGoal::finished()
-{
     trace("finished");
-    amDone(ecSuccess);
+    co_return amDone(ecSuccess);
 }
 
 std::string DrvOutputSubstitutionGoal::key()
@@ -163,14 +152,9 @@ std::string DrvOutputSubstitutionGoal::key()
     return "a$" + std::string(id.to_string());
 }
 
-void DrvOutputSubstitutionGoal::work()
-{
-    (this->*state)();
-}
-
 void DrvOutputSubstitutionGoal::handleEOF(Descriptor fd)
 {
-    if (fd == downloadState->outPipe.readSide.get()) worker.wakeUp(shared_from_this());
+    worker.wakeUp(shared_from_this());
 }
 
 

src/libstore/build/drv-output-substitution-goal.hh

@@ -27,52 +27,19 @@ class DrvOutputSubstitutionGoal : public Goal {
      */
     DrvOutput id;
 
-    /**
-     * The realisation corresponding to the given output id.
-     * Will be filled once we can get it.
-     */
-    std::shared_ptr<const Realisation> outputInfo;
-
-    /**
-     * The remaining substituters.
-     */
-    std::list<ref<Store>> subs;
-
-    /**
-     * The current substituter.
-     */
-    std::shared_ptr<Store> sub;
-
-    struct DownloadState
-    {
-        MuxablePipe outPipe;
-        std::promise<std::shared_ptr<const Realisation>> promise;
-    };
-
-    std::shared_ptr<DownloadState> downloadState;
-
-    /**
-     * Whether a substituter failed.
-     */
-    bool substituterFailed = false;
-
 public:
     DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
 
     typedef void (DrvOutputSubstitutionGoal::*GoalState)();
     GoalState state;
 
-    void init();
-    void tryNext();
-    void realisationFetched();
-    void outPathValid();
-    void finished();
+    Co init() override;
+    Co realisationFetched(std::shared_ptr<const Realisation> outputInfo, nix::ref<nix::Store> sub);
 
     void timedOut(Error && ex) override { abort(); };
 
     std::string key() override;
 
-    void work() override;
     void handleEOF(Descriptor fd) override;
 
     JobCategory jobCategory() const override {

src/libstore/build/entry-points.cc

@@ -4,6 +4,7 @@
 #  include "derivation-goal.hh"
 #endif
 #include "local-store.hh"
+#include "strings.hh"
 
 namespace nix {
 

src/libstore/build/goal.cc

@@ -3,6 +3,97 @@
 
 namespace nix {
 
+using Co = nix::Goal::Co;
+using promise_type = nix::Goal::promise_type;
+using handle_type = nix::Goal::handle_type;
+using Suspend = nix::Goal::Suspend;
+
+Co::Co(Co&& rhs) {
+    this->handle = rhs.handle;
+    rhs.handle = nullptr;
+}
+void Co::operator=(Co&& rhs) {
+    this->handle = rhs.handle;
+    rhs.handle = nullptr;
+}
+Co::~Co() {
+    if (handle) {
+        handle.promise().alive = false;
+        handle.destroy();
+    }
+}
+
+Co promise_type::get_return_object() {
+    auto handle = handle_type::from_promise(*this);
+    return Co{handle};
+};
+
+std::coroutine_handle<> promise_type::final_awaiter::await_suspend(handle_type h) noexcept {
+    auto& p = h.promise();
+    auto goal = p.goal;
+    assert(goal);
+    goal->trace("in final_awaiter");
+    auto c = std::move(p.continuation);
+
+    if (c) {
+        // We still have a continuation, i.e. work to do.
+        // We assert that the goal is still busy.
+        assert(goal->exitCode == ecBusy);
+        assert(goal->top_co); // Goal must have an active coroutine.
+        assert(goal->top_co->handle == h); // The active coroutine must be us.
+        assert(p.alive); // We must not have been destructed.
+
+        // we move continuation to the top,
+        // note: previous top_co is actually h, so by moving into it,
+        // we're calling the destructor on h, DON'T use h and p after this!
+
+        // We move our continuation into `top_co`, i.e. the marker for the active continuation.
+        // By doing this we destruct the old `top_co`, i.e. us, so `h` can't be used anymore.
+        // Be careful not to access freed memory!
+        goal->top_co = std::move(c);
+
+        // We resume `top_co`.
+        return goal->top_co->handle;
+    } else {
+        // We have no continuation, i.e. no more work to do,
+        // so the goal must not be busy anymore.
+        assert(goal->exitCode != ecBusy);
+
+        // We reset `top_co` for good measure.
+        p.goal->top_co = {};
+
+        // We jump to the noop coroutine, which doesn't do anything and immediately suspends.
+        // This passes control back to the caller of goal.work().
+        return std::noop_coroutine();
+    }
+}
+
+void promise_type::return_value(Co&& next) {
+    goal->trace("return_value(Co&&)");
+    // Save old continuation.
+    auto old_continuation = std::move(continuation);
+    // We set next as our continuation.
+    continuation = std::move(next);
+    // We set next's goal, and thus it must not have one already.
+    assert(!continuation->handle.promise().goal);
+    continuation->handle.promise().goal = goal;
+    // Nor can next have a continuation, as we set it to our old one.
+    assert(!continuation->handle.promise().continuation);
+    continuation->handle.promise().continuation = std::move(old_continuation);
+}
+
+std::coroutine_handle<> nix::Goal::Co::await_suspend(handle_type caller) {
+    assert(handle); // we must be a valid coroutine
+    auto& p = handle.promise();
+    assert(!p.continuation); // we must have no continuation
+    assert(!p.goal); // we must not have a goal yet
+    auto goal = caller.promise().goal;
+    assert(goal);
+    p.goal = goal;
+    p.continuation = std::move(goal->top_co); // we set our continuation to be top_co (i.e. caller)
+    goal->top_co = std::move(*this); // we set top_co to ourselves, don't use this anymore after this!
+    return p.goal->top_co->handle; // we execute ourselves
+}
 
 bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
     std::string s1 = a->key();
@@ -75,10 +166,10 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
     }
 }
 
-
-void Goal::amDone(ExitCode result, std::optional<Error> ex)
+Goal::Done Goal::amDone(ExitCode result, std::optional<Error> ex)
 {
     trace("done");
+    assert(top_co);
     assert(exitCode == ecBusy);
     assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure);
     exitCode = result;
@@ -98,6 +189,13 @@ void Goal::amDone(ExitCode result, std::optional<Error> ex)
     worker.removeGoal(shared_from_this());
 
     cleanup();
+
+    // We drop the continuation.
+    // In `final_awaiter` this will signal that there is no more work to be done.
+    top_co->handle.promise().continuation = {};
+
+    // won't return to caller because of logic in final_awaiter
+    return Done{};
 }
 
 
@@ -106,4 +204,16 @@ void Goal::trace(std::string_view s)
     debug("%1%: %2%", name, s);
 }
 
+void Goal::work()
+{
+    assert(top_co);
+    assert(top_co->handle);
+    assert(top_co->handle.promise().alive);
+    top_co->handle.resume();
+    // We either should be in a state where we can be work()-ed again,
+    // or we should be done.
+    assert(top_co || exitCode != ecBusy);
+}
+
+
 }

src/libstore/build/goal.hh

@@ -1,10 +1,11 @@
 #pragma once
 ///@file
 
-#include "types.hh"
 #include "store-api.hh"
 #include "build-result.hh"
 
+#include <coroutine>
+
 namespace nix {
 
 /**
@@ -103,9 +104,263 @@ protected:
      * Build result.
      */
     BuildResult buildResult;
-
 public:
 
+    /**
+     * Suspend our goal and wait until we get @ref work()-ed again.
+     * `co_await`-able by @ref Co.
+     */
+    struct Suspend {};
+
+    /**
+     * Return from the current coroutine and suspend our goal
+     * if we're not busy anymore, or jump to the next coroutine
+     * set to be executed/resumed.
+     */
+    struct Return {};
+
+    /**
+     * `co_return`-ing this will end the goal.
+     * If you're not inside a coroutine, you can safely discard this.
+     */
+    struct [[nodiscard]] Done {
+        private:
+        Done(){}
+
+        friend Goal;
+    };
+
+    // forward declaration of promise_type, see below
+    struct promise_type;
+
+    /**
+     * Handle to coroutine using @ref Co and @ref promise_type.
+     */
+    using handle_type = std::coroutine_handle<promise_type>;
+
+    /**
+     * C++20 coroutine wrapper for use in goal logic.
+     * Coroutines are functions that use `co_await`/`co_return` (and `co_yield`, but not supported by @ref Co).
+     *
+     * @ref Co is meant to be used by methods of subclasses of @ref Goal.
+     * The main functionality provided by `Co` is
+     * - `co_await Suspend{}`: Suspends the goal.
+     * - `co_await f()`: Waits until `f()` finishes.
+     * - `co_return f()`: Tail-calls `f()`.
+     * - `co_return Return{}`: Ends coroutine.
+     *
+     * The idea is that you implement the goal logic using coroutines,
+     * and do the core thing a goal can do, suspension, when you have
+     * children you're waiting for.
+     * Coroutines allow you to resume the work cleanly.
+     *
+     * @note Brief explanation of C++20 coroutines:
+     *       When you `Co f()`, a `std::coroutine_handle<promise_type>` is created,
+     *       alongside its @ref promise_type.
+     *       There are suspension points at the beginning of the coroutine,
+     *       at every `co_await`, and at the final (possibly implicit) `co_return`.
+     *       Once suspended, you can resume the `std::coroutine_handle` by doing `coroutine_handle.resume()`.
+     *       Suspension points are implemented by passing a struct to the compiler
+     *       that implements `await_sus`pend.
+     *       `await_suspend` can either say "cancel suspension", in which case execution resumes,
+     *       "suspend", in which case control is passed back to the caller of `coroutine_handle.resume()`
+     *       or the place where the coroutine function is initially executed in the case of the initial
+     *       suspension, or `await_suspend` can specify another coroutine to jump to, which is
+     *       how tail calls are implemented.
+     *
+     * @note Resources:
+     *       - https://lewissbaker.github.io/
+     *       - https://www.chiark.greenend.org.uk/~sgtatham/quasiblog/coroutines-c++20/
+     *       - https://www.scs.stanford.edu/~dm/blog/c++-coroutines.html
+     *
+     * @todo Allocate explicitly on stack since HALO thing doesn't really work,
+     *       specifically, there's no way to uphold the requirements when trying to do
+     *       tail-calls without using a trampoline AFAICT.
+     *
+     * @todo Support returning data natively
+     */
+    struct [[nodiscard]] Co {
+        /**
+         * The underlying handle.
+         */
+        handle_type handle;
+
+        explicit Co(handle_type handle) : handle(handle) {};
+        void operator=(Co&&);
+        Co(Co&& rhs);
+        ~Co();
+
+        bool await_ready() { return false; };
+        /**
+         * When we `co_await` another @ref Co-returning coroutine,
+         * we tell the caller of `caller_coroutine.resume()` to switch to our coroutine (@ref handle).
+         * To make sure we return to the original coroutine, we set it as the continuation of our
+         * coroutine. In @ref promise_type::final_awaiter we check if it's set and if so we return to it.
+         *
+         * To explain in more understandable terms:
+         * When we `co_await Co_returning_function()`, this function is called on the resultant @ref Co of
+         * the _called_ function, and C++ automatically passes the caller in.
+         *
+         * `goal` field of @ref promise_type is also set here by copying it from the caller.
+         */
+        std::coroutine_handle<> await_suspend(handle_type handle);
+        void await_resume() {};
+    };
+
+    /**
+     * Used on initial suspend, does the same as @ref std::suspend_always,
+     * but asserts that everything has been set correctly.
+     */
+    struct InitialSuspend {
+        /**
+         * Handle of coroutine that does the
+         * initial suspend
+         */
+        handle_type handle;
+
+        bool await_ready() { return false; };
+        void await_suspend(handle_type handle_) {
+            handle = handle_;
+        }
+        void await_resume() {
+            assert(handle);
+            assert(handle.promise().goal); // goal must be set
+            assert(handle.promise().goal->top_co); // top_co of goal must be set
+            assert(handle.promise().goal->top_co->handle == handle); // top_co of goal must be us
+        }
+    };
+
+    /**
+     * Promise type for coroutines defined using @ref Co.
+     * Attached to coroutine handle.
+     */
+    struct promise_type {
+        /**
+         * Either this is who called us, or it is who we will tail-call.
+         * It is what we "jump" to once we are done.
+         */
+        std::optional<Co> continuation;
+
+        /**
+         * The goal that we're a part of.
+         * Set either in @ref Co::await_suspend or in constructor of @ref Goal.
+         */
+        Goal* goal = nullptr;
+
+        /**
+         * Is set to false when destructed to ensure we don't use a
+         * destructed coroutine by accident
+         */
+        bool alive = true;
+
+        /**
+         * The awaiter used by @ref final_suspend.
+         */
+        struct final_awaiter {
+            bool await_ready() noexcept { return false; };
+            /**
+             * Here we execute our continuation, by passing it back to the caller.
+             * C++ compiler will create code that takes that and executes it promptly.
+             * `h` is the handle for the coroutine that is finishing execution,
+             * thus it must be destroyed.
+             */
+            std::coroutine_handle<> await_suspend(handle_type h) noexcept;
+            void await_resume() noexcept { assert(false); };
+        };
+
+        /**
+         * Called by compiler generated code to construct the @ref Co
+         * that is returned from a @ref Co-returning coroutine.
+         */
+        Co get_return_object();
+
+        /**
+         * Called by compiler generated code before body of coroutine.
+         * We use this opportunity to set the @ref goal field
+         * and `top_co` field of @ref Goal.
+         */
+        InitialSuspend initial_suspend() { return {}; };
+
+        /**
+         * Called on `co_return`. Creates @ref final_awaiter which
+         * either jumps to continuation or suspends goal.
+         */
+        final_awaiter final_suspend() noexcept { return {}; };
+
+        /**
+         * Does nothing, but provides an opportunity for
+         * @ref final_suspend to happen.
+         */
+        void return_value(Return) {}
+
+        /**
+         * Does nothing, but provides an opportunity for
+         * @ref final_suspend to happen.
+         */
+        void return_value(Done) {}
+
+        /**
+         * When "returning" another coroutine, what happens is that
+         * we set it as our own continuation, thus once the final suspend
+         * happens, we transfer control to it.
+         * The original continuation we had is set as the continuation
+         * of the coroutine passed in.
+         * @ref final_suspend is called after this, and @ref final_awaiter will
+         * pass control off to @ref continuation.
+         *
+         * If we already have a continuation, that continuation is set as
+         * the continuation of the new continuation. Thus, the continuation
+         * passed to @ref return_value must not have a continuation set.
+         */
+        void return_value(Co&&);
+
+        /**
+         * If an exception is thrown inside a coroutine,
+         * we re-throw it in the context of the "resumer" of the continuation.
+         */
+        void unhandled_exception() { throw; };
+
+        /**
+         * Allows awaiting a @ref Co.
+         */
+        Co&& await_transform(Co&& co) { return static_cast<Co&&>(co); }
+
+        /**
+         * Allows awaiting a @ref Suspend.
+         * Always suspends.
+         */
+        std::suspend_always await_transform(Suspend) { return {}; };
+    };
+
+    /**
+     * The coroutine being currently executed.
+     * MUST be updated when switching the coroutine being executed.
+     * This is used both for memory management and to resume the last
+     * coroutine executed.
+     * Destroying this should destroy all coroutines created for this goal.
+     */
+    std::optional<Co> top_co;
+
+    /**
+     * The entry point for the goal
+     */
+    virtual Co init() = 0;
+
+    /**
+     * Wrapper around @ref init since virtual functions
+     * can't be used in constructors.
+     */
+    inline Co init_wrapper();
+
+    /**
+     * Signals that the goal is done.
+     * `co_return` the result. If you're not inside a coroutine, you can ignore
+     * the return value safely.
+     */
+    Done amDone(ExitCode result, std::optional<Error> ex = {});
+
+    virtual void cleanup() { }
+
     /**
      * Project a `BuildResult` with just the information that pertains
      * to the given request.
@@ -124,15 +379,20 @@ public:
     std::optional<Error> ex;
 
     Goal(Worker & worker, DerivedPath path)
-        : worker(worker)
-    { }
+        : worker(worker), top_co(init_wrapper())
+    {
+        // top_co shouldn't have a goal already, should be nullptr.
+        assert(!top_co->handle.promise().goal);
+        // we set it such that top_co can pass it down to its subcoroutines.
+        top_co->handle.promise().goal = this;
+    }
 
     virtual ~Goal()
     {
         trace("goal destroyed");
     }
 
-    virtual void work() = 0;
+    void work();
 
     void addWaitee(GoalPtr waitee);
 
@@ -164,10 +424,6 @@ public:
 
     virtual std::string key() = 0;
 
-    void amDone(ExitCode result, std::optional<Error> ex = {});
-
-    virtual void cleanup() { }
-
     /**
      * @brief Hint for the scheduler, which concurrency limit applies.
      * @see JobCategory
@@ -178,3 +434,12 @@ public:
 void addToWeakGoals(WeakGoals & goals, GoalPtr p);
 
 }
+
+template<typename... ArgTypes>
+struct std::coroutine_traits<nix::Goal::Co, ArgTypes...> {
+    using promise_type = nix::Goal::promise_type;
+};
+
+nix::Goal::Co nix::Goal::init_wrapper() {
+    co_return init();
+}

src/libstore/build/substitution-goal.cc

@@ -3,6 +3,7 @@
 #include "nar-info.hh"
 #include "finally.hh"
 #include "signals.hh"
+#include <coroutine>
 
 namespace nix {
 
@@ -12,7 +13,6 @@ PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker &
     , repair(repair)
     , ca(ca)
 {
-    state = &PathSubstitutionGoal::init;
     name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath));
     trace("created");
     maintainExpectedSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.expectedSubstitutions);
@@ -25,7 +25,7 @@ PathSubstitutionGoal::~PathSubstitutionGoal()
 }
 
 
-void PathSubstitutionGoal::done(
+Goal::Done PathSubstitutionGoal::done(
     ExitCode result,
     BuildResult::Status status,
     std::optional<std::string> errorMsg)
@@ -35,17 +35,11 @@ void PathSubstitutionGoal::done(
         debug(*errorMsg);
         buildResult.errorMsg = *errorMsg;
     }
-    amDone(result);
+    return amDone(result);
 }
 
 
-void PathSubstitutionGoal::work()
-{
-    (this->*state)();
-}
-
-
-void PathSubstitutionGoal::init()
+Goal::Co PathSubstitutionGoal::init()
 {
     trace("init");
 
@@ -53,152 +47,135 @@ void PathSubstitutionGoal::init()
 
     /* If the path already exists we're done. */
     if (!repair && worker.store.isValidPath(storePath)) {
-        done(ecSuccess, BuildResult::AlreadyValid);
-        return;
+        co_return done(ecSuccess, BuildResult::AlreadyValid);
     }
 
     if (settings.readOnlyMode)
         throw Error("cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath));
 
-    subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
+    auto subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
 
-    tryNext();
-}
+    bool substituterFailed = false;
 
+    for (auto sub : subs) {
+        trace("trying next substituter");
 
-void PathSubstitutionGoal::tryNext()
-{
-    trace("trying next substituter");
+        cleanup();
 
-    cleanup();
+        /* The path the substituter refers to the path as. This will be
+         * different when the stores have different names. */
+        std::optional<StorePath> subPath;
 
-    if (subs.size() == 0) {
-        /* None left.  Terminate this goal and let someone else deal
-           with it. */
+        /* Path info returned by the substituter's query info operation. */
+        std::shared_ptr<const ValidPathInfo> info;
 
-        /* Hack: don't indicate failure if there were no substituters.
-           In that case the calling derivation should just do a
-           build. */
-        done(
-            substituterFailed ? ecFailed : ecNoSubstituters,
-            BuildResult::NoSubstituters,
-            fmt("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath)));
-
-        if (substituterFailed) {
-            worker.failedSubstitutions++;
-            worker.updateProgress();
+        if (ca) {
+            subPath = sub->makeFixedOutputPathFromCA(
+                std::string { storePath.name() },
+                ContentAddressWithReferences::withoutRefs(*ca));
+            if (sub->storeDir == worker.store.storeDir)
+                assert(subPath == storePath);
+        } else if (sub->storeDir != worker.store.storeDir) {
+            continue;
         }
 
-        return;
-    }
-
-    sub = subs.front();
-    subs.pop_front();
-
-    if (ca) {
-        subPath = sub->makeFixedOutputPathFromCA(
-            std::string { storePath.name() },
-            ContentAddressWithReferences::withoutRefs(*ca));
-        if (sub->storeDir == worker.store.storeDir)
-            assert(subPath == storePath);
-    } else if (sub->storeDir != worker.store.storeDir) {
-        tryNext();
-        return;
-    }
-
-    try {
-        // FIXME: make async
-        info = sub->queryPathInfo(subPath ? *subPath : storePath);
-    } catch (InvalidPath &) {
-        tryNext();
-        return;
-    } catch (SubstituterDisabled &) {
-        if (settings.tryFallback) {
-            tryNext();
-            return;
+        try {
+            // FIXME: make async
+            info = sub->queryPathInfo(subPath ? *subPath : storePath);
+        } catch (InvalidPath &) {
+            continue;
+        } catch (SubstituterDisabled & e) {
+            if (settings.tryFallback) continue;
+            else throw e;
+        } catch (Error & e) {
+            if (settings.tryFallback) {
+                logError(e.info());
+                continue;
+            } else throw e;
         }
-        throw;
-    } catch (Error & e) {
-        if (settings.tryFallback) {
-            logError(e.info());
-            tryNext();
-            return;
+
+        if (info->path != storePath) {
+            if (info->isContentAddressed(*sub) && info->references.empty()) {
+                auto info2 = std::make_shared<ValidPathInfo>(*info);
+                info2->path = storePath;
+                info = info2;
+            } else {
+                printError("asked '%s' for '%s' but got '%s'",
+                    sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path));
+                continue;
+            }
         }
-        throw;
+
+        /* Update the total expected download size. */
+        auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info);
+
+        maintainExpectedNar = std::make_unique<MaintainCount<uint64_t>>(worker.expectedNarSize, info->narSize);
+
+        maintainExpectedDownload =
+            narInfo && narInfo->fileSize
+            ? std::make_unique<MaintainCount<uint64_t>>(worker.expectedDownloadSize, narInfo->fileSize)
+            : nullptr;
+
+        worker.updateProgress();
+
+        /* Bail out early if this substituter lacks a valid
+           signature. LocalStore::addToStore() also checks for this, but
+           only after we've downloaded the path. */
+        if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info))
+        {
+            warn("ignoring substitute for '%s' from '%s', as it's not signed by any of the keys in 'trusted-public-keys'",
+                worker.store.printStorePath(storePath), sub->getUri());
+            continue;
+        }
+
+        /* To maintain the closure invariant, we first have to realise the
+           paths referenced by this one. */
+        for (auto & i : info->references)
+            if (i != storePath) /* ignore self-references */
+                addWaitee(worker.makePathSubstitutionGoal(i));
+
+        if (!waitees.empty()) co_await Suspend{};
+
+        // FIXME: consider returning boolean instead of passing in reference
+        bool out = false; // is mutated by tryToRun
+        co_await tryToRun(subPath ? *subPath : storePath, sub, info, out);
+        substituterFailed = substituterFailed || out;
     }
 
-    if (info->path != storePath) {
-        if (info->isContentAddressed(*sub) && info->references.empty()) {
-            auto info2 = std::make_shared<ValidPathInfo>(*info);
-            info2->path = storePath;
-            info = info2;
-        } else {
-            printError("asked '%s' for '%s' but got '%s'",
-                sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path));
-            tryNext();
-            return;
-        }
-    }
-
-    /* Update the total expected download size. */
-    auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info);
-
-    maintainExpectedNar = std::make_unique<MaintainCount<uint64_t>>(worker.expectedNarSize, info->narSize);
-
-    maintainExpectedDownload =
-        narInfo && narInfo->fileSize
-        ? std::make_unique<MaintainCount<uint64_t>>(worker.expectedDownloadSize, narInfo->fileSize)
-        : nullptr;
+    /* None left.  Terminate this goal and let someone else deal
+       with it. */
 
+    worker.failedSubstitutions++;
     worker.updateProgress();
 
-    /* Bail out early if this substituter lacks a valid
-       signature. LocalStore::addToStore() also checks for this, but
-       only after we've downloaded the path. */
-    if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info))
-    {
-        warn("ignoring substitute for '%s' from '%s', as it's not signed by any of the keys in 'trusted-public-keys'",
-            worker.store.printStorePath(storePath), sub->getUri());
-        tryNext();
-        return;
-    }
-
-    /* To maintain the closure invariant, we first have to realise the
-       paths referenced by this one. */
-    for (auto & i : info->references)
-        if (i != storePath) /* ignore self-references */
-            addWaitee(worker.makePathSubstitutionGoal(i));
-
-    if (waitees.empty()) /* to prevent hang (no wake-up event) */
-        referencesValid();
-    else
-        state = &PathSubstitutionGoal::referencesValid;
+    /* Hack: don't indicate failure if there were no substituters.
+       In that case the calling derivation should just do a
+       build. */
+    co_return done(
+        substituterFailed ? ecFailed : ecNoSubstituters,
+        BuildResult::NoSubstituters,
+        fmt("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath)));
 }
 
 
-void PathSubstitutionGoal::referencesValid()
+Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref<Store> sub, std::shared_ptr<const ValidPathInfo> info, bool& substituterFailed)
 {
     trace("all references realised");
 
     if (nrFailed > 0) {
-        done(
+        co_return done(
             nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed,
             BuildResult::DependencyFailed,
             fmt("some references of path '%s' could not be realised", worker.store.printStorePath(storePath)));
-        return;
     }
 
     for (auto & i : info->references)
         if (i != storePath) /* ignore self-references */
             assert(worker.store.isValidPath(i));
 
-    state = &PathSubstitutionGoal::tryToRun;
     worker.wakeUp(shared_from_this());
-}
+    co_await Suspend{};
 
-
-void PathSubstitutionGoal::tryToRun()
-{
     trace("trying to run");
 
     /* Make sure that we are allowed to start a substitution.  Note that even
@@ -206,10 +183,10 @@ void PathSubstitutionGoal::tryToRun()
        prevents infinite waiting. */
     if (worker.getNrSubstitutions() >= std::max(1U, (unsigned int) settings.maxSubstitutionJobs)) {
         worker.waitForBuildSlot(shared_from_this());
-        return;
+        co_await Suspend{};
     }
 
-    maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions);
+    auto maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions);
     worker.updateProgress();
 
 #ifndef _WIN32
@@ -218,9 +195,9 @@ void PathSubstitutionGoal::tryToRun()
     outPipe.createAsyncPipe(worker.ioport.get());
 #endif
 
-    promise = std::promise<void>();
+    auto promise = std::promise<void>();
 
-    thr = std::thread([this]() {
+    thr = std::thread([this, &promise, &subPath, &sub]() {
         try {
             ReceiveInterrupts receiveInterrupts;
 
@@ -231,7 +208,7 @@ void PathSubstitutionGoal::tryToRun()
             PushActivity pact(act.id);
 
             copyStorePath(*sub, worker.store,
-                subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
+                subPath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
 
             promise.set_value();
         } catch (...) {
@@ -247,12 +224,8 @@ void PathSubstitutionGoal::tryToRun()
 #endif
     }, true, false);
 
-    state = &PathSubstitutionGoal::finished;
-}
+    co_await Suspend{};
 
-
-void PathSubstitutionGoal::finished()
-{
     trace("substitute finished");
 
     thr.join();
@@ -274,10 +247,7 @@ void PathSubstitutionGoal::finished()
             substituterFailed = true;
         }
 
-        /* Try the next substitute. */
-        state = &PathSubstitutionGoal::tryNext;
-        worker.wakeUp(shared_from_this());
-        return;
+        co_return Return{};
     }
 
     worker.markContentsGood(storePath);
@@ -295,23 +265,19 @@ void PathSubstitutionGoal::finished()
         worker.doneDownloadSize += fileSize;
     }
 
+    assert(maintainExpectedNar);
     worker.doneNarSize += maintainExpectedNar->delta;
     maintainExpectedNar.reset();
 
     worker.updateProgress();
 
-    done(ecSuccess, BuildResult::Substituted);
-}
-
-
-void PathSubstitutionGoal::handleChildOutput(Descriptor fd, std::string_view data)
-{
+    co_return done(ecSuccess, BuildResult::Substituted);
 }
 
 
 void PathSubstitutionGoal::handleEOF(Descriptor fd)
 {
-    if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this());
+    worker.wakeUp(shared_from_this());
 }
 
 

src/libstore/build/substitution-goal.hh

@@ -1,14 +1,16 @@
 #pragma once
 ///@file
 
+#include "worker.hh"
 #include "store-api.hh"
 #include "goal.hh"
 #include "muxable-pipe.hh"
+#include <coroutine>
+#include <future>
+#include <source_location>
 
 namespace nix {
 
-class Worker;
-
 struct PathSubstitutionGoal : public Goal
 {
     /**
@@ -17,30 +19,9 @@ struct PathSubstitutionGoal : public Goal
     StorePath storePath;
 
     /**
-     * The path the substituter refers to the path as. This will be
-     * different when the stores have different names.
+     * Whether to try to repair a valid path.
      */
-    std::optional<StorePath> subPath;
-
-    /**
-     * The remaining substituters.
-     */
-    std::list<ref<Store>> subs;
-
-    /**
-     * The current substituter.
-     */
-    std::shared_ptr<Store> sub;
-
-    /**
-     * Whether a substituter failed.
-     */
-    bool substituterFailed = false;
-
-    /**
-     * Path info returned by the substituter's query info operation.
-     */
-    std::shared_ptr<const ValidPathInfo> info;
+    RepairFlag repair;
 
     /**
      * Pipe for the substituter's standard output.
@@ -52,31 +33,15 @@ struct PathSubstitutionGoal : public Goal
      */
     std::thread thr;
 
-    std::promise<void> promise;
-
-    /**
-     * Whether to try to repair a valid path.
-     */
-    RepairFlag repair;
-
-    /**
-     * Location where we're downloading the substitute.  Differs from
-     * storePath when doing a repair.
-     */
-    Path destPath;
-
     std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
         maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload;
 
-    typedef void (PathSubstitutionGoal::*GoalState)();
-    GoalState state;
-
     /**
      * Content address for recomputing store path
      */
     std::optional<ContentAddress> ca;
 
-    void done(
+    Done done(
         ExitCode result,
         BuildResult::Status status,
         std::optional<std::string> errorMsg = {});
@@ -96,22 +61,18 @@ public:
         return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath);
     }
 
-    void work() override;
-
     /**
      * The states.
      */
-    void init();
-    void tryNext();
-    void gotInfo();
-    void referencesValid();
-    void tryToRun();
-    void finished();
+    Co init() override;
+    Co gotInfo();
+    Co tryToRun(StorePath subPath, nix::ref<Store> sub, std::shared_ptr<const ValidPathInfo> info, bool& substituterFailed);
+    Co finished();
 
     /**
      * Callback used by the worker to write to the log.
      */
-    void handleChildOutput(Descriptor fd, std::string_view data) override;
+    void handleChildOutput(Descriptor fd, std::string_view data) override {};
     void handleEOF(Descriptor fd) override;
 
     /* Called by destructor, can't be overridden */

src/libstore/build/worker.cc

@@ -337,31 +337,27 @@ void Worker::run(const Goals & _topGoals)
         /* Wait for input. */
         if (!children.empty() || !waitingForAWhile.empty())
             waitForInput();
-        else {
-            if (awake.empty() && 0U == settings.maxBuildJobs)
-            {
-                if (getMachines().empty())
-                   throw Error(
-                        R"(
-                        Unable to start any build;
-                        either increase '--max-jobs' or enable remote builds.
+        else if (awake.empty() && 0U == settings.maxBuildJobs) {
+            if (getMachines().empty())
+               throw Error(
+                    R"(
+                    Unable to start any build;
+                    either increase '--max-jobs' or enable remote builds.
 
-                        For more information run 'man nix.conf' and search for '/machines'.
-                        )"
-                    );
-                else
-                   throw Error(
-                        R"(
-                        Unable to start any build;
-                        remote machines may not have all required system features.
+                    For more information run 'man nix.conf' and search for '/machines'.
+                    )"
+                );
+            else
+               throw Error(
+                    R"(
+                    Unable to start any build;
+                    remote machines may not have all required system features.
 
-                        For more information run 'man nix.conf' and search for '/machines'.
-                        )"
-                    );
+                    For more information run 'man nix.conf' and search for '/machines'.
+                    )"
+                );
 
-            }
-            assert(!awake.empty());
-        }
+        } else assert(!awake.empty());
     }
 
     /* If --keep-going is not set, it's possible that the main goal

src/libstore/common-ssh-store-config.cc

@@ -1,6 +1,6 @@
 #include <regex>
 
-#include "ssh-store-config.hh"
+#include "common-ssh-store-config.hh"
 #include "ssh.hh"
 
 namespace nix {

src/libstore/content-address.hh

@@ -73,6 +73,7 @@ struct ContentAddressMethod
 
     Raw raw;
 
+    bool operator ==(const ContentAddressMethod &) const = default;
     auto operator <=>(const ContentAddressMethod &) const = default;
 
     MAKE_WRAPPER_CONSTRUCTOR(ContentAddressMethod);
@@ -159,6 +160,7 @@ struct ContentAddress
      */
     Hash hash;
 
+    bool operator ==(const ContentAddress &) const = default;
     auto operator <=>(const ContentAddress &) const = default;
 
     /**
@@ -218,7 +220,9 @@ struct StoreReferences
      */
     size_t size() const;
 
-    auto operator <=>(const StoreReferences &) const = default;
+    bool operator ==(const StoreReferences &) const = default;
+    // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet.
+    //auto operator <=>(const StoreReferences &) const = default;
 };
 
 // This matches the additional info that we need for makeTextPath
@@ -235,7 +239,9 @@ struct TextInfo
      */
     StorePathSet references;
 
-    auto operator <=>(const TextInfo &) const = default;
+    bool operator ==(const TextInfo &) const = default;
+    // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet.
+    //auto operator <=>(const TextInfo &) const = default;
 };
 
 struct FixedOutputInfo
@@ -255,7 +261,9 @@ struct FixedOutputInfo
      */
     StoreReferences references;
 
-    auto operator <=>(const FixedOutputInfo &) const = default;
+    bool operator ==(const FixedOutputInfo &) const = default;
+    // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet.
+    //auto operator <=>(const FixedOutputInfo &) const = default;
 };
 
 /**
@@ -272,7 +280,9 @@ struct ContentAddressWithReferences
 
     Raw raw;
 
-    auto operator <=>(const ContentAddressWithReferences &) const = default;
+    bool operator ==(const ContentAddressWithReferences &) const = default;
+    // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet.
+    //auto operator <=>(const ContentAddressWithReferences &) const = default;
 
     MAKE_WRAPPER_CONSTRUCTOR(ContentAddressWithReferences);
 

src/libstore/derivations.cc

@@ -10,6 +10,8 @@
 #include <boost/container/small_vector.hpp>
 #include <nlohmann/json.hpp>
 
+#include "strings-inline.hh"
+
 namespace nix {
 
 std::optional<StorePath> DerivationOutput::path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const

src/libstore/derivations.hh

@@ -8,13 +8,11 @@
 #include "repair-flag.hh"
 #include "derived-path-map.hh"
 #include "sync.hh"
-#include "comparator.hh"
 #include "variant-wrapper.hh"
 
 #include <map>
 #include <variant>
 
-
 namespace nix {
 
 struct StoreDirConfig;
@@ -33,7 +31,8 @@ struct DerivationOutput
     {
         StorePath path;
 
-        GENERATE_CMP(InputAddressed, me->path);
+        bool operator == (const InputAddressed &) const = default;
+        auto operator <=> (const InputAddressed &) const = default;
     };
 
     /**
@@ -57,7 +56,8 @@ struct DerivationOutput
          */
         StorePath path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const;
 
-        GENERATE_CMP(CAFixed, me->ca);
+        bool operator == (const CAFixed &) const = default;
+        auto operator <=> (const CAFixed &) const = default;
     };
 
     /**
@@ -77,7 +77,8 @@ struct DerivationOutput
          */
         HashAlgorithm hashAlgo;
 
-        GENERATE_CMP(CAFloating, me->method, me->hashAlgo);
+        bool operator == (const CAFloating &) const = default;
+        auto operator <=> (const CAFloating &) const = default;
     };
 
     /**
@@ -85,7 +86,8 @@ struct DerivationOutput
      * isn't known yet.
      */
     struct Deferred {
-        GENERATE_CMP(Deferred);
+        bool operator == (const Deferred &) const = default;
+        auto operator <=> (const Deferred &) const = default;
     };
 
     /**
@@ -104,7 +106,8 @@ struct DerivationOutput
          */
         HashAlgorithm hashAlgo;
 
-        GENERATE_CMP(Impure, me->method, me->hashAlgo);
+        bool operator == (const Impure &) const = default;
+        auto operator <=> (const Impure &) const = default;
     };
 
     typedef std::variant<
@@ -117,7 +120,8 @@ struct DerivationOutput
 
     Raw raw;
 
-    GENERATE_CMP(DerivationOutput, me->raw);
+    bool operator == (const DerivationOutput &) const = default;
+    auto operator <=> (const DerivationOutput &) const = default;
 
     MAKE_WRAPPER_CONSTRUCTOR(DerivationOutput);
 
@@ -178,7 +182,8 @@ struct DerivationType {
         */
         bool deferred;
 
-        GENERATE_CMP(InputAddressed, me->deferred);
+        bool operator == (const InputAddressed &) const = default;
+        auto operator <=> (const InputAddressed &) const = default;
     };
 
     /**
@@ -202,7 +207,8 @@ struct DerivationType {
          */
         bool fixed;
 
-        GENERATE_CMP(ContentAddressed, me->sandboxed, me->fixed);
+        bool operator == (const ContentAddressed &) const = default;
+        auto operator <=> (const ContentAddressed &) const = default;
     };
 
     /**
@@ -212,7 +218,8 @@ struct DerivationType {
      * type, but has some restrictions on its usage.
      */
     struct Impure {
-        GENERATE_CMP(Impure);
+        bool operator == (const Impure &) const = default;
+        auto operator <=> (const Impure &) const = default;
     };
 
     typedef std::variant<
@@ -223,7 +230,8 @@ struct DerivationType {
 
     Raw raw;
 
-    GENERATE_CMP(DerivationType, me->raw);
+    bool operator == (const DerivationType &) const = default;
+    auto operator <=> (const DerivationType &) const = default;
 
     MAKE_WRAPPER_CONSTRUCTOR(DerivationType);
 
@@ -313,14 +321,9 @@ struct BasicDerivation
 
     static std::string_view nameFromPath(const StorePath & storePath);
 
-    GENERATE_CMP(BasicDerivation,
-        me->outputs,
-        me->inputSrcs,
-        me->platform,
-        me->builder,
-        me->args,
-        me->env,
-        me->name);
+    bool operator == (const BasicDerivation &) const = default;
+    // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet.
+    //auto operator <=> (const BasicDerivation &) const = default;
 };
 
 class Store;
@@ -378,9 +381,9 @@ struct Derivation : BasicDerivation
         const nlohmann::json & json,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
-    GENERATE_CMP(Derivation,
-        static_cast<const BasicDerivation &>(*me),
-        me->inputDrvs);
+    bool operator == (const Derivation &) const = default;
+    // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet.
+    //auto operator <=> (const Derivation &) const = default;
 };
 
 

src/libstore/derived-path-map.cc

@@ -54,17 +54,18 @@ typename DerivedPathMap<V>::ChildNode * DerivedPathMap<V>::findSlot(const Single
 
 namespace nix {
 
-GENERATE_CMP_EXT(
-    template<>,
-    DerivedPathMap<std::set<std::string>>::ChildNode,
-    me->value,
-    me->childMap);
+template<>
+bool DerivedPathMap<std::set<std::string>>::ChildNode::operator == (
+    const DerivedPathMap<std::set<std::string>>::ChildNode &) const noexcept = default;
 
-GENERATE_CMP_EXT(
-    template<>,
-    DerivedPathMap<std::set<std::string>>,
-    me->map);
+// TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet.
+#if 0
+template<>
+std::strong_ordering DerivedPathMap<std::set<std::string>>::ChildNode::operator <=> (
+    const DerivedPathMap<std::set<std::string>>::ChildNode &) const noexcept = default;
+#endif
 
+template struct DerivedPathMap<std::set<std::string>>::ChildNode;
 template struct DerivedPathMap<std::set<std::string>>;
 
 };

src/libstore/derived-path-map.hh

@@ -47,7 +47,11 @@ struct DerivedPathMap {
          */
         Map childMap;
 
-        DECLARE_CMP(ChildNode);
+        bool operator == (const ChildNode &) const noexcept;
+
+        // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet.
+        // decltype(std::declval<V>() <=> std::declval<V>())
+        // operator <=> (const ChildNode &) const noexcept;
     };
 
     /**
@@ -60,7 +64,10 @@ struct DerivedPathMap {
      */
     Map map;
 
-    DECLARE_CMP(DerivedPathMap);
+    bool operator == (const DerivedPathMap &) const = default;
+
+    // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet.
+    // auto operator <=> (const DerivedPathMap &) const noexcept;
 
     /**
      * Find the node for `k`, creating it if needed.
@@ -83,14 +90,21 @@ struct DerivedPathMap {
     ChildNode * findSlot(const SingleDerivedPath & k);
 };
 
+template<>
+bool DerivedPathMap<std::set<std::string>>::ChildNode::operator == (
+    const DerivedPathMap<std::set<std::string>>::ChildNode &) const noexcept;
 
-DECLARE_CMP_EXT(
-    template<>,
-    DerivedPathMap<std::set<std::string>>::,
-    DerivedPathMap<std::set<std::string>>);
-DECLARE_CMP_EXT(
-    template<>,
-    DerivedPathMap<std::set<std::string>>::ChildNode::,
-    DerivedPathMap<std::set<std::string>>::ChildNode);
+// TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet.
+#if 0
+template<>
+std::strong_ordering DerivedPathMap<std::set<std::string>>::ChildNode::operator <=> (
+    const DerivedPathMap<std::set<std::string>>::ChildNode &) const noexcept;
+
+template<>
+inline auto DerivedPathMap<std::set<std::string>>::operator <=> (const DerivedPathMap<std::set<std::string>> &) const noexcept = default;
+#endif
+
+extern template struct DerivedPathMap<std::set<std::string>>::ChildNode;
+extern template struct DerivedPathMap<std::set<std::string>>;
 
 }

src/libstore/derived-path.cc

@@ -1,6 +1,7 @@
 #include "derived-path.hh"
 #include "derivations.hh"
 #include "store-api.hh"
+#include "comparator.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -8,26 +9,32 @@
 
 namespace nix {
 
-#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \
-    bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \
-    { \
-        const MY_TYPE* me = this; \
-        auto fields1 = std::tie(*me->drvPath, me->FIELD); \
-        me = &other; \
-        auto fields2 = std::tie(*me->drvPath, me->FIELD); \
-        return fields1 COMPARATOR fields2; \
-    }
-#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \
-    CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \
-    CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \
-    CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <)
+// Custom implementation to avoid `ref` ptr equality
+GENERATE_CMP_EXT(
+    ,
+    std::strong_ordering,
+    SingleDerivedPathBuilt,
+    *me->drvPath,
+    me->output);
 
-CMP(SingleDerivedPath, SingleDerivedPathBuilt, output)
+// Custom implementation to avoid `ref` ptr equality
 
-CMP(SingleDerivedPath, DerivedPathBuilt, outputs)
-
-#undef CMP
-#undef CMP_ONE
+// TODO no `GENERATE_CMP_EXT` because no `std::set::operator<=>` on
+// Darwin, per header.
+GENERATE_EQUAL(
+    ,
+    DerivedPathBuilt ::,
+    DerivedPathBuilt,
+    *me->drvPath,
+    me->outputs);
+GENERATE_ONE_CMP(
+    ,
+    bool,
+    DerivedPathBuilt ::,
+    <,
+    DerivedPathBuilt,
+    *me->drvPath,
+    me->outputs);
 
 nlohmann::json DerivedPath::Opaque::toJSON(const StoreDirConfig & store) const
 {

src/libstore/derived-path.hh

@@ -3,8 +3,8 @@
 
 #include "path.hh"
 #include "outputs-spec.hh"
-#include "comparator.hh"
 #include "config.hh"
+#include "ref.hh"
 
 #include <variant>
 
@@ -31,7 +31,8 @@ struct DerivedPathOpaque {
     static DerivedPathOpaque parse(const StoreDirConfig & store, std::string_view);
     nlohmann::json toJSON(const StoreDirConfig & store) const;
 
-    GENERATE_CMP(DerivedPathOpaque, me->path);
+    bool operator == (const DerivedPathOpaque &) const = default;
+    auto operator <=> (const DerivedPathOpaque &) const = default;
 };
 
 struct SingleDerivedPath;
@@ -78,7 +79,8 @@ struct SingleDerivedPathBuilt {
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
     nlohmann::json toJSON(Store & store) const;
 
-    DECLARE_CMP(SingleDerivedPathBuilt);
+    bool operator == (const SingleDerivedPathBuilt &) const noexcept;
+    std::strong_ordering operator <=> (const SingleDerivedPathBuilt &) const noexcept;
 };
 
 using _SingleDerivedPathRaw = std::variant<
@@ -108,6 +110,9 @@ struct SingleDerivedPath : _SingleDerivedPathRaw {
         return static_cast<const Raw &>(*this);
     }
 
+    bool operator == (const SingleDerivedPath &) const = default;
+    auto operator <=> (const SingleDerivedPath &) const = default;
+
     /**
      * Get the store path this is ultimately derived from (by realising
      * and projecting outputs).
@@ -201,7 +206,9 @@ struct DerivedPathBuilt {
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
     nlohmann::json toJSON(Store & store) const;
 
-    DECLARE_CMP(DerivedPathBuilt);
+    bool operator == (const DerivedPathBuilt &) const noexcept;
+    // TODO libc++ 16 (used by darwin) missing `std::set::operator <=>`, can't do yet.
+    bool operator < (const DerivedPathBuilt &) const noexcept;
 };
 
 using _DerivedPathRaw = std::variant<
@@ -230,6 +237,10 @@ struct DerivedPath : _DerivedPathRaw {
         return static_cast<const Raw &>(*this);
     }
 
+    bool operator == (const DerivedPath &) const = default;
+    // TODO libc++ 16 (used by darwin) missing `std::set::operator <=>`, can't do yet.
+    //auto operator <=> (const DerivedPath &) const = default;
+
     /**
      * Get the store path this is ultimately derived from (by realising
      * and projecting outputs).

src/libstore/filetransfer.hh

@@ -1,13 +1,15 @@
 #pragma once
 ///@file
 
-#include "types.hh"
-#include "hash.hh"
-#include "config.hh"
-
 #include <string>
 #include <future>
 
+#include "logging.hh"
+#include "types.hh"
+#include "ref.hh"
+#include "config.hh"
+#include "serialise.hh"
+
 namespace nix {
 
 struct FileTransferSettings : Config

src/libstore/gc-store.hh

@@ -1,8 +1,9 @@
 #pragma once
 ///@file
 
-#include "store-api.hh"
+#include <unordered_set>
 
+#include "store-api.hh"
 
 namespace nix {
 

src/libstore/globals.cc

@@ -35,6 +35,8 @@
 #include <sys/sysctl.h>
 #endif
 
+#include "strings.hh"
+
 namespace nix {
 
 
@@ -82,7 +84,7 @@ Settings::Settings()
         Strings ss;
         for (auto & p : tokenizeString<Strings>(*s, ":"))
             ss.push_back("@" + p);
-        builders = concatStringsSep(" ", ss);
+        builders = concatStringsSep("\n", ss);
     }
 
 #if defined(__linux__) && defined(SANDBOX_SHELL)

src/libstore/legacy-ssh-store.cc

@@ -1,5 +1,5 @@
 #include "legacy-ssh-store.hh"
-#include "ssh-store-config.hh"
+#include "common-ssh-store-config.hh"
 #include "archive.hh"
 #include "pool.hh"
 #include "remote-store.hh"
@@ -15,6 +15,15 @@
 
 namespace nix {
 
+LegacySSHStoreConfig::LegacySSHStoreConfig(
+    std::string_view scheme,
+    std::string_view authority,
+    const Params & params)
+    : StoreConfig(params)
+    , CommonSSHStoreConfig(scheme, authority, params)
+{
+}
+
 std::string LegacySSHStoreConfig::doc()
 {
     return
@@ -35,7 +44,7 @@ LegacySSHStore::LegacySSHStore(
     const Params & params)
     : StoreConfig(params)
     , CommonSSHStoreConfig(scheme, host, params)
-    , LegacySSHStoreConfig(params)
+    , LegacySSHStoreConfig(scheme, host, params)
     , Store(params)
     , connections(make_ref<Pool<Connection>>(
         std::max(1, (int) maxConnections),